1975 files changed, 119745 insertions, 39202 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index 13d0d116ae..134a0c49ff 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -16,7 +16,7 @@ environment:
   - target: i686-w64-mingw32
     compiler_path: mingw32
     mingw_prefix: mingw-w64-i686
-    hardening:
+    hardening: --enable-all-bugs-are-fatal
   - target: x86_64-w64-mingw32
     compiler_path: mingw64
     mingw_prefix: mingw-w64-x86_64
@@ -34,7 +34,7 @@ matrix:
       target: i686-w64-mingw32
       compiler_path: mingw32
       mingw_prefix: mingw-w64-i686
-      hardening:
+      hardening: --enable-all-bugs-are-fatal
 
 install:
 - ps: >-
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000000..7be73e0612
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,163 @@
+# DO NOT COMMIT OR MERGE CODE THAT IS RUN THROUGH THIS TOOL YET.
+#
+# WE ARE STILL DISCUSSING OUR DESIRED STYLE AND ITERATING ON IT.
+#     (12 Feb 2020)
+
+---
+Language:        Cpp
+# Out of all supported styles, LLVM seems closest to our own.
+BasedOnStyle:    LLVM
+
+################
+#
+# Deviations from LLVM's style.
+#
+################
+
+# We prefer an indentation width of 4 columns; LLVM likes 2.
+## OVERRIDE FOR COMPARISON
+IndentWidth:     2
+
+## OVERRIDE FOR COMPARISON
+## for now i'm not sorting includes, since that makes every file get touched.
+SortIncludes: false
+
+# We prefer 79; llvm likes 80.
+ColumnLimit: 79
+
+# Where do we want to put backslashes on multiline macros?  Our choices are
+# "as far left as possible", "as far right as possible", and "make no changes."
+# LLVM defaults to right, but we don't dig that.
+AlignEscapedNewlines: Left
+
+# When we see a bunch of things in a row with comments after them, should we
+# try to align those comments?  Doing so makes some of our code pretty ugly.
+AlignTrailingComments: false
+
+# We use a function declaration style much closer to BSD KNF than to LLVM's.
+# We say:
+#      int foo(int x);
+#      int
+#      foo(int x)
+#      {
+#          ...
+#      }
+# whereas llvm prefers:
+#      int foo(int x);
+#      int foo(int x) {
+#          ...
+#      }
+# or even:
+#      int foo(int x) { ... }
+#
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterFunction:   true
+AllowShortFunctionsOnASingleLine: None
+AlwaysBreakAfterReturnType: AllDefinitions
+
+# We don't like blocks to start with an empty line.
+#
+KeepEmptyLinesAtTheStartOfBlocks: false
+
+################
+#
+# Tor-specific magic
+#
+################
+
+#
+# These comments are magical, and should not be changed.
+#
+CommentPragmas:  'LCOV_EXCL|COVERITY'
+
+#
+# Remove duplicate empty lines.
+#
+MaxEmptyLinesToKeep: 1
+
+#
+# Indent preprocessor directives, for clarity.
+#
+IndentPPDirectives: AfterHash
+
+#
+# These introduce an iteration, and work a bit like a for loop.
+#
+# Note that we can NOT include ones that don't work like "for".  For example,
+# if the body is an argument to the macro, we can't list it here.
+#
+ForEachMacros:
+  - MAP_FOREACH
+  - MAP_FOREACH_MODIFY
+  - TOR_SIMPLEQ_FOREACH
+  - TOR_SIMPLEQ_FOREACH_SAFE
+  - TOR_SLIST_FOREACH
+  - TOR_SLIST_FOREACH_SAFE
+  - TOR_LIST_FOREACH
+  - TOR_LIST_FOREACH_SAFE
+  - TOR_TAILQ_FOREACH
+  - TOR_TAILQ_FOREACH_SAFE
+  - TOR_TAILQ_FOREACH_REVERSE
+  - TOR_TAILQ_FOREACH_REVERSE_SAFE
+  - TOR_CIRCLEQ_FOREACH
+  - TOR_CIRCLEQ_FOREACH_SAFE
+  - TOR_CIRCLEQ_FOREACH_REVERSE
+  - TOR_CIRCLEQ_FOREACH_REVERSE_SAFE
+  - HT_FOREACH
+  - SMARTLIST_FOREACH_BEGIN
+  - DIGESTMAP_FOREACH
+  - DIGESTMAP_FOREACH_MODIFY
+  - DIGEST256MAP_FOREACH
+  - DIGEST256MAP_FOREACH_MODIFY
+  - SDMAP_FOREACH
+  - RIMAP_FOREACH
+  - EIMAP_FOREACH
+
+#
+# Omitting:
+#
+# - SMARTLIST_FOREACH, since the body of the loop is an argument.
+
+#
+# This explains how to sort our headers.
+#
+# This is more complex than it truly should be, but I've edited this till
+# compilation still mostly passes.
+#
+# I'm disabling this, however, since it's a distraction from the other
+# formatting issues. See SortIncludes above.
+#
+IncludeCategories:
+  - Regex:           '^"orconfig.h'
+    Priority:        -30
+  - Regex:           '^"ext/'
+    Priority:        -18
+  - Regex:           '^"lib/'
+    Priority:        -10
+  - Regex:           '^"core/or/or.h'
+    Priority:        -5
+  - Regex:           '^"core/'
+    Priority:        5
+  - Regex:           '^"feature/'
+    Priority:        10
+  - Regex:           '^"app/'
+    Priority:        20
+
+#
+# These macros should always cause indentation, as though they were { and }.
+#
+# Do NOT put macros here unless you want an extra level of indentation between
+# them whenever they appear.
+#
+MacroBlockBegin: "^STMT_BEGIN|TT_STMT_BEGIN$"
+MacroBlockEnd:   "^STMT_END|TT_STMT_END$"
+
+#
+# These macros are interpreted as types.
+# (Not supported in my clang-format)
+#
+# TypenameMacros:
+#    - "STACK_OF"
+
+...
diff --git a/.gitignore b/.gitignore
index 736abbedc3..53d7e5715b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@
 *.swo
 # C stuff
 *.o
+*.a
 *.obj
 # Diff droppings
 *.orig
@@ -84,13 +85,10 @@ core.*
 /coverage_html/
 /callgraph/
 
-
 # /contrib/
-/contrib/dist/tor.sh
 /contrib/dist/torctl
 /contrib/dist/tor.service
 /contrib/operator-tools/tor.logrotate
-/contrib/dist/suse/tor.sh
 
 # /debian/
 /debian/files
@@ -144,9 +142,6 @@ core.*
 /src/Makefile
 /src/Makefile.in
 
-# /src/trace
-/src/trace/libor-trace.a
-
 # /src/config/
 /src/config/Makefile
 /src/config/Makefile.in
@@ -156,77 +151,10 @@ core.*
 /src/config/torrc.minimal
 
 # /src/ext/
-/src/ext/ed25519/ref10/libed25519_ref10.a
 /src/ext/ed25519/ref10/libed25519_ref10.lib
-/src/ext/ed25519/donna/libed25519_donna.a
 /src/ext/ed25519/donna/libed25519_donna.lib
-/src/ext/keccak-tiny/libkeccak-tiny.a
 /src/ext/keccak-tiny/libkeccak-tiny.lib
 
-# /src/lib
-/src/lib/libcurve25519_donna.a
-/src/lib/libtor-compress.a
-/src/lib/libtor-compress-testing.a
-/src/lib/libtor-container.a
-/src/lib/libtor-container-testing.a
-/src/lib/libtor-crypt-ops.a
-/src/lib/libtor-crypt-ops-testing.a
-/src/lib/libtor-ctime.a
-/src/lib/libtor-ctime-testing.a
-/src/lib/libtor-encoding.a
-/src/lib/libtor-encoding-testing.a
-/src/lib/libtor-evloop.a
-/src/lib/libtor-evloop-testing.a
-/src/lib/libtor-err.a
-/src/lib/libtor-err-testing.a
-/src/lib/libtor-fdio.a
-/src/lib/libtor-fdio-testing.a
-/src/lib/libtor-fs.a
-/src/lib/libtor-fs-testing.a
-/src/lib/libtor-geoip.a
-/src/lib/libtor-geoip-testing.a
-/src/lib/libtor-intmath.a
-/src/lib/libtor-intmath-testing.a
-/src/lib/libtor-lock.a
-/src/lib/libtor-lock-testing.a
-/src/lib/libtor-log.a
-/src/lib/libtor-log-testing.a
-/src/lib/libtor-malloc.a
-/src/lib/libtor-malloc-testing.a
-/src/lib/libtor-math.a
-/src/lib/libtor-math-testing.a
-/src/lib/libtor-memarea.a
-/src/lib/libtor-memarea-testing.a
-/src/lib/libtor-meminfo.a
-/src/lib/libtor-meminfo-testing.a
-/src/lib/libtor-net.a
-/src/lib/libtor-net-testing.a
-/src/lib/libtor-osinfo.a
-/src/lib/libtor-osinfo-testing.a
-/src/lib/libtor-process.a
-/src/lib/libtor-process-testing.a
-/src/lib/libtor-sandbox.a
-/src/lib/libtor-sandbox-testing.a
-/src/lib/libtor-string.a
-/src/lib/libtor-string-testing.a
-/src/lib/libtor-smartlist-core.a
-/src/lib/libtor-smartlist-core-testing.a
-/src/lib/libtor-term.a
-/src/lib/libtor-term-testing.a
-/src/lib/libtor-thread.a
-/src/lib/libtor-thread-testing.a
-/src/lib/libtor-time.a
-/src/lib/libtor-time-testing.a
-/src/lib/libtor-tls.a
-/src/lib/libtor-tls-testing.a
-/src/lib/libtor-trace.a
-/src/lib/libtor-wallclock.a
-/src/lib/libtor-wallclock-testing.a
-
-# /src/tor
-/src/core/libtor-app.a
-/src/core/libtor-app-testing.a
-
 # /src/app
 /src/app/tor
 /src/app/tor.exe
@@ -247,20 +175,22 @@ core.*
 /src/test/test
 /src/test/test-slow
 /src/test/test-bt-cl
-/src/test/test-child
+/src/test/test-process
 /src/test/test-memwipe
 /src/test/test-ntor-cl
 /src/test/test-hs-ntor-cl
+/src/test/test-rng
 /src/test/test-switch-id
 /src/test/test-timers
 /src/test/test_workqueue
 /src/test/test.exe
 /src/test/test-slow.exe
 /src/test/test-bt-cl.exe
-/src/test/test-child.exe
+/src/test/test-process.exe
 /src/test/test-ntor-cl.exe
 /src/test/test-hs-ntor-cl.exe
 /src/test/test-memwipe.exe
+/src/test/test-rng.exe
 /src/test/test-switch-id.exe
 /src/test/test-timers.exe
 /src/test/test_workqueue.exe
@@ -270,7 +200,6 @@ core.*
 /src/test/fuzz/lf-fuzz-*
 
 # /src/tools/
-/src/tools/libtorrunner.a
 /src/tools/tor-checkkey
 /src/tools/tor-resolve
 /src/tools/tor-cov-resolve
@@ -286,10 +215,6 @@ core.*
 /src/tools/Makefile
 /src/tools/Makefile.in
 
-# /src/trunnel/
-/src/trunnel/libor-trunnel-testing.a
-/src/trunnel/libor-trunnel.a
-
 # /src/win32/
 /src/win32/Makefile
 /src/win32/Makefile.in
diff --git a/.travis.yml b/.travis.yml
index 8bc9898f8f..01343e65d9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,12 +26,16 @@ env:
     - MAKEFLAGS="-k -j 2"
     ## We turn on hardening by default
     ## Also known as --enable-fragile-hardening in 0.3.0.3-alpha and later
-    - HARDENING_OPTIONS="--enable-expensive-hardening"
+    - HARDENING_OPTIONS="--enable-all-bugs-are-fatal --enable-expensive-hardening"
     ## We turn off asciidoc by default, because it's slow
     - ASCIIDOC_OPTIONS="--disable-asciidoc"
+    ## Our default rust version is the minimum supported version
+    - RUST_VERSION="1.31.0"
     ## Turn off tor's sandbox in chutney, until we fix sandbox errors that are
     ## triggered by Ubuntu Xenial and Bionic. See #32722.
     - CHUTNEY_TOR_SANDBOX="0"
+    ## The default target for chutney jobs
+    - CHUTNEY_MAKE="test-network-all"
   matrix:
     ## This matrix entry is required, but it doesn't actually create any jobs
     -
@@ -40,14 +44,16 @@ matrix:
   ## include creates builds with gcc, linux, unless we override those defaults
   include:
     ## We run chutney on macOS, because macOS Travis has IPv6
-    - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
+    ## But we only run the IPv6 chutney tests, to speed up the job
+    - env: CHUTNEY_MAKE="test-network-ipv6" CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
       os: osx
 
     ## We also run basic tests on macOS
     - compiler: clang
       os: osx
       ## Turn off some newer features, turn on clang's -Wtypedef-redefinition
-      env: C_DIALECT_OPTIONS="-std=gnu99"
+      ## Also, disable ALL_BUGS_ARE_FATAL macro.
+      env: C_DIALECT_OPTIONS="-std=gnu99" HARDENING_OPTIONS="--enable-expensive-hardening"
 
     ## We run chutney on Linux, because it's faster than chutney on macOS
     ## Chutney is a fast job, clang is slower on Linux, so we do Chutney clang
@@ -57,8 +63,10 @@ matrix:
     ## We check asciidoc with distcheck, to make sure we remove doc products
     - env: DISTCHECK="yes" ASCIIDOC_OPTIONS="" SKIP_MAKE_CHECK="yes"
 
+    ## We check disable module relay
+    - env: MODULES_OPTIONS="--disable-module-relay" HARDENING_OPTIONS="--enable-expensive-hardening"
     ## We check disable module dirauth
-    - env: MODULES_OPTIONS="--disable-module-dirauth"
+    - env: MODULES_OPTIONS="--disable-module-dirauth" HARDENING_OPTIONS="--enable-expensive-hardening"
 
     ## We run rust on Linux, because it's faster than rust on macOS
     ## We check rust offline
@@ -67,19 +75,22 @@ matrix:
     ## We check NSS
     ## Use -std=gnu99 to turn off some newer features, and maybe turn on some
     ## extra gcc warnings?
-    - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99"
+    - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99" HARDENING_OPTIONS="--enable-expensive-hardening"
 
     ## We include a single coverage build with the best options for coverage
-    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS=""
+    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765"
 
     ## We clone our stem repo and run `make test-stem`
     - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
 
+    ## We run `make doxygen` without `make check`.
+    - env: SKIP_MAKE_CHECK="yes" DOXYGEN="yes"
+
     ## macOS builds are very slow, and we have a limited number of
     ## concurrent macOS jobs. We're not actively developing Rust, so it is
     ## the lowest priority.
     ## We run rust on macOS, because we have seen macOS rust failures before
-    #- env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+    #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
     #  compiler: clang
     #  os: osx
 
@@ -96,13 +107,13 @@ matrix:
 
     ## Since this job is disabled, there's not much point having an exception
     ## for it
-    #- env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+    #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
     #  compiler: clang
     #  os: osx
 
     ## Since we're actively developing IPv6, we want to require the IPv6
     ## chutney tests
-    #- env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
+    #- env: CHUTNEY_MAKE="test-network-ipv6" CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
     #  os: osx
 
 ## (Linux only) Use a recent Linux image (Ubuntu Bionic)
@@ -125,12 +136,16 @@ addons:
       - libscrypt-dev
       - libseccomp-dev
       - libzstd-dev
+      ## Optional build dependencies
+      - coccinelle
+      - shellcheck
       ## Conditional build dependencies
       ## Always installed, so we don't need sudo
       - asciidoc
       - docbook-xsl
       - docbook-xml
       - xmlto
+      - doxygen
       ## Utilities
       ## preventing or diagnosing hangs
       - timelimit
@@ -153,6 +168,8 @@ addons:
       - pkg-config
       ## Optional build dependencies
       - ccache
+      - coccinelle
+      - shellcheck
       ## Conditional build dependencies
       ## Always installed, because manual brew installs are hard to get right
       - asciidoc
@@ -168,6 +185,8 @@ addons:
 osx_image: xcode11.2
 
 before_install:
+  ## Set pipefail: we use pipes
+  - set -o pipefail || echo "pipefail failed"
   ## Create empty rust directories for non-Rust builds, so caching succeeds
   - if [[ "$RUST_OPTIONS" == "" ]]; then mkdir -p $HOME/.cargo $TRAVIS_BUILD_DIR/src/rust/target; fi
 
@@ -183,8 +202,8 @@ install:
   - if [[ "$ASCIIDOC_OPTIONS" == "" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export XML_CATALOG_FILES="/usr/local/etc/xml/catalog"; fi
   ## If we're using Rust, download rustup
   - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
-  ## Install the nightly channels of rustc and cargo and setup our toolchain environment
-  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain nightly; fi
+  ## Install the stable channels of rustc and cargo and setup our toolchain environment
+  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain $RUST_VERSION; fi
   - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
   ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
   - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
@@ -209,6 +228,13 @@ install:
   - if [[ "$CHUTNEY" != "" ]]; then pushd "$CHUTNEY_PATH"; git log -1 ; popd ; fi
   ## If we're running stem, show the stem version and commit
   - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi
+  ## Get the coccinelle version
+  ## Installs are unreliable on macOS, so we just rely on brew list --versions
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then spatch --version; fi
+  ## We don't want Tor tests to depend on default configuration file at
+  ## ~/.torrc. So we put some random bytes in there, to make sure we get build
+  ## failures in case Tor is reading it during CI jobs.
+  - dd ibs=1 count=1024 if=/dev/urandom > ~/.torrc
 
 script:
   # Skip test_rebind on macOS
@@ -216,14 +242,15 @@ script:
   - ./autogen.sh
   - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $NSS_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules"
   - echo "Configure flags are $CONFIGURE_FLAGS CC=\"$CC $C_DIALECT_OPTIONS\""
-  - ./configure $CONFIGURE_FLAGS CC="$CC $C_DIALECT_OPTIONS"
+  - ./configure $CONFIGURE_FLAGS CC="$CC $C_DIALECT_OPTIONS";
   ## We run `make check` because that's what https://jenkins.torproject.org does.
   - if [[ "$SKIP_MAKE_CHECK" == "" ]]; then make check; fi
   - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi
-  - if [[ "$CHUTNEY" != "" ]]; then make test-network-all; fi
+  - if [[ "$CHUTNEY" != "" ]]; then make "$CHUTNEY_MAKE"; fi
   ## Diagnostic for bug 29437: kill stem if it hangs for 9.5 minutes
   ## Travis will kill the job after 10 minutes with no output
   - if [[ "$TEST_STEM" != "" ]]; then make src/app/tor; timelimit -p -t 540 -s USR1 -T 30 -S ABRT python3 "$STEM_SOURCE_DIR"/run_tests.py --tor src/app/tor --integ --test control.controller --test control.base_controller --test process --log TRACE --log-file stem.log; fi
+  - if [[ "$DOXYGEN" != "" ]]; then make doxygen; fi
   ## If this build was one that produced coverage, upload it.
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then coveralls -b . --exclude src/test --exclude src/trunnel --gcov-options '\-p' || echo "Coverage failed"; fi
 
diff --git a/ChangeLog b/ChangeLog
index cdf7249059..9ddbfa95fd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,5048 @@
+Changes in version 0.4.3.5 - 2020-05-15
+  Tor 0.4.3.5 is the first stable release in the 0.4.3.x series. This
+  series adds support for building without relay code enabled, and
+  implements functionality needed for OnionBalance with v3 onion
+  services. It includes significant refactoring of our configuration and
+  controller functionality, and fixes numerous smaller bugs and
+  performance issues.
+
+  Per our support policy, we support each stable release series for nine
+  months after its first stable release, or three months after the first
+  stable release of the next series: whichever is longer. This means
+  that 0.4.3.x will be supported until around February 2021--later, if
+  0.4.4.x is later than anticipated.
+
+  Note also that support for 0.4.1.x is about to end on May 20 of this
+  year; 0.4.2.x will be supported until September 15. We still plan to
+  continue supporting 0.3.5.x, our long-term stable series, until
+  Feb 2022.
+
+  Below are the changes since 0.4.3.4-rc. For a complete list of changes
+  since 0.4.2.6, see the ReleaseNotes file.
+
+  o Minor bugfixes (compiler compatibility):
+    - Avoid compiler warnings from Clang 10 related to the use of GCC-
+      style "/* falls through */" comments. Both Clang and GCC allow
+      __attribute__((fallthrough)) instead, so that's what we're using
+      now. Fixes bug 34078; bugfix on 0.3.1.3-alpha.
+    - Fix compilation warnings with GCC 10.0.1. Fixes bug 34077; bugfix
+      on 0.4.0.3-alpha.
+
+  o Minor bugfixes (logging):
+    - Stop truncating IPv6 addresses and ports in channel and connection
+      logs. Fixes bug 33918; bugfix on 0.2.4.4-alpha.
+    - Fix a logic error in a log message about whether an address was
+      invalid. Previously, the code would never report that onion
+      addresses were onion addresses. Fixes bug 34131; bugfix
+      on 0.4.3.1-alpha.
+
+
+Changes in version 0.4.3.4-rc - 2020-04-13
+  Tor 0.4.3.4-rc is the first release candidate in its series. It fixes
+  several bugs from earlier versions, including one affecting DoS
+  defenses on bridges using pluggable transports.
+
+  o Major bugfixes (DoS defenses, bridges, pluggable transport):
+    - Fix a bug that was preventing DoS defenses from running on bridges
+      with a pluggable transport. Previously, the DoS subsystem was not
+      given the transport name of the client connection, thus failed to
+      find the GeoIP cache entry for that client address. Fixes bug
+      33491; bugfix on 0.3.3.2-alpha.
+
+  o Minor feature (sendme, flow control):
+    - Default to sending SENDME version 1 cells. (Clients are already
+      sending these, because of a consensus parameter telling them to do
+      so: this change only affects what clients would do if the
+      consensus didn't contain a recommendation.) Closes ticket 33623.
+
+  o Minor features (testing):
+    - The unit tests now support a "TOR_SKIP_TESTCASES" environment
+      variable to specify a list of space-separated test cases that
+      should not be executed. We will use this to disable certain tests
+      that are failing on Appveyor because of mismatched OpenSSL
+      libraries. Part of ticket 33643.
+
+  o Minor bugfixes (--disable-module-relay):
+    - Fix an assertion failure when Tor is built without the relay
+      module, and then invoked with the "User" option. Fixes bug 33668;
+      bugfix on 0.4.3.1-alpha.
+
+  o Minor bugfixes (--disable-module-relay,--disable-module-dirauth):
+    - Set some output arguments in the relay and dirauth module stubs,
+      to guard against future stub argument handling bugs like 33668.
+      Fixes bug 33674; bugfix on 0.4.3.1-alpha.
+
+  o Minor bugfixes (build system):
+    - Correctly output the enabled module in the configure summary.
+      Before that, the list shown was just plain wrong. Fixes bug 33646;
+      bugfix on 0.4.3.2-alpha.
+
+  o Minor bugfixes (client, IPv6):
+    - Stop forcing all non-SocksPorts to prefer IPv6 exit connections.
+      Instead, prefer IPv6 connections by default, but allow users to
+      change their configs using the "NoPreferIPv6" port flag. Fixes bug
+      33608; bugfix on 0.4.3.1-alpha.
+    - Revert PreferIPv6 set by default on the SocksPort because it broke
+      the torsocks use case. Tor doesn't have a way for an application
+      to request the hostname to be resolved for a specific IP version,
+      but torsocks requires that. Up until now, IPv4 was used by default
+      so torsocks is expecting that, and can't handle a possible IPv6
+      being returned. Fixes bug 33804; bugfix on 0.4.3.1-alpha.
+
+  o Minor bugfixes (key portability):
+    - When reading PEM-encoded key data, tolerate CRLF line-endings even
+      if we are not running on Windows. Previously, non-Windows hosts
+      would reject these line-endings in certain positions, making
+      certain key files hard to move from one host to another. Fixes bug
+      33032; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Flush stderr, stdout, and file logs during shutdown, if supported
+      by the OS. This change helps make sure that any final logs are
+      recorded. Fixes bug 33087; bugfix on 0.4.1.6.
+    - Stop closing stderr and stdout during shutdown. Closing these file
+      descriptors can hide sanitiser logs. Fixes bug 33087; bugfix
+      on 0.4.1.6.
+
+  o Minor bugfixes (onion services v3):
+    - Relax severity of a log message that can appear naturally when
+      decoding onion service descriptors as a relay. Also add some
+      diagnostics to debug any future bugs in that area. Fixes bug
+      31669; bugfix on 0.3.0.1-alpha.
+    - Block a client-side assertion by disallowing the registration of
+      an x25519 client auth key that's all zeroes. Fixes bug 33545;
+      bugfix on 0.4.3.1-alpha. Based on patch from "cypherpunks".
+
+  o Code simplification and refactoring:
+    - Disable our coding standards best practices tracker in our git
+      hooks. (0.4.3 branches only.) Closes ticket 33678.
+
+  o Testing:
+    - Avoid conflicts between the fake sockets in tor's unit tests, and
+      real file descriptors. Resolves issues running unit tests with
+      GitHub Actions, where the process that embeds or launches the
+      tests has already opened a large number of file descriptors. Fixes
+      bug 33782; bugfix on 0.2.8.1-alpha. Found and fixed by
+      Putta Khunchalee.
+
+  o Testing (CI):
+    - In our Appveyor Windows CI, copy required DLLs to test and app
+      directories, before running tor's tests. This ensures that tor.exe
+      and test*.exe use the correct version of each DLL. This fix is not
+      required, but we hope it will avoid DLL search issues in future.
+      Fixes bug 33673; bugfix on 0.3.4.2-alpha.
+    - On Appveyor, skip the crypto/openssl_version test, which is
+      failing because of a mismatched library installation. Fix
+      for 33643.
+
+
+Changes in version 0.4.3.3-alpha - 2020-03-18
+  Tor 0.4.3.3-alpha fixes several bugs in previous releases, including
+  TROVE-2020-002, a major denial-of-service vulnerability that affected
+  all released Tor instances since 0.2.1.5-alpha. Using this
+  vulnerability, an attacker could cause Tor instances to consume a huge
+  amount of CPU, disrupting their operations for several seconds or
+  minutes. This attack could be launched by anybody against a relay, or
+  by a directory cache against any client that had connected to it. The
+  attacker could launch this attack as much as they wanted, thereby
+  disrupting service or creating patterns that could aid in traffic
+  analysis. This issue was found by OSS-Fuzz, and is also tracked
+  as CVE-2020-10592.
+
+  We do not have reason to believe that this attack is currently being
+  exploited in the wild, but nonetheless we advise everyone to upgrade
+  as soon as packages are available.
+
+  o Major bugfixes (security, denial-of-service):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (circuit padding, memory leak):
+    - Avoid a remotely triggered memory leak in the case that a circuit
+      padding machine is somehow negotiated twice on the same circuit.
+      Fixes bug 33619; bugfix on 0.4.0.1-alpha. Found by Tobias Pulls.
+      This is also tracked as TROVE-2020-004 and CVE-2020-10593.
+
+  o Major bugfixes (directory authority):
+    - Directory authorities will now send a 503 (not enough bandwidth)
+      code to clients when under bandwidth pressure. Known relays and
+      other authorities will always be answered regardless of the
+      bandwidth situation. Fixes bug 33029; bugfix on 0.1.2.5-alpha.
+
+  o Minor features (diagnostic):
+    - Improve assertions and add some memory-poisoning code to try to
+      track down possible causes of a rare crash (32564) in the EWMA
+      code. Closes ticket 33290.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors from relays running
+      Tor versions from the 0.2.9 and 0.4.0 series. The 0.3.5 series is
+      still allowed. Resolves ticket 32672. Patch by Neel Chauhan.
+
+  o Minor features (usability):
+    - Include more information when failing to parse a configuration
+      value. This should make it easier to tell what's going wrong when
+      a configuration file doesn't parse. Closes ticket 33460.
+
+  o Minor bugfix (relay, configuration):
+    - Warn if the ContactInfo field is not set, and tell the relay
+      operator that not having a ContactInfo field set might cause their
+      relay to get rejected in the future. Fixes bug 33361; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (coding best practices checks):
+    - Allow the "practracker" script to read unicode files when using
+      Python 2. We made the script use unicode literals in 0.4.3.1-alpha,
+      but didn't change the codec for opening files. Fixes bug 33374;
+      bugfix on 0.4.3.1-alpha.
+
+  o Minor bugfixes (continuous integration):
+    - Remove the buggy and unused mirroring job. Fixes bug 33213; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (onion service v3, client):
+    - Remove a BUG() warning that would cause a stack trace if an onion
+      service descriptor was freed while we were waiting for a
+      rendezvous circuit to complete. Fixes bug 28992; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services v3):
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+
+  o Documentation (manpage):
+    - Alphabetize the Server and Directory server sections of the tor
+      manpage. Also split Statistics options into their own section of
+      the manpage. Closes ticket 33188. Work by Swati Thacker as part of
+      Google Season of Docs.
+    - Document the __OwningControllerProcess torrc option and specify
+      its polling interval. Resolves issue 32971.
+
+  o Testing (Travis CI):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+      allow_failure), to speed up the build. Closes ticket 33195.
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.4.2.7 - 2020-03-18
+  This is the third stable release in the 0.4.2.x series. It backports
+  numerous fixes from later releases, including a fix for TROVE-2020-
+  002, a major denial-of-service vulnerability that affected all
+  released Tor instances since 0.2.1.5-alpha. Using this vulnerability,
+  an attacker could cause Tor instances to consume a huge amount of CPU,
+  disrupting their operations for several seconds or minutes. This
+  attack could be launched by anybody against a relay, or by a directory
+  cache against any client that had connected to it. The attacker could
+  launch this attack as much as they wanted, thereby disrupting service
+  or creating patterns that could aid in traffic analysis. This issue
+  was found by OSS-Fuzz, and is also tracked as CVE-2020-10592.
+
+  We do not have reason to believe that this attack is currently being
+  exploited in the wild, but nonetheless we advise everyone to upgrade
+  as soon as packages are available.
+
+  o Major bugfixes (security, denial-of-service, backport from 0.4.3.3-alpha):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (circuit padding, memory leak, backport from 0.4.3.3-alpha):
+    - Avoid a remotely triggered memory leak in the case that a circuit
+      padding machine is somehow negotiated twice on the same circuit.
+      Fixes bug 33619; bugfix on 0.4.0.1-alpha. Found by Tobias Pulls.
+      This is also tracked as TROVE-2020-004 and CVE-2020-10593.
+
+  o Major bugfixes (directory authority, backport from 0.4.3.3-alpha):
+    - Directory authorities will now send a 503 (not enough bandwidth)
+      code to clients when under bandwidth pressure. Known relays and
+      other authorities will always be answered regardless of the
+      bandwidth situation. Fixes bug 33029; bugfix on 0.1.2.5-alpha.
+
+  o Minor features (continuous integration, backport from 0.4.3.2-alpha):
+    - Stop allowing failures on the Travis CI stem tests job. It looks
+      like all the stem hangs we were seeing before are now fixed.
+      Closes ticket 33075.
+
+  o Minor bugfixes (bridges, backport from 0.4.3.1-alpha):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.3.2-alpha):
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (onion services v3, backport from 0.4.3.3-alpha):
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+
+  o Minor bugfixes (rust, build, backport from 0.4.3.2-alpha):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Testing (Travis CI, backport from 0.4.3.3-alpha):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+      allow_failure), to speed up the build. Closes ticket 33195.
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.4.1.9 - 2020-03-18
+  Tor 0.4.1.9 backports important fixes from later Tor releases,
+  including a fix for TROVE-2020-002, a major denial-of-service
+  vulnerability that affected all released Tor instances since
+  0.2.1.5-alpha. Using this vulnerability, an attacker could cause Tor
+  instances to consume a huge amount of CPU, disrupting their operations
+  for several seconds or minutes. This attack could be launched by
+  anybody against a relay, or by a directory cache against any client
+  that had connected to it. The attacker could launch this attack as
+  much as they wanted, thereby disrupting service or creating patterns
+  that could aid in traffic analysis. This issue was found by OSS-Fuzz,
+  and is also tracked as CVE-2020-10592.
+
+  We do not have reason to believe that this attack is currently being
+  exploited in the wild, but nonetheless we advise everyone to upgrade
+  as soon as packages are available.
+
+  o Major bugfixes (security, denial-of-service, backport from 0.4.3.3-alpha):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (circuit padding, memory leak, backport from 0.4.3.3-alpha):
+    - Avoid a remotely triggered memory leak in the case that a circuit
+      padding machine is somehow negotiated twice on the same circuit.
+      Fixes bug 33619; bugfix on 0.4.0.1-alpha. Found by Tobias Pulls.
+      This is also tracked as TROVE-2020-004 and CVE-2020-10593.
+
+  o Minor bugfixes (bridges, backport from 0.4.3.1-alpha):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.3.2-alpha):
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (onion services v3, backport from 0.4.3.3-alpha):
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+
+  o Minor bugfixes (rust, build, backport from 0.4.3.2-alpha):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Testing (Travis CI, backport from 0.4.3.3-alpha):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+      allow_failure), to speed up the build. Closes ticket 33195.
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.3.5.10 - 2020-03-18
+  Tor 0.3.5.10 backports many fixes from later Tor releases, including a
+  fix for TROVE-2020-002, a major denial-of-service vulnerability that
+  affected all released Tor instances since 0.2.1.5-alpha. Using this
+  vulnerability, an attacker could cause Tor instances to consume a huge
+  amount of CPU, disrupting their operations for several seconds or
+  minutes. This attack could be launched by anybody against a relay, or
+  by a directory cache against any client that had connected to it. The
+  attacker could launch this attack as much as they wanted, thereby
+  disrupting service or creating patterns that could aid in traffic
+  analysis. This issue was found by OSS-Fuzz, and is also tracked
+  as CVE-2020-10592.
+
+  We do not have reason to believe that this attack is currently being
+  exploited in the wild, but nonetheless we advise everyone to upgrade
+  as soon as packages are available.
+
+  o Major bugfixes (security, denial-of-service, backport from 0.4.3.3-alpha):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (linux seccomp sandbox, backport from 0.4.3.1-alpha):
+    - Correct how we use libseccomp. Particularly, stop assuming that
+      rules are applied in a particular order or that more rules are
+      processed after the first match. Neither is the case! In
+      libseccomp <2.4.0 this lead to some rules having no effect.
+      libseccomp 2.4.0 changed how rules are generated, leading to a
+      different ordering, which in turn led to a fatal crash during
+      startup. Fixes bug 29819; bugfix on 0.2.5.1-alpha. Patch by
+      Peter Gerber.
+
+  o Minor features (continuous integration, backport from 0.4.3.2-alpha):
+    - Stop allowing failures on the Travis CI stem tests job. It looks
+      like all the stem hangs we were seeing before are now fixed.
+      Closes ticket 33075.
+
+  o Minor bugfixes (bridges, backport from 0.4.3.1-alpha):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (crash, backport from 0.4.2.4-rc):
+    - When running Tor with an option like --verify-config or
+      --dump-config that does not start the event loop, avoid crashing
+      if we try to exit early because of an error. Fixes bug 32407;
+      bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.3.2-alpha):
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (onion services v3, backport from 0.4.3.3-alpha):
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+
+  o Minor bugfixes (rust, build, backport from 0.4.3.2-alpha):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Testing (backport from 0.4.3.1-alpha):
+    - Re-enable the Travis CI macOS Chutney build, but don't let it
+      prevent the Travis job from finishing. (The Travis macOS jobs are
+      slow, so we don't want to have it delay the whole CI process.)
+      Closes ticket 32629.
+    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on
+      Ubuntu Bionic. Turning off the Sandbox is a work-around, until we
+      fix the sandbox errors in 32722. Closes ticket 32240.
+
+  o Testing (continuous integration, backport from 0.4.3.1-alpha):
+    - Use zstd in our Travis Linux builds. Closes ticket 32242.
+
+  o Testing (Travis CI, backport from 0.4.3.3-alpha):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.4.3.2-alpha - 2020-02-10
+  This is the second stable alpha release in the Tor 0.4.3.x series. It
+  fixes several bugs present in the previous alpha release. Anybody
+  running the previous alpha should upgrade, and look for bugs in this
+  one instead.
+
+  o Major bugfixes (onion service client, authorization):
+    - On a NEWNYM signal, purge entries from the ephemeral client
+      authorization cache. The permanent ones are kept. Fixes bug 33139;
+      bugfix on 0.4.3.1-alpha.
+
+  o Minor features (best practices tracker):
+    - Practracker now supports a --regen-overbroad option to regenerate
+      the exceptions file, but only to revise exceptions to be _less_
+      tolerant of best-practices violations. Closes ticket 32372.
+
+  o Minor features (continuous integration):
+    - Run Doxygen Makefile target on Travis, so we can learn about
+      regressions in our internal documentation. Closes ticket 32455.
+    - Stop allowing failures on the Travis CI stem tests job. It looks
+      like all the stem hangs we were seeing before are now fixed.
+      Closes ticket 33075.
+
+  o Minor bugfixes (build system):
+    - Revise configure options that were either missing or incorrect in
+      the configure summary. Fixes bug 32230; bugfix on 0.4.3.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix a memory leak introduced by refactoring of control reply
+      formatting code. Fixes bug 33039; bugfix on 0.4.3.1-alpha.
+    - Fix a memory leak in GETINFO responses. Fixes bug 33103; bugfix
+      on 0.4.3.1-alpha.
+    - When receiving "ACTIVE" or "DORMANT" signals on the control port,
+      report them as SIGNAL events. Previously we would log a bug
+      warning. Fixes bug 33104; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (logging):
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+    - When logging a bug, do not say "Future instances of this warning
+      will be silenced" unless we are actually going to silence them.
+      Previously we would say this whenever a BUG() check failed in the
+      code. Fixes bug 33095; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (onion service v2):
+    - Move a series of v2 onion service warnings to protocol-warning
+      level because they can all be triggered remotely by a malformed
+      request. Fixes bug 32706; bugfix on 0.1.1.14-alpha.
+
+  o Minor bugfixes (onion service v3, client authorization):
+    - When removing client authorization credentials using the control
+      port, also remove the associated descriptor, so the onion service
+      can no longer be contacted. Fixes bug 33148; bugfix
+      on 0.4.3.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - When receiving a message on standard error from a pluggable
+      transport, log it at info level, rather than as a warning. Fixes
+      bug 33005; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (rust, build):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (TLS bug handling):
+    - When encountering a bug in buf_read_from_tls(), return a "MISC"
+      error code rather than "WANTWRITE". This change might help avoid
+      some CPU-wasting loops if the bug is ever triggered. Bug reported
+      by opara. Fixes bug 32673; bugfix on 0.3.0.4-alpha.
+
+  o Code simplification and refactoring (mainloop):
+    - Simplify the ip_address_changed() function by removing redundant
+      checks. Closes ticket 33091.
+
+  o Documentation (manpage):
+    - Split "Circuit Timeout" options and "Node Selection" options into
+      their own sections of the tor manpage. Closes tickets 32928 and
+      32929. Work by Swati Thacker as part of Google Season of Docs.
+
+
+Changes in version 0.4.2.6 - 2020-01-30
+  This is the second stable release in the 0.4.2.x series. It backports
+  several bugfixes from 0.4.3.1-alpha, including some that had affected
+  the Linux seccomp2 sandbox or Windows services. If you're running with
+  one of those configurations, you'll probably want to upgrade;
+  otherwise, you should be fine with 0.4.2.5.
+
+  o Major bugfixes (linux seccomp sandbox, backport from 0.4.3.1-alpha):
+    - Correct how we use libseccomp. Particularly, stop assuming that
+      rules are applied in a particular order or that more rules are
+      processed after the first match. Neither is the case! In
+      libseccomp <2.4.0 this led to some rules having no effect.
+      libseccomp 2.4.0 changed how rules are generated, leading to a
+      different ordering, which in turn led to a fatal crash during
+      startup. Fixes bug 29819; bugfix on 0.2.5.1-alpha. Patch by
+      Peter Gerber.
+    - Fix crash when reloading logging configuration while the
+      experimental sandbox is enabled. Fixes bug 32841; bugfix on
+      0.4.1.7. Patch by Peter Gerber.
+
+  o Minor bugfixes (correctness checks, backport from 0.4.3.1-alpha):
+    - Use GCC/Clang's printf-checking feature to make sure that
+      tor_assertf() arguments are correctly typed. Fixes bug 32765;
+      bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (logging, crash, backport from 0.4.3.1-alpha):
+    - Avoid a possible crash when trying to log a (fatal) assertion
+      failure about mismatched magic numbers in configuration objects.
+      Fixes bug 32771; bugfix on 0.4.2.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.3.1-alpha):
+    - When TOR_DISABLE_PRACTRACKER is set, do not apply it to the
+      test_practracker.sh script. Doing so caused a test failure. Fixes
+      bug 32705; bugfix on 0.4.2.1-alpha.
+    - When TOR_DISABLE_PRACTRACKER is set, log a notice to stderr when
+      skipping practracker checks. Fixes bug 32705; bugfix
+      on 0.4.2.1-alpha.
+
+  o Minor bugfixes (windows service, backport from 0.4.3.1-alpha):
+    - Initialize the publish/subscribe system when running as a windows
+      service. Fixes bug 32778; bugfix on 0.4.1.1-alpha.
+
+  o Testing (backport from 0.4.3.1-alpha):
+    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on
+      Ubuntu Bionic. Turning off the Sandbox is a work-around, until we
+      fix the sandbox errors in 32722. Closes ticket 32240.
+    - Re-enable the Travis CI macOS Chutney build, but don't let it
+      prevent the Travis job from finishing. (The Travis macOS jobs are
+      slow, so we don't want to have it delay the whole CI process.)
+      Closes ticket 32629.
+
+  o Testing (continuous integration, backport from 0.4.3.1-alpha):
+    - Use zstd in our Travis Linux builds. Closes ticket 32242.
+
+
+Changes in version 0.4.1.8 - 2020-01-30
+  This release backports several bugfixes from later release series,
+  including some that had affected the Linux seccomp2 sandbox or Windows
+  services. If you're running with one of those configurations, you'll
+  probably want to upgrade; otherwise, you should be fine with your
+  current version of 0.4.1.x.
+
+  o Major bugfixes (linux seccomp sandbox, backport from 0.4.3.1-alpha):
+    - Correct how we use libseccomp. Particularly, stop assuming that
+      rules are applied in a particular order or that more rules are
+      processed after the first match. Neither is the case! In
+      libseccomp <2.4.0 this led to some rules having no effect.
+      libseccomp 2.4.0 changed how rules are generated, leading to a
+      different ordering, which in turn led to a fatal crash during
+      startup. Fixes bug 29819; bugfix on 0.2.5.1-alpha. Patch by
+      Peter Gerber.
+    - Fix crash when reloading logging configuration while the
+      experimental sandbox is enabled. Fixes bug 32841; bugfix on
+      0.4.1.7. Patch by Peter Gerber.
+
+  o Minor bugfixes (crash, backport form 0.4.2.4-rc):
+    - When running Tor with an option like --verify-config or
+      --dump-config that does not start the event loop, avoid crashing
+      if we try to exit early because of an error. Fixes bug 32407;
+      bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (windows service, backport from 0.4.3.1-alpha):
+    - Initialize the publish/subscribe system when running as a windows
+      service. Fixes bug 32778; bugfix on 0.4.1.1-alpha.
+
+  o Testing (backport from 0.4.3.1-alpha):
+    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on
+      Ubuntu Bionic. Turning off the Sandbox is a work-around, until we
+      fix the sandbox errors in 32722. Closes ticket 32240.
+    - Re-enable the Travis CI macOS Chutney build, but don't let it
+      prevent the Travis job from finishing. (The Travis macOS jobs are
+      slow, so we don't want to have it delay the whole CI process.)
+      Closes ticket 32629.
+
+  o Testing (continuous integration, backport from 0.4.3.1-alpha):
+    - Use zstd in our Travis Linux builds. Closes ticket 32242.
+
+
+Changes in version 0.4.3.1-alpha - 2020-01-22
+  This is the first alpha release in the 0.4.3.x series. It includes
+  improved support for application integration of onion services, support
+  for building in a client-only mode, and newly improved internal
+  documentation (online at https://src-ref.docs.torproject.org/tor/). It
+  also has numerous other small bugfixes and features, as well as
+  improvements to our code's internal organization that should help us
+  write better code in the future.
+
+  o New system requirements:
+    - When building Tor, you now need to have Python 3 in order to run
+      the integration tests. (Python 2 is officially unsupported
+      upstream, as of 1 Jan 2020.) Closes ticket 32608.
+
+  o Major features (build system):
+    - The relay code can now be disabled using the --disable-module-relay
+      configure option. When this option is set, we also disable the
+      dirauth module. Closes ticket 32123.
+    - When Tor is compiled --disable-module-relay, we also omit the code
+      used to act as a directory cache. Closes ticket 32487.
+
+  o Major features (directory authority, ed25519):
+    - Add support for banning a relay's ed25519 keys in the approved-
+      routers file. This will help us migrate away from RSA keys in the
+      future. Previously, only RSA keys could be banned in approved-
+      routers. Resolves ticket 22029. Patch by Neel Chauhan.
+
+  o Major features (onion service, controller):
+    - New control port commands to manage client-side onion service
+      authorization credentials. The ONION_CLIENT_AUTH_ADD command adds
+      a credential, ONION_CLIENT_AUTH_REMOVE deletes a credential, and
+      ONION_CLIENT_AUTH_VIEW lists the credentials. Closes ticket 30381.
+
+  o Major features (onion service, SOCKS5):
+    - Introduce a new SocksPort flag, ExtendedErrors, to support more
+      detailed error codes in information for applications that support
+      them. Closes ticket 30382; implements proposal 304.
+
+  o Major features (proxy):
+    - In addition to its current supported proxy types (HTTP CONNECT,
+      SOCKS4, and SOCKS5), Tor can now make its OR connections through a
+      HAProxy server. A new torrc option was added to specify the
+      address/port of the server: TCPProxy <protocol> <host>:<port>.
+      Currently the only supported protocol for the option is haproxy.
+      Closes ticket 31518. Patch done by Suphanat Chunhapanya (haxxpop).
+
+  o Major bugfixes (linux seccomp sandbox):
+    - Correct how we use libseccomp. Particularly, stop assuming that
+      rules are applied in a particular order or that more rules are
+      processed after the first match. Neither is the case! In
+      libseccomp <2.4.0 this led to some rules having no effect.
+      libseccomp 2.4.0 changed how rules are generated, leading to a
+      different ordering, which in turn led to a fatal crash during
+      startup. Fixes bug 29819; bugfix on 0.2.5.1-alpha. Patch by
+      Peter Gerber.
+    - Fix crash when reloading logging configuration while the
+      experimental sandbox is enabled. Fixes bug 32841; bugfix on
+      0.4.1.7. Patch by Peter Gerber.
+
+  o Major bugfixes (networking):
+    - Correctly handle IPv6 addresses in SOCKS5 RESOLVE_PTR requests,
+      and accept strings as well as binary addresses. Fixes bug 32315;
+      bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (onion service):
+    - Report HS circuit failure back into the HS subsystem so we take
+      appropriate action with regards to the client introduction point
+      failure cache. This improves reachability of onion services, since
+      now clients notice failing introduction circuits properly. Fixes
+      bug 32020; bugfix on 0.3.2.1-alpha.
+
+  o Minor feature (configure, build system):
+    - Output a list of enabled/disabled features at the end of the
+      configure process in a pleasing way. Closes ticket 31373.
+
+  o Minor feature (heartbeat, onion service):
+    - Add the DoS INTRODUCE2 defenses counter to the heartbeat DoS
+      message. Closes ticket 31371.
+
+  o Minor features (configuration validation):
+    - Configuration validation can now be done by per-module callbacks,
+      rather than a global validation function. This will let us reduce
+      the size of config.c and some of its more cumbersome functions.
+      Closes ticket 31241.
+
+  o Minor features (configuration):
+    - If a configured hardware crypto accelerator in AccelName is
+      prefixed with "!", Tor now exits when it cannot be found. Closes
+      ticket 32406.
+    - We now use flag-driven logic to warn about obsolete configuration
+      fields, so that we can include their names. In 0.4.2, we used a
+      special type, which prevented us from generating good warnings.
+      Implements ticket 32404.
+
+  o Minor features (controller):
+    - Add stream isolation data to STREAM event. Closes ticket 19859.
+    - Implement a new GETINFO command to fetch microdescriptor
+      consensus. Closes ticket 31684.
+
+  o Minor features (debugging, directory system):
+    - Don't crash when we find a non-guard with a guard-fraction value
+      set. Instead, log a bug warning, in an attempt to figure out how
+      this happened. Diagnostic for ticket 32868.
+
+  o Minor features (defense in depth):
+    - Add additional checks around tor_vasprintf() usage, in case the
+      function returns an error. Patch by Tobias Stoeckmann. Fixes
+      ticket 31147.
+
+  o Minor features (developer tooling):
+    - Remove the 0.2.9.x series branches from git scripts (git-merge-
+      forward.sh, git-pull-all.sh, git-push-all.sh, git-setup-dirs.sh).
+      Closes ticket 32772.
+
+  o Minor features (developer tools):
+    - Add a check_cocci_parse.sh script that checks that new code is
+      parseable by Coccinelle. Add an exceptions file for unparseable
+      files, and run the script from travis CI. Closes ticket 31919.
+    - Call the check_cocci_parse.sh script from a 'check-cocci' Makefile
+      target. Closes ticket 31919.
+    - Add a rename_c_identifiers.py tool to rename a bunch of C
+      identifiers at once, and generate a well-formed commit message
+      describing the change. This should help with refactoring. Closes
+      ticket 32237.
+    - Add some scripts in "scripts/coccinelle" to invoke the Coccinelle
+      semantic patching tool with the correct flags. These flags are
+      fairly easy to forget, and these scripts should help us use
+      Coccinelle more effectively in the future. Closes ticket 31705.
+
+  o Minor features (Doxygen):
+    - Update Doxygen configuration file to a more recent template (from
+      1.8.15). Closes ticket 32110.
+    - "make doxygen" now works with out-of-tree builds. Closes
+      ticket 32113.
+    - Make sure that doxygen outputs documentation for all of our C
+      files. Previously, some were missing @file declarations, causing
+      them to be ignored. Closes ticket 32307.
+    - Our "make doxygen" target now respects --enable-fatal-warnings by
+      default, and does not warn about items that are missing
+      documentation. To warn about missing documentation, run configure
+      with the "--enable-missing-doc-warnings" flag: doing so suspends
+      fatal warnings for doxygen. Closes ticket 32385.
+
+  o Minor features (git scripts):
+    - Add TOR_EXTRA_CLONE_ARGS to git-setup-dirs.sh for git clone
+      customisation. Closes ticket 32347.
+    - Add git-setup-dirs.sh, which sets up an upstream git repository
+      and worktrees for tor maintainers. Closes ticket 29603.
+    - Add TOR_EXTRA_REMOTE_* to git-setup-dirs.sh for a custom extra
+      remote. Closes ticket 32347.
+    - Call the check_cocci_parse.sh script from the git commit and push
+      hooks. Closes ticket 31919.
+    - Make git-push-all.sh skip unchanged branches when pushing to
+      upstream. The script already skipped unchanged test branches.
+      Closes ticket 32216.
+    - Make git-setup-dirs.sh create a master symlink in the worktree
+      directory. Closes ticket 32347.
+    - Skip unmodified source files when doing some existing git hook
+      checks. Related to ticket 31919.
+
+  o Minor features (IPv6, client):
+    - Make Tor clients tell dual-stack exits that they prefer IPv6
+      connections. This change is equivalent to setting the PreferIPv6
+      flag on SOCKSPorts (and most other listener ports). Tor Browser
+      has been setting this flag for some time, and we want to remove a
+      client distinguisher at exits. Closes ticket 32637.
+
+  o Minor features (portability, android):
+    - When building for Android, disable some tests that depend on $HOME
+      and/or pwdb, which Android doesn't have. Closes ticket 32825.
+      Patch from Hans-Christoph Steiner.
+
+  o Minor features (relay modularity):
+    - Split the relay and server pluggable transport config code into
+      separate files in the relay module. Disable this code when the
+      relay module is disabled. Closes part of ticket 32213.
+    - When the relay module is disabled, reject attempts to set the
+      ORPort, DirPort, DirCache, BridgeRelay, ExtORPort, or
+      ServerTransport* options, rather than ignoring the values of these
+      options. Closes part of ticket 32213.
+
+  o Minor features (relay):
+    - When the relay module is disabled, change the default config so
+      that DirCache is 0, and ClientOnly is 1. Closes ticket 32410.
+
+  o Minor features (release tools):
+    - Port our ChangeLog formatting and sorting tools to Python 3.
+      Closes ticket 32704.
+
+  o Minor features (testing):
+    - Detect some common failure cases for test_parseconf.sh in
+      src/test/conf_failures. Closes ticket 32451.
+    - Allow test_parseconf.sh to test expected log outputs for successful
+      configs, as well as failed configs. Closes ticket 32451.
+    - The test_parseconf.sh script now supports result variants for any
+      combination of the optional libraries lzma, nss, and zstd. Closes
+      ticket 32397.
+
+  o Minor features (tests, Android):
+    - When running the unit tests on Android, create temporary files in
+      a subdirectory of /data/local/tmp. Closes ticket 32172. Based on a
+      patch from Hans-Christoph Steiner.
+
+  o Minor bugfixes (bridges):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (build system):
+    - Fix "make autostyle" for out-of-tree builds. Fixes bug 32370;
+      bugfix on 0.4.1.2-alpha.
+
+  o Minor bugfixes (configuration handling):
+    - Make control_event_conf_changed() take in a config_line_t instead
+      of a smartlist of alternating key/value entries. Fixes bug 31531;
+      bugfix on 0.2.3.3-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (configuration):
+    - Check for multiplication overflow when parsing memory units inside
+      configuration. Fixes bug 30920; bugfix on 0.0.9rc1.
+    - When dumping the configuration, stop adding a trailing space after
+      the option name when there is no option value. This issue only
+      affects options that accept an empty value or list. (Most options
+      reject empty values, or delete the entire line from the dumped
+      options.) Fixes bug 32352; bugfix on 0.0.9pre6.
+    - Avoid changing the user's value of HardwareAccel as stored by
+      SAVECONF, when AccelName is set but HardwareAccel is not. Fixes
+      bug 32382; bugfix on 0.2.2.1-alpha.
+    - When creating a KeyDirectory with the same location as the
+      DataDirectory (not recommended), respect the DataDirectory's
+      group-readable setting if one has not been set for the
+      KeyDirectory. Fixes bug 27992; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (controller):
+    - In routerstatus_has_changed(), check all the fields that are
+      output over the control port. Fixes bug 20218; bugfix
+      on 0.1.1.11-alpha
+
+  o Minor bugfixes (correctness checks):
+    - Use GCC/Clang's printf-checking feature to make sure that
+      tor_assertf() arguments are correctly typed. Fixes bug 32765;
+      bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (developer tools):
+    - Allow paths starting with ./ in scripts/add_c_file.py. Fixes bug
+      31336; bugfix on 0.4.1.2-alpha.
+
+  o Minor bugfixes (dirauth module):
+    - Split the dirauth config code into a separate file in the dirauth
+      module. Disable this code when the dirauth module is disabled.
+      Closes ticket 32213.
+    - When the dirauth module is disabled, reject attempts to set the
+      AuthoritativeDir option, rather than ignoring the value of the
+      option. Fixes bug 32213; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (embedded Tor):
+    - When starting Tor any time after the first time in a process,
+      register the thread in which it is running as the main thread.
+      Previously, we only did this on Windows, which could lead to bugs
+      like 23081 on non-Windows platforms. Fixes bug 32884; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (git scripts):
+    - Avoid sleeping before the last push in git-push-all.sh. Closes
+      ticket 32216.
+    - Forward all unrecognised arguments in git-push-all.sh to git push.
+      Closes ticket 32216.
+
+  o Minor bugfixes (hidden service v3):
+    - Do not rely on a "circuit established" flag for intro circuits but
+      instead always query the HS circuit map. This is to avoid sync
+      issue with that flag and the map. Fixes bug 32094; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (logging, crash):
+    - Avoid a possible crash when trying to log a (fatal) assertion
+      failure about mismatched magic numbers in configuration objects.
+      Fixes bug 32771; bugfix on 0.4.2.1-alpha.
+
+  o Minor bugfixes (onion service v2):
+    - When sending the INTRO cell for a v2 Onion Service, look at the
+      failure cache alongside timeout values to check if the intro point
+      is marked as failed. Previously, we only looked at the relay
+      timeout values. Fixes bug 25568; bugfix on 0.2.7.3-rc. Patch by
+      Neel Chauhan.
+
+  o Minor bugfixes (onion services v3, client):
+    - Properly handle the client rendezvous circuit timeout. Previously
+      Tor would sometimes timeout a rendezvous circuit awaiting the
+      introduction ACK, and find itself unable to re-establish all
+      circuits because the rendezvous circuit timed out too early. Fixes
+      bug 32021; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - In cancel_descriptor_fetches(), use
+      connection_list_by_type_purpose() instead of
+      connection_list_by_type_state(). Fixes bug 32639; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (scripts):
+    - Fix update_versions.py for out-of-tree builds. Fixes bug 32371;
+      bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (test):
+    - Use the same code to find the tor binary in all of our test
+      scripts. This change makes sure we are always using the coverage
+      binary when coverage is enabled. Fixes bug 32368; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing):
+    - Stop ignoring "tor --dump-config" errors in test_parseconf.sh.
+      Fixes bug 32468; bugfix on 0.4.2.1-alpha.
+    - When TOR_DISABLE_PRACTRACKER is set, do not apply it to the
+      test_practracker.sh script. Doing so caused a test failure. Fixes
+      bug 32705; bugfix on 0.4.2.1-alpha.
+    - When TOR_DISABLE_PRACTRACKER is set, log a notice to stderr when
+      skipping practracker checks. Fixes bug 32705; bugfix
+      on 0.4.2.1-alpha.
+
+  o Minor bugfixes (tests):
+    - Our option-validation tests no longer depend on specially
+      configured non-default, non-passing sets of options. Previously,
+      the tests had been written to assume that options would _not_ be
+      set to their defaults, which led to needless complexity and
+      verbosity. Fixes bug 32175; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (windows service):
+    - Initialize the publish/subscribe system when running as a windows
+      service. Fixes bug 32778; bugfix on 0.4.1.1-alpha.
+
+  o Deprecated features:
+    - Deprecate the ClientAutoIPv6ORPort option. This option was not
+      true "Happy Eyeballs", and often failed on connections that
+      weren't reliably dual-stack. Closes ticket 32942. Patch by
+      Neel Chauhan.
+
+  o Documentation:
+    - Provide a quickstart guide for a Circuit Padding Framework, and
+      documentation for researchers to implement and study circuit
+      padding machines. Closes ticket 28804.
+    - Add documentation in 'HelpfulTools.md' to describe how to build a
+      tag file. Closes ticket 32779.
+    - Create a high-level description of the long-term software
+      architecture goals. Closes ticket 32206.
+    - Describe the --dump-config command in the manual page. Closes
+      ticket 32467.
+    - Unite coding advice from this_not_that.md in torguts repo into our
+      coding standards document. Resolves ticket 31853.
+
+  o Removed features:
+    - Our Doxygen configuration no longer generates LaTeX output. The
+      reference manual produced by doing this was over 4000 pages long,
+      and generally unusable. Closes ticket 32099.
+    - The option "TestingEstimatedDescriptorPropagationTime" is now
+      marked as obsolete. It has had no effect since 0.3.0.7, when
+      clients stopped rejecting consensuses "from the future". Closes
+      ticket 32807.
+    - We no longer support consensus methods before method 28; these
+      methods were only used by authorities running versions of Tor that
+      are now at end-of-life. In effect, this means that clients,
+      relays, and authorities now assume that authorities will be
+      running version 0.3.5.x or later. Closes ticket 32695.
+
+  o Testing:
+    - Add more test cases for tor's UTF-8 validation function. Also,
+      check the arguments passed to the function for consistency. Closes
+      ticket 32845.
+    - Improve test coverage for relay and dirauth config code, focusing
+      on option validation and normalization. Closes ticket 32213.
+    - Improve the consistency of test_parseconf.sh output, and run all
+      the tests, even if one fails. Closes ticket 32213.
+    - Re-enable the Travis CI macOS Chutney build, but don't let it
+      prevent the Travis job from finishing. (The Travis macOS jobs are
+      slow, so we don't want to have it delay the whole CI process.)
+      Closes ticket 32629.
+    - Run the practracker unit tests in the pre-commit git hook. Closes
+      ticket 32609.
+    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on
+      Ubuntu Bionic. Turning off the Sandbox is a work-around, until we
+      fix the sandbox errors in 32722. Closes ticket 32240.
+
+  o Code simplification and refactoring (channel):
+    - Channel layer had a variable length cell handler that was not used
+      and thus removed. Closes ticket 32892.
+
+  o Code simplification and refactoring (configuration):
+    - Immutability is now implemented as a flag on individual
+      configuration options rather than as part of the option-transition
+      checking code. Closes ticket 32344.
+    - Instead of keeping a list of configuration options to check for
+      relative paths, check all the options whose type is "FILENAME".
+      Solves part of ticket 32339.
+    - Our default log (which ordinarily sends NOTICE-level messages to
+      standard output) is now handled in a more logical manner.
+      Previously, we replaced the configured log options if they were
+      empty. Now, we interpret an empty set of log options as meaning
+      "use the default log". Closes ticket 31999.
+    - Remove some unused arguments from the options_validate() function,
+      to simplify our code and tests. Closes ticket 32187.
+    - Simplify the options_validate() code so that it looks at the
+      default options directly, rather than taking default options as an
+      argument. This change lets us simplify its interface. Closes
+      ticket 32185.
+    - Use our new configuration architecture to move most authority-
+      related options to the directory authority module. Closes
+      ticket 32806.
+    - When parsing the command line, handle options that determine our
+      "quiet level" and our mode of operation (e.g., --dump-config and
+      so on) all in one table. Closes ticket 32003.
+
+  o Code simplification and refactoring (controller):
+    - Create a new abstraction for formatting control protocol reply
+      lines based on key-value pairs. Refactor some existing control
+      protocol code to take advantage of this. Closes ticket 30984.
+    - Create a helper function that can fetch network status or
+      microdesc consensuses. Closes ticket 31684.
+
+  o Code simplification and refactoring (dirauth modularization):
+    - Remove the last remaining HAVE_MODULE_DIRAUTH inside a function.
+      Closes ticket 32163.
+    - Replace some confusing identifiers in process_descs.c. Closes
+      ticket 29826.
+    - Simplify some relay and dirauth config code. Closes ticket 32213.
+
+  o Code simplification and refactoring (misc):
+    - Make all the structs we declare follow the same naming convention
+      of ending with "_t". Closes ticket 32415.
+    - Move and rename some configuration-related code for clarity.
+      Closes ticket 32304.
+    - Our include.am files are now broken up by subdirectory.
+      Previously, src/core/include.am covered all of the subdirectories
+      in "core", "feature", and "app". Closes ticket 32137.
+    - Remove underused NS*() macros from test code: they make our tests
+      more confusing, especially for code-formatting tools. Closes
+      ticket 32887.
+
+  o Code simplification and refactoring (relay modularization):
+    - Disable relay_periodic when the relay module is disabled. Closes
+      ticket 32244.
+    - Disable relay_sys when the relay module is disabled. Closes
+      ticket 32245.
+
+  o Code simplification and refactoring (tool support):
+    - Add numerous missing dependencies to our include files, so that
+      they can be included in different reasonable orders and still
+      compile. Addresses part of ticket 32764.
+    - Fix some parts of our code that were difficult for Coccinelle to
+      parse. Related to ticket 31705.
+    - Fix some small issues in our code that prevented automatic
+      formatting tools from working. Addresses part of ticket 32764.
+
+  o Documentation (manpage):
+    - Alphabetize the Client Options section of the tor manpage. Closes
+      ticket 32846.
+    - Alphabetize the General Options section of the tor manpage. Closes
+      ticket 32708.
+    - In the tor(1) manpage, reword and improve formatting of the
+      COMMAND-LINE OPTIONS and DESCRIPTION sections. Closes ticket
+      32277. Based on work by Swati Thacker as part of Google Season
+      of Docs.
+    - In the tor(1) manpage, reword and improve formatting of the FILES,
+      SEE ALSO, and BUGS sections. Closes ticket 32176. Based on work by
+      Swati Thacker as part of Google Season of Docs.
+
+  o Testing (circuit, EWMA):
+    - Add unit tests for circuitmux and EWMA subsystems. Closes
+      ticket 32196.
+
+  o Testing (continuous integration):
+    - Use zstd in our Travis Linux builds. Closes ticket 32242.
+
+
+Changes in version 0.4.2.5 - 2019-12-09
+  This is the first stable release in the 0.4.2.x series. This series
+  improves reliability and stability, and includes several stability and
+  correctness improvements for onion services. It also fixes many smaller
+  bugs present in previous series.
+
+  Per our support policy, we will support the 0.4.2.x series for nine
+  months, or until three months after the release of a stable 0.4.3.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Per our support policy, we will support the 0.4.2.x series for nine
+  months, or until three months after the release of a stable 0.4.3.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.1.4-rc. For a complete list of changes
+  since 0.4.1.5, see the ReleaseNotes file.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Testing:
+    - Require C99 standards-conforming code in Travis CI, but allow GNU
+      gcc extensions. Also activates clang's -Wtypedef-redefinition
+      warnings. Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+
+
+Changes in version 0.4.1.7 - 2019-12-09
+  This release backports several bugfixes to improve stability and
+  correctness.  Anyone experiencing build problems or crashes with 0.4.1.6,
+  including all relays relying on AccountingMax, should upgrade.
+
+  o Major features (directory authorities, backport from 0.4.2.2-alpha):
+    - Directory authorities now reject relays running all currently
+      deprecated release series. The currently supported release series
+      are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549.
+
+  o Major bugfixes (embedded Tor, backport from 0.4.2.2-alpha):
+    - Avoid a possible crash when restarting Tor in embedded mode and
+      enabling a different set of publish/subscribe messages. Fixes bug
+      31898; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.4.2.3-alpha):
+    - Relays now respect their AccountingMax bandwidth again. When
+      relays entered "soft" hibernation (which typically starts when
+      we've hit 90% of our AccountingMax), we had stopped checking
+      whether we should enter hard hibernation. Soft hibernation refuses
+      new connections and new circuits, but the existing circuits can
+      continue, meaning that relays could have exceeded their configured
+      AccountingMax. Fixes bug 32108; bugfix on 0.4.0.1-alpha.
+
+  o Major bugfixes (torrc parsing, backport from 0.4.2.2-alpha):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (v3 onion services, backport from 0.4.2.3-alpha):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.4.2.2-alpha):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (geoip, backport from 0.4.2.5):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Minor bugfixes (Appveyor CI, backport from 0.4.2.2-alpha):
+    - Avoid spurious errors when Appveyor CI fails before the install step.
+      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (client, onion service v3, backport from 0.4.2.4-rc):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (connections, backport from 0.4.2.3-rc):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (error handling, backport from 0.4.2.1-alpha):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - When tor aborts due to an error, close log file descriptors before
+      aborting. Closing the logs makes some OSes flush log file buffers,
+      rather than deleting buffered log lines. Fixes bug 31594; bugfix
+      on 0.2.5.2-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.2-alpha):
+    - Add a missing check for HAVE_PTHREAD_H, because the backtrace code
+      uses mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha.
+    - Disable backtrace signal handlers when shutting down tor. Fixes
+      bug 31614; bugfix on 0.2.5.2-alpha.
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations, backport from 0.4.2.2-alpha):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API, backport from 0.4.2.3-alpha):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (multithreading, backport from 0.4.2.2-alpha):
+    - Avoid some undefined behaviour when freeing mutexes. Fixes bug
+      31736; bugfix on 0.0.7.
+
+  o Minor bugfixes (process management, backport from 0.4.2.3-alpha):
+    - Remove overly strict assertions that triggered when a pluggable
+      transport failed to launch. Fixes bug 31091; bugfix
+      on 0.4.0.1-alpha.
+    - Remove an assertion in the Unix process backend. This assertion
+      would trigger when we failed to find the executable for a child
+      process. Fixes bug 31810; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.4.2.2-alpha):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.2.3-alpha):
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (tests, SunOS, backport from 0.4.2.2-alpha):
+    - Avoid a map_anon_nofork test failure due to a signed/unsigned
+      integer comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (tls, logging, backport from 0.4.2.3-alpha):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Documentation (backport from 0.4.2.2-alpha):
+    - Explain why we can't destroy the backtrace buffer mutex. Explain
+      why we don't need to destroy the log mutex. Closes ticket 31736.
+
+  o Testing (continuous integration, backport from 0.4.2.3-alpha):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Testing (continuous integration, backport from 0.4.2.4-rc):
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+
+  o Testing (continuous integration, backport from 0.4.2.5):
+    - Require C99 standards-conforming code in Travis CI, but allow GNU gcc
+      extensions. Also activates clang's -Wtypedef-redefinition warnings.
+      Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+
+
+Changes in version 0.4.0.6 - 2019-12-09
+  This is the second stable release in the 0.4.0.x series. This release
+  backports several bugfixes to improve stability and correctness.  Anyone
+  experiencing build problems or crashes with 0.4.0.5, including all relays
+  relying on AccountingMax, should upgrade.
+
+  Note that, per our support policy, support for the 0.4.0.x series will end
+  on 2 Feb 2020.  Anyone still running 0.4.0.x should plan to upgrade to the
+  latest stable release, or downgrade to 0.3.5.x, which will get long-term
+  support until 1 Feb 2022.
+
+  o Directory authority changes (backport from 0.4.1.5):
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Major bugfixes (bridges, backport from 0.4.1.2-alpha):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit build, guard, backport from 0.4.1.4-rc):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (Onion service reachability, backport from 0.4.1.3-alpha):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Major bugfixes (onion service v3, backport from 0.4.1.1-alpha):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.4.2.3-alpha):
+    - Relays now respect their AccountingMax bandwidth again. When
+      relays entered "soft" hibernation (which typically starts when
+      we've hit 90% of our AccountingMax), we had stopped checking
+      whether we should enter hard hibernation. Soft hibernation refuses
+      new connections and new circuits, but the existing circuits can
+      continue, meaning that relays could have exceeded their configured
+      AccountingMax. Fixes bug 32108; bugfix on 0.4.0.1-alpha.
+
+  o Major bugfixes (torrc parsing, backport from 0.4.2.2-alpha):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (v3 onion services, backport from 0.4.2.3-alpha):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (compile-time modules, backport from version 0.4.1.1-alpha):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration, backport from 0.4.1.1-alpha):
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+
+  o Minor features (continuous integration, backport from 0.4.1.4-rc):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+
+  o Minor features (continuous integration, backport from 0.4.2.2-alpha):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (fallback directory list, backport from 0.4.1.4-rc):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor features (geoip, backport from 0.4.2.5):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (Appveyor CI, backport from 0.4.2.2-alpha):
+    - Avoid spurious errors when Appveyor CI fails before the install step.
+      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (circuit isolation, backport from 0.4.1.3-alpha):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (circuit padding, backport from 0.4.1.4-rc):
+    - On relays, properly check that a padding machine is absent before
+      logging a warning about it being absent. Fixes bug 30649; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (client, onion service v3, backport from 0.4.2.4-rc):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (clock skew detection, backport from 0.4.1.5):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation warning, backport from 0.4.1.5):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.1.5):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations, backport from 0.4.1.1-alpha):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (configuration, proxies, backport from 0.4.1.2-alpha):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (connections, backport from 0.4.2.3-rc):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.4.1.3-alpha):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (crash on exit, backport from 0.4.1.4-rc):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (directory authorities, backport from 0.4.1.3-alpha):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.4.1.2-alpha):
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (error handling, backport from 0.4.2.1-alpha):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6, backport from 0.4.2.1-alpha):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.1.1-alpha):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.2-alpha):
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations, backport from 0.4.2.2-alpha):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API, backport from 0.4.2.3-alpha):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.1-alpha):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.4-rc):
+    - Fix a trivial memory leak when parsing an invalid value
+      from a download schedule in the configuration. Fixes bug
+      30894; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (NetBSD, backport from 0.4.1.2-alpha):
+    - Fix usage of minherit() on NetBSD and other platforms that define
+      MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug
+      30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell.
+
+  o Minor bugfixes (onion services, backport from 0.4.1.1-alpha):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (out-of-memory handler, backport from 0.4.1.2-alpha):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.4.1.2-alpha):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (process management, backport from 0.4.2.3-alpha):
+    - Remove overly strict assertions that triggered when a pluggable
+      transport failed to launch. Fixes bug 31091; bugfix
+      on 0.4.0.1-alpha.
+    - Remove an assertion in the Unix process backend. This assertion
+      would trigger when we failed to find the executable for a child
+      process. Fixes bug 31810; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.4.2.2-alpha):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.2.3-alpha):
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (tls, logging, backport from 0.4.2.3-alpha):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+  o Testing (backport from 0.4.1.2-alpha):
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+  o Testing (continuous integration, backport from 0.4.1.1-alpha):
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Testing (continuous integration, backport from 0.4.1.5):
+    - In Travis, make stem log a controller trace to the console, and
+      tail stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary. Closes
+      ticket 30694.
+
+  o Testing (continuous integration, backport from 0.4.2.3-alpha):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Testing (continuous integration, backport from 0.4.2.4-rc):
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+
+  o Testing (continuous integration, backport from 0.4.2.5):
+    - Require C99 standards-conforming code in Travis CI, but allow GNU gcc
+      extensions. Also activates clang's -Wtypedef-redefinition warnings.
+      Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+
+Changes in version 0.3.5.9 - 2019-12-09
+  Tor 0.3.5.9 backports serveral fixes from later releases, including
+  several that affect bridge users, relay stability, onion services,
+  and much more.
+
+  o Directory authority changes (backport from 0.4.1.5):
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Major bugfixes (bridges, backport from 0.4.1.2-alpha):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit build, guard, backport from 0.4.1.4-rc):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (NSS, relay, backport from 0.4.0.4-rc):
+    - When running with NSS, disable TLS 1.2 ciphersuites that use
+      SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for
+      these ciphersuites don't work -- which caused relays to fail to
+      handshake with one another when these ciphersuites were enabled.
+      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (Onion service reachability, backport from 0.4.1.3-alpha):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Major bugfixes (onion service v3, backport from 0.4.1.1-alpha):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (torrc parsing, backport from 0.4.2.2-alpha):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (v3 onion services, backport from 0.4.2.3-alpha):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (address selection, backport from 0.4.0.3-alpha):
+    - Treat the subnet 100.64.0.0/10 as public for some purposes;
+      private for others. This subnet is the RFC 6598 (Carrier Grade
+      NAT) IP range, and is deployed by many ISPs as an alternative to
+      RFC 1918 that does not break existing internal networks. Tor now
+      blocks SOCKS and control ports on these addresses and warns users
+      if client ports or ExtORPorts are listening on a RFC 6598 address.
+      Closes ticket 28525. Patch by Neel Chauhan.
+
+  o Minor features (bandwidth authority, backport from 0.4.0.4-rc):
+    - Make bandwidth authorities ignore relays that are reported in the
+      bandwidth file with the flag "vote=0". This change allows us to
+      report unmeasured relays for diagnostic reasons without including
+      their bandwidth in the bandwidth authorities' vote. Closes
+      ticket 29806.
+
+  o Minor features (compile-time modules, backport from version 0.4.1.1-alpha):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration, backport from 0.4.0.4-rc):
+    - On Travis Rust builds, cleanup Rust registry and refrain from
+      caching the "target/" directory to speed up builds. Resolves
+      issue 29962.
+
+  o Minor features (continuous integration, backport from 0.4.0.5):
+    - In Travis, tell timelimit to use stem's backtrace signals, and
+      launch python directly from timelimit, so python receives the
+      signals from timelimit, rather than make. Closes ticket 30117.
+
+  o Minor features (continuous integration, backport from 0.4.1.1-alpha):
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+
+  o Minor features (continuous integration, backport from 0.4.1.4-rc):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+
+  o Minor features (continuous integration, backport from 0.4.2.2-alpha):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (fallback directory list, backport from 0.4.1.4-rc):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor features (geoip, backport from 0.4.2.5):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Minor features (NSS, diagnostic, backport from 0.4.0.4-rc):
+    - Try to log an error from NSS (if there is any) and a more useful
+      description of our situation if we are using NSS and a call to
+      SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (security, backport from 0.4.0.4-rc):
+    - Verify in more places that we are not about to create a buffer
+      with more than INT_MAX bytes, to avoid possible OOB access in the
+      event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and
+      fixed by Tobias Stoeckmann.
+    - Fix a potential double free bug when reading huge bandwidth files.
+      The issue is not exploitable in the current Tor network because
+      the vulnerable code is only reached when directory authorities
+      read bandwidth files, but bandwidth files come from a trusted
+      source (usually the authorities themselves). Furthermore, the
+      issue is only exploitable in rare (non-POSIX) 32-bit architectures,
+      which are not used by any of the current authorities. Fixes bug
+      30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfix (continuous integration, backport from 0.4.0.4-rc):
+    - Reset coverage state on disk after Travis CI has finished. This
+      should prevent future coverage merge errors from causing the test
+      suite for the "process" subsystem to fail. The process subsystem
+      was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix
+      on 0.2.9.15.
+    - Terminate test-stem if it takes more than 9.5 minutes to run.
+      (Travis terminates the job after 10 minutes of no output.)
+      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (Appveyor CI, backport from 0.4.2.2-alpha):
+    - Avoid spurious errors when Appveyor CI fails before the install step.
+      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.4.0.4-rc):
+    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug
+      29824; bugfix on 0.3.1.1-alpha. This is Coverity warning
+      CID 1444119.
+
+  o Minor bugfixes (circuit isolation, backport from 0.4.1.3-alpha):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (client, onion service v3, backport from 0.4.2.4-rc):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (clock skew detection, backport from 0.4.1.5):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation warning, backport from 0.4.1.5):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.0.2-alpha):
+    - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug
+      29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (compilation, backport from 0.4.1.5):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations, backport from 0.4.1.1-alpha):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (configuration, proxies, backport from 0.4.1.2-alpha):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (connections, backport from 0.4.2.3-rc):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.4.1.3-alpha):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (crash on exit, backport from 0.4.1.4-rc):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (directory authorities, backport from 0.4.1.3-alpha):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.4.1.2-alpha):
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (error handling, backport from 0.4.2.1-alpha):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6, backport from 0.4.2.1-alpha):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.0.2-alpha):
+    - Avoid logging that we are relaxing a circuit timeout when that
+      timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.0.3-alpha):
+    - Correct a misleading error message when IPv4Only or IPv6Only is
+      used but the resolved address can not be interpreted as an address
+      of the specified IP version. Fixes bug 13221; bugfix on
+      0.2.3.9-alpha. Patch from Kris Katterjohn.
+    - Log the correct port number for listening sockets when "auto" is
+      used to let Tor pick the port number. Previously, port 0 was
+      logged instead of the actual port number. Fixes bug 29144; bugfix
+      on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+    - Stop logging a BUG() warning when Tor is waiting for exit
+      descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.1.1-alpha):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.2-alpha):
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations, backport from 0.4.2.2-alpha):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API, backport from 0.4.2.3-alpha):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.1-alpha):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.4-rc):
+    - Fix a trivial memory leak when parsing an invalid value
+      from a download schedule in the configuration. Fixes bug
+      30894; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (memory management, backport from 0.4.0.3-alpha):
+    - Refactor the shared random state's memory management so that it
+      actually takes ownership of the shared random value pointers.
+      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (memory management, testing, backport from 0.4.0.3-alpha):
+    - Stop leaking parts of the shared random state in the shared-random
+      unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.1.1-alpha):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (out-of-memory handler, backport from 0.4.1.2-alpha):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.4.1.2-alpha):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (relay, backport from 0.4.2.2-alpha):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.0.5):
+    - Abort on panic in all build profiles, instead of potentially
+      unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (single onion services, backport from 0.4.0.3-alpha):
+    - Allow connections to single onion services to remain idle without
+      being disconnected. Previously, relays acting as rendezvous points
+      for single onion services were mistakenly closing idle rendezvous
+      circuits after 60 seconds, thinking that they were unused
+      directory-fetching circuits that had served their purpose. Fixes
+      bug 29665; bugfix on 0.2.1.26.
+
+  o Minor bugfixes (stats, backport from 0.4.0.3-alpha):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.0.3-alpha):
+    - Downgrade some LOG_ERR messages in the address/* tests to
+      warnings. The LOG_ERR messages were occurring when we had no
+      configured network. We were failing the unit tests, because we
+      backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug
+      29530; bugfix on 0.3.5.8.
+    - Fix our gcov wrapper script to look for object files at the
+      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.0.4-rc):
+    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a
+      recent test-network.sh to use new chutney features in CI. Fixes
+      bug 29703; bugfix on 0.2.9.1-alpha.
+    - Fix a test failure on Windows caused by an unexpected "BUG"
+      warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.2.3-alpha):
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (TLS protocol, backport form 0.4.0.4-rc):
+    - When classifying a client's selection of TLS ciphers, if the
+      client ciphers are not yet available, do not cache the result.
+      Previously, we had cached the unavailability of the cipher list
+      and never looked again, which in turn led us to assume that the
+      client only supported the ancient V1 link protocol. This, in turn,
+      was causing Stem integration tests to stall in some cases. Fixes
+      bug 30021; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (tls, logging, backport from 0.4.2.3-alpha):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (Windows, CI, backport from 0.4.0.3-alpha):
+    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit
+      Windows Server 2012 R2 job. The remaining 2 jobs still provide
+      coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set
+      fast_finish, so failed jobs terminate the build immediately. Fixes
+      bug 29601; bugfix on 0.3.5.4-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+  o Testing (backport from 0.4.1.2-alpha):
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+  o Testing (continuous integration, backport from 0.4.1.1-alpha):
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Testing (continuous integration, backport from 0.4.1.5):
+    - In Travis, make stem log a controller trace to the console, and
+      tail stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary. Closes
+      ticket 30694.
+
+  o Testing (continuous integration, backport from 0.4.2.3-alpha):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Testing (continuous integration, backport from 0.4.2.4-rc):
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+
+  o Testing (continuous integration, backport from 0.4.2.5):
+    - Require C99 standards-conforming code in Travis CI, but allow GNU gcc
+      extensions. Also activates clang's -Wtypedef-redefinition warnings.
+      Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+
+Changes in version 0.4.2.4-rc - 2019-11-15
+  Tor 0.4.2.4-rc is the first release candidate in its series. It fixes
+  several bugs from earlier versions, including a few that would result in
+  stack traces or incorrect behavior.
+
+  o Minor features (build system):
+    - Make pkg-config use --prefix when cross-compiling, if
+      PKG_CONFIG_PATH is not set. Closes ticket 32191.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32440.
+
+  o Minor bugfixes (client, onion service v3):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (code quality):
+    - Fix "make check-includes" so it runs correctly on out-of-tree
+      builds. Fixes bug 31335; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (configuration):
+    - Log the option name when skipping an obsolete option. Fixes bug
+      32295; bugfix on 0.4.2.1-alpha.
+
+  o Minor bugfixes (crash):
+    - When running Tor with an option like --verify-config or
+      --dump-config that does not start the event loop, avoid crashing
+      if we try to exit early because of an error. Fixes bug 32407;
+      bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (directory):
+    - When checking if a directory connection is anonymous, test if the
+      circuit was marked for close before looking at its channel. This
+      avoids a BUG() stacktrace if the circuit was previously closed.
+      Fixes bug 31958; bugfix on 0.4.2.1-alpha.
+
+  o Minor bugfixes (shellcheck):
+    - Fix minor shellcheck errors in the git-*.sh scripts. Fixes bug
+      32402; bugfix on 0.4.2.1-alpha.
+    - Start checking most scripts for shellcheck errors again. Fixes bug
+      32402; bugfix on 0.4.2.1-alpha.
+
+  o Testing (continuous integration):
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+
+
+Changes in version 0.4.2.3-alpha - 2019-10-24
+  This release fixes several bugs from the previous alpha release, and
+  from earlier versions of Tor.
+
+  o Major bugfixes (relay):
+    - Relays now respect their AccountingMax bandwidth again. When
+      relays entered "soft" hibernation (which typically starts when
+      we've hit 90% of our AccountingMax), we had stopped checking
+      whether we should enter hard hibernation. Soft hibernation refuses
+      new connections and new circuits, but the existing circuits can
+      continue, meaning that relays could have exceeded their configured
+      AccountingMax. Fixes bug 32108; bugfix on 0.4.0.1-alpha.
+
+  o Major bugfixes (v3 onion services):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor feature (onion services, control port):
+    - The ADD_ONION command's keyword "BEST" now defaults to ED25519-V3
+      (v3) onion services. Previously it defaulted to RSA1024 (v2).
+      Closes ticket 29669.
+
+  o Minor features (testing):
+    - When running tests that attempt to look up hostnames, replace the
+      libc name lookup functions with ones that do not actually touch
+      the network. This way, the tests complete more quickly in the
+      presence of a slow or missing DNS resolver. Closes ticket 31841.
+
+  o Minor features (testing, continuous integration):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Minor bugfixes (build system):
+    - Interpret "--disable-module-dirauth=no" correctly. Fixes bug
+      32124; bugfix on 0.3.4.1-alpha.
+    - Interpret "--with-tcmalloc=no" correctly. Fixes bug 32124; bugfix
+      on 0.2.0.20-rc.
+    - Stop failing when jemalloc is requested, but tcmalloc is not
+      found. Fixes bug 32124; bugfix on 0.3.5.1-alpha.
+    - When pkg-config is not installed, or a library that depends on
+      pkg-config is not found, tell the user what to do to fix the
+      problem. Fixes bug 31922; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (connections):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (error handling):
+    - Always lock the backtrace buffer before it is used. Fixes bug
+      31734; bugfix on 0.2.5.3-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (process management):
+    - Remove overly strict assertions that triggered when a pluggable
+      transport failed to launch. Fixes bug 31091; bugfix
+      on 0.4.0.1-alpha.
+    - Remove an assertion in the Unix process backend. This assertion
+      would trigger when we failed to find the executable for a child
+      process. Fixes bug 31810; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Avoid intermittent test failures due to a test that had relied on
+      inconsistent timing sources. Fixes bug 31995; bugfix
+      on 0.3.1.3-alpha.
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (tls, logging):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Minor bugfixes (v3 onion services):
+    - Fix an implicit conversion from ssize_t to size_t discovered by
+      Coverity. Fixes bug 31682; bugfix on 0.4.2.1-alpha.
+    - Fix a memory leak in an unlikely error code path when encoding HS
+      DoS establish intro extension cell. Fixes bug 32063; bugfix
+      on 0.4.2.1-alpha.
+    - When cleaning up intro circuits for a v3 onion service, don't
+      remove circuits that have an established or pending circuit, even
+      if they ran out of retries. This way, we don't remove a circuit on
+      its last retry. Fixes bug 31652; bugfix on 0.3.2.1-alpha.
+
+  o Documentation:
+    - Correct the description of "GuardLifetime". Fixes bug 31189;
+      bugfix on 0.3.0.1-alpha.
+    - Make clear in the man page, in both the bandwidth section and the
+      AccountingMax section, that Tor counts in powers of two, not
+      powers of ten: 1 GByte is 1024*1024*1024 bytes, not one billion
+      bytes. Resolves ticket 32106.
+
+
+Changes in version 0.4.2.2-alpha - 2019-10-07
+  This release fixes several bugs from the previous alpha release, and
+  from earlier versions. It also includes a change in authorities, so
+  that they begin to reject the currently unsupported release series.
+
+  o Major features (directory authorities):
+    - Directory authorities now reject relays running all currently
+      deprecated release series. The currently supported release series
+      are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549.
+
+  o Major bugfixes (embedded Tor):
+    - Avoid a possible crash when restarting Tor in embedded mode and
+      enabling a different set of publish/subscribe messages. Fixes bug
+      31898; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (torrc parsing):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (auto-formatting scripts):
+    - When annotating C macros, never generate a line that our check-
+      spaces script would reject. Closes ticket 31759.
+    - When annotating C macros, try to remove cases of double-negation.
+      Closes ticket 31779.
+
+  o Minor features (continuous integration):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2
+      Country database. Closes ticket 31931.
+
+  o Minor features (maintenance scripts):
+    - Add a Coccinelle script to detect bugs caused by incrementing or
+      decrementing a variable inside a call to log_debug(). Since
+      log_debug() is a macro whose arguments are conditionally
+      evaluated, it is usually an error to do this. One such bug was
+      30628, in which SENDME cells were miscounted by a decrement
+      operator inside a log_debug() call. Closes ticket 30743.
+
+  o Minor features (onion services v3):
+    - Assist users who try to setup v2 client authorization in v3 onion
+      services by pointing them to the right documentation. Closes
+      ticket 28966.
+
+  o Minor bugfixes (Appveyor continuous integration):
+    - Avoid spurious errors when Appveyor CI fails before the install
+      step. Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (best practices tracker):
+    - When listing overbroad exceptions, do not also list problems, and
+      do not list insufficiently broad exceptions. Fixes bug 31338;
+      bugfix on 0.4.2.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix the MAPADDRESS controller command to accept one or more
+      arguments. Previously, it required two or more arguments, and
+      ignored the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Add a missing check for HAVE_PTHREAD_H, because the backtrace code
+      uses mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha.
+    - Disable backtrace signal handlers when shutting down tor. Fixes
+      bug 31614; bugfix on 0.2.5.2-alpha.
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+    - When initialising log domain masks, only set known log domains.
+      Fixes bug 31854; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (modules):
+    - Explain what the optional Directory Authority module is, and what
+      happens when it is disabled. Fixes bug 31825; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (multithreading):
+    - Avoid some undefined behaviour when freeing mutexes. Fixes bug
+      31736; bugfix on 0.0.7.
+
+  o Minor bugfixes (relay):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (tests, SunOS):
+    - Avoid a map_anon_nofork test failure due to a signed/unsigned
+      integer comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha.
+
+  o Code simplification and refactoring:
+    - Refactor connection_control_process_inbuf() to reduce the size of
+      a practracker exception. Closes ticket 31840.
+    - Refactor the microdescs_parse_from_string() function into smaller
+      pieces, for better comprehensibility. Closes ticket 31675.
+    - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit
+      tests and fuzzers, rather than using hard-coded values. Closes
+      ticket 31334.
+    - Interface for function `decrypt_desc_layer` cleaned up. Closes
+      ticket 31589.
+
+  o Documentation:
+    - Document the signal-safe logging behaviour in the tor man page.
+      Also add some comments to the relevant functions. Closes
+      ticket 31839.
+    - Explain why we can't destroy the backtrace buffer mutex. Explain
+      why we don't need to destroy the log mutex. Closes ticket 31736.
+    - The Tor source code repository now includes a (somewhat dated)
+      description of Tor's modular architecture, in doc/HACKING/design.
+      This is based on the old "tor-guts.git" repository, which we are
+      adopting and superseding. Closes ticket 31849.
+
+
+Changes in version 0.4.1.6 - 2019-09-19
+  This release backports several bugfixes to improve stability and
+  correctness.  Anyone experiencing build problems or crashes with 0.4.1.5,
+  or experiencing reliability issues with single onion services, should
+  upgrade.
+
+  o Major bugfixes (crash, Linux, Android, backport from 0.4.2.1-alpha):
+    - Tolerate systems (including some Android installations) where
+      madvise and MADV_DONTDUMP are available at build-time, but not at
+      run time. Previously, these systems would notice a failed syscall
+      and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha.
+    - Tolerate systems (including some Linux installations) where
+      madvise and/or MADV_DONTFORK are available at build-time, but not
+      at run time. Previously, these systems would notice a failed
+      syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Add more stub functions to fix compilation on Android with link-
+      time optimization when --disable-module-dirauth is used.
+      Previously, these compilation settings would make the compiler
+      look for functions that didn't exist. Fixes bug 31552; bugfix
+      on 0.4.1.1-alpha.
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix the MAPADDRESS controller command to accept one or more
+      arguments. Previously, it required two or more arguments, and ignored
+      the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6, backport from 0.4.2.1-alpha):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (v3 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v3 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v3 single onion services used a 3-hop
+      path when rend circuits were retried after a remote or delayed
+      failure, but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+
+Changes in version 0.4.2.1-alpha - 2019-09-17
+  This is the first alpha release in the 0.4.2.x series. It adds new
+  defenses for denial-of-service attacks against onion services. It also
+  includes numerous kinds of bugfixes and refactoring to help improve
+  Tor's stability and ease of development.
+
+  o Major features (onion service v3, denial of service):
+    - Add onion service introduction denial of service defenses. Intro
+      points can now rate-limit client introduction requests, using
+      parameters that can be sent by the service within the
+      ESTABLISH_INTRO cell. If the cell extension for this is not used,
+      the intro point will honor the consensus parameters. Closes
+      ticket 30924.
+
+  o Major bugfixes (circuit build, guard):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close.
+      Previously we could end up in the situation where a subsystem is
+      notified of a circuit opening, but the circuit is still marked for
+      close, leading to undesirable behavior. Fixes bug 30871; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (crash, Linux, Android):
+    - Tolerate systems (including some Android installations) where
+      madvise and MADV_DONTDUMP are available at build-time, but not at
+      run time. Previously, these systems would notice a failed syscall
+      and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha.
+    - Tolerate systems (including some Linux installations) where
+      madvise and/or MADV_DONTFORK are available at build-time, but not
+      at run time. Previously, these systems would notice a failed
+      syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha.
+
+  o Minor features (best practices tracker):
+    - Our best-practices tracker now integrates with our include-checker
+      tool to keep track of how many layering violations we have not yet
+      fixed. We hope to reduce this number over time to improve Tor's
+      modularity. Closes ticket 31176.
+    - Add a TOR_PRACTRACKER_OPTIONS variable for passing arguments to
+      practracker from the environment. We may want this for continuous
+      integration. Closes ticket 31309.
+    - Give a warning rather than an error when a practracker exception
+      is violated by a small amount, add a --list-overbroad option to
+      practracker that lists exceptions that are stricter than they need
+      to be, and provide an environment variable for disabling
+      practracker. Closes ticket 30752.
+    - Our best-practices tracker now looks at headers as well as C
+      files. Closes ticket 31175.
+
+  o Minor features (build system):
+    - Add --disable-manpage and --disable-html-manual options to
+      configure script. This will enable shortening build times by not
+      building documentation. Resolves issue 19381.
+
+  o Minor features (compilation):
+    - Log a more useful error message when we are compiling and one of
+      the compile-time hardening options we have selected can be linked
+      but not executed. Closes ticket 27530.
+
+  o Minor features (configuration):
+    - The configuration code has been extended to allow splitting
+      configuration data across multiple objects. Previously, all
+      configuration data needed to be kept in a single object, which
+      tended to become bloated. Closes ticket 31240.
+
+  o Minor features (continuous integration):
+    - When running CI builds on Travis, put some random data in
+      ~/.torrc, to make sure no tests are reading the Tor configuration
+      file from its default location. Resolves issue 30102.
+
+  o Minor features (debugging):
+    - Log a nonfatal assertion failure if we encounter a configuration
+      line whose command is "CLEAR" but which has a nonempty value. This
+      should be impossible, according to the rules of our configuration
+      line parsing. Closes ticket 31529.
+
+  o Minor features (git hooks):
+    - Our pre-commit git hook now checks for a special file before
+      running practracker, so that practracker only runs on branches
+      that are based on master. Since the pre-push hook calls the pre-
+      commit hook, practracker will also only run before pushes of
+      branches based on master. Closes ticket 30979.
+
+  o Minor features (git scripts):
+    - Add a "--" command-line argument, to separate git-push-all.sh
+      script arguments from arguments that are passed through to git
+      push. Closes ticket 31314.
+    - Add a -r <remote-name> argument to git-push-all.sh, so the script
+      can push test branches to a personal remote. Closes ticket 31314.
+    - Add a -t <test-branch-prefix> argument to git-merge-forward.sh and
+      git-push-all.sh, which makes these scripts create, merge forward,
+      and push test branches. Closes ticket 31314.
+    - Add a -u argument to git-merge-forward.sh, so that the script can
+      re-use existing test branches after a merge failure and fix.
+      Closes ticket 31314.
+    - Add a TOR_GIT_PUSH env var, which sets the default git push
+      command and arguments for git-push-all.sh. Closes ticket 31314.
+    - Add a TOR_PUSH_DELAY variable to git-push-all.sh, which makes the
+      script push master and maint branches with a delay between each
+      branch. These delays trigger the CI jobs in a set order, which
+      should show the most likely failures first. Also make pushes
+      atomic by default, and make the script pass any command-line
+      arguments to git push. Closes ticket 29879.
+    - Call the shellcheck script from the pre-commit hook. Closes
+      ticket 30967.
+    - Skip pushing test branches that are the same as a remote
+      maint/release/master branch in git-push-all.sh by default. Add a
+      -s argument, so git-push-all.sh can push all test branches. Closes
+      ticket 31314.
+
+  o Minor features (IPv6, logging):
+    - Log IPv6 addresses as well as IPv4 addresses when describing
+      routerinfos, routerstatuses, and nodes. Closes ticket 21003.
+
+  o Minor features (onion service v3):
+    - Do not allow single hop clients to fetch or post an HS descriptor
+      from an HSDir. Closes ticket 24964.
+
+  o Minor features (onion service):
+    - Disallow single-hop clients at the introduction point. We've
+      removed Tor2web support a while back and single-hop rendezvous
+      attempts are blocked at the relays. This change should remove load
+      off the network from spammy clients. Close ticket 24963.
+
+  o Minor features (stem tests):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor features (testing):
+    - Add a script to invoke "tor --dump-config" and "tor
+      --verify-config" with various configuration options, and see
+      whether tor's resulting configuration or error messages are what
+      we expect. Use it for integration testing of our +Option and
+      /Option flags. Closes ticket 31637.
+    - Improve test coverage for our existing configuration parsing and
+      management API. Closes ticket 30893.
+    - Add integration tests to make sure that practracker gives the
+      outputs we expect. Closes ticket 31477.
+    - The practracker self-tests are now run as part of the Tor test
+      suite. Closes ticket 31304.
+
+  o Minor features (token bucket):
+    - Implement a generic token bucket that uses a single counter, for
+      use in anti-DoS onion service work. Closes ticket 30687.
+
+  o Minor bugfixes (best practices tracker):
+    - Fix a few issues in the best-practices script, including tests,
+      tab tolerance, error reporting, and directory-exclusion logic.
+      Fixes bug 29746; bugfix on 0.4.1.1-alpha.
+    - When running check-best-practices, only consider files in the src
+      subdirectory. Previously we had recursively considered all
+      subdirectories, which made us get confused by the temporary
+      directories made by "make distcheck". Fixes bug 31578; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (build system):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (chutney, makefiles, documentation):
+    - "make test-network-all" now shows the warnings from each test-
+      network.sh run on the console, so developers see new warnings
+      early. We've also improved the documentation for this feature, and
+      renamed a Makefile variable so the code is self-documenting. Fixes
+      bug 30455; bugfix on 0.3.0.4-rc.
+
+  o Minor bugfixes (compilation):
+    - Add more stub functions to fix compilation on Android with link-
+      time optimization when --disable-module-dirauth is used.
+      Previously, these compilation settings would make the compiler
+      look for functions that didn't exist. Fixes bug 31552; bugfix
+      on 0.4.1.1-alpha.
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (configuration):
+    - Invalid floating-point values in the configuration file are now
+      treated as errors in the configuration. Previously, they were
+      ignored and treated as zero. Fixes bug 31475; bugfix on 0.0.1.
+
+  o Minor bugfixes (coverity):
+    - Add an assertion when parsing a BEGIN cell so that coverity can be
+      sure that we are not about to dereference a NULL address. Fixes
+      bug 31026; bugfix on 0.2.4.7-alpha. This is CID 1447296.
+    - In our siphash implementation, when building for coverity, use
+      memcpy in place of a switch statement, so that coverity can tell
+      we are not accessing out-of-bounds memory. Fixes bug 31025; bugfix
+      on 0.2.8.1-alpha. This is tracked as CID 1447293 and 1447295.
+    - Fix several coverity warnings from our unit tests. Fixes bug
+      31030; bugfix on 0.2.4.1-alpha, 0.3.2.1-alpha, and 0.4.0.1-alpha.
+
+  o Minor bugfixes (developer tooling):
+    - Only log git script changes in the post-merge script when the
+      merge was to the master branch. Fixes bug 31040; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Return a distinct status when formatting annotations fails. Fixes
+      bug 30780; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (error handling):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - When tor aborts due to an error, close log file descriptors before
+      aborting. Closing the logs makes some OSes flush log file buffers,
+      rather than deleting buffered log lines. Fixes bug 31594; bugfix
+      on 0.2.5.2-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (git hooks):
+    - Remove a duplicate call to practracker from the pre-push hook. The
+      pre-push hook already calls the pre-commit hook, which calls
+      practracker. Fixes bug 31462; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (git scripts):
+    - Stop hard-coding the bash path in the git scripts. Some OSes don't
+      have bash in /usr/bin, others have an ancient bash at this path.
+      Fixes bug 30840; bugfix on 0.4.0.1-alpha.
+    - Stop hard-coding the tor master branch name and worktree path in
+      the git scripts. Fixes bug 30841; bugfix on 0.4.0.1-alpha.
+    - Allow git-push-all.sh to be run from any directory. Previously,
+      the script only worked if run from an upstream worktree directory.
+      Closes ticket 31678.
+
+  o Minor bugfixes (guards):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (ipv6):
+    - Check for private IPv6 addresses alongside their IPv4 equivalents
+      when authorities check descriptors. Previously, we only checked
+      for private IPv4 addresses. Fixes bug 31088; bugfix on
+      0.2.3.21-rc. Patch by Neel Chauhan.
+    - When parsing microdescriptors, we should check the IPv6 exit
+      policy alongside IPv4. Previously, we checked both exit policies
+      for only router info structures, while microdescriptors were
+      IPv4-only. Fixes bug 27284; bugfix on 0.2.3.1-alpha. Patch by
+      Neel Chauhan.
+
+  o Minor bugfixes (logging):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+    - Fix a code issue that would have broken our parsing of log domains
+      as soon as we had 33 of them. Fortunately, we still only have 29.
+      Fixes bug 31451; bugfix on 0.4.1.4-rc.
+
+  o Minor bugfixes (memory management):
+    - Stop leaking a small amount of memory in nt_service_install(), in
+      unreachable code. Fixes bug 30799; bugfix on 0.2.0.7-alpha. Patch
+      by Xiaoyin Liu.
+
+  o Minor bugfixes (networking, IP addresses):
+    - When parsing addresses via Tor's internal DNS lookup API, reject
+      IPv4 addresses in square brackets, and accept IPv6 addresses in
+      square brackets. This change completes the work started in 23082,
+      making address parsing consistent between tor's internal DNS
+      lookup and address parsing APIs. Fixes bug 30721; bugfix
+      on 0.2.1.5-alpha.
+    - When parsing addresses via Tor's internal address:port parsing and
+      DNS lookup APIs, require IPv6 addresses with ports to have square
+      brackets. But allow IPv6 addresses without ports, whether or not
+      they have square brackets. Fixes bug 30721; bugfix
+      on 0.2.1.5-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - When purging the client descriptor cache, close any introduction
+      point circuits associated with purged cache entries. This avoids
+      picking those circuits later when connecting to the same
+      introduction points. Fixes bug 30921; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - In the hs_ident_circuit_t data structure, remove the unused field
+      circuit_type and the respective argument in hs_ident_circuit_new().
+      This field was set by clients (for introduction) and services (for
+      introduction and rendezvous) but was never used afterwards. Fixes
+      bug 31490; bugfix on 0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (operator tools):
+    - Make tor-print-ed-signing-cert(1) print certificate expiration
+      date in RFC 1123 and UNIX timestamp formats, to make output
+      machine readable. Fixes bug 31012; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (rust):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+    - Raise the minimum rustc version to 1.31.0, as checked by configure
+      and CI. Fixes bug 31442; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (sendme, code structure):
+    - Rename the trunnel SENDME file definition from sendme.trunnel to
+      sendme_cell.trunnel to avoid having twice sendme.{c|h} in the
+      repository. Fixes bug 30769; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (statistics):
+    - Stop removing the ed25519 signature if the extra info file is too
+      big. If the signature data was removed, but the keyword was kept,
+      this could result in an unparseable extra info file. Fixes bug
+      30958; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (subsystems):
+    - Make the subsystem init order match the subsystem module
+      dependencies. Call windows process security APIs as early as
+      possible. Initialize logging before network and time, so that
+      network and time can use logging. Fixes bug 31615; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Teach the util/socketpair_ersatz test to work correctly when we
+      have no network stack configured. Fixes bug 30804; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (v2 single onion services):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (v3 single onion services):
+    - Always retry v3 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v3 single onion services used a 3-hop
+      path when rend circuits were retried after a remote or delayed
+      failure, but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation:
+    - Improve documentation in circuit padding subsystem. Patch by
+      Tobias Pulls. Closes ticket 31113.
+    - Include an example usage for IPv6 ORPort in our sample torrc.
+      Closes ticket 31320; patch from Ali Raheem.
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+  o Removed features:
+    - No longer include recommended package digests in votes as detailed
+      in proposal 301. The RecommendedPackages torrc option is
+      deprecated and will no longer have any effect. "package" lines
+      will still be considered when computing consensuses for consensus
+      methods that include them. (This change has no effect on the list
+      of recommended Tor versions, which is still in use.) Closes
+      ticket 29738.
+    - Remove torctl.in from contrib/dist directory. Resolves
+      ticket 30550.
+
+  o Testing:
+    - Run shellcheck for all non-third-party shell scripts that are
+      shipped with Tor. Closes ticket 29533.
+    - When checking shell scripts, ignore any user-created directories.
+      Closes ticket 30967.
+
+  o Code simplification and refactoring (config handling):
+    - Extract our variable manipulation code from confparse.c to a new
+      lower-level typedvar.h module. Closes ticket 30864.
+    - Lower another layer of object management from confparse.c to a
+      more general tool. Now typed structure members are accessible via
+      an abstract type. Implements ticket 30914.
+    - Move our backend logic for working with configuration and state
+      files into a lower-level library, since it no longer depends on
+      any tor-specific functionality. Closes ticket 31626.
+    - Numerous simplifications in configuration-handling logic: remove
+      duplicated macro definitions, replace magical names with flags,
+      and refactor "TestingTorNetwork" to use the same default-option
+      logic as the rest of Tor. Closes ticket 30935.
+    - Replace our ad-hoc set of flags for configuration variables and
+      configuration variable types with fine-grained orthogonal flags
+      corresponding to the actual behavior we want. Closes ticket 31625.
+
+  o Code simplification and refactoring (misc):
+    - Eliminate some uses of lower-level control reply abstractions,
+      primarily in the onion_helper functions. Closes ticket 30889.
+    - Rework bootstrap tracking to use the new publish-subscribe
+      subsystem. Closes ticket 29976.
+    - Rewrite format_node_description() and router_get_verbose_nickname()
+      to use strlcpy() and strlcat(). The previous implementation used
+      memcpy() and pointer arithmetic, which was error-prone. Closes
+      ticket 31545. This is CID 1452819.
+    - Split extrainfo_dump_to_string() into smaller functions. Closes
+      ticket 30956.
+    - Use the ptrdiff_t type consistently for expressing variable
+      offsets and pointer differences. Previously we incorrectly (but
+      harmlessly) used int and sometimes off_t for these cases. Closes
+      ticket 31532.
+    - Use the subsystems mechanism to manage the main event loop code.
+      Closes ticket 30806.
+    - Various simplifications and minor improvements to the circuit
+      padding machines. Patch by Tobias Pulls. Closes tickets 31112
+      and 31098.
+
+  o Documentation (hard-coded directories):
+    - Improve the documentation for the DirAuthority and FallbackDir
+      torrc options. Closes ticket 30955.
+
+  o Documentation (tor.1 man page):
+    - Fix typo in tor.1 man page: the option is "--help", not "-help".
+      Fixes bug 31008; bugfix on 0.2.2.9-alpha.
+
+
+Changes in version 0.4.1.5 - 2019-08-20
+  This is the first stable release in the 0.4.1.x series. This series
+  adds experimental circuit-level padding, authenticated SENDME cells to
+  defend against certain attacks, and several performance improvements
+  to save on CPU consumption. It fixes bugs in bootstrapping and v3
+  onion services. It also includes numerous smaller features and
+  bugfixes on earlier versions.
+
+  Per our support policy, we will support the 0.4.1.x series for nine
+  months, or until three months after the release of a stable 0.4.2.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.1.4-rc. For a complete list of changes
+  since 0.4.0.5, see the ReleaseNotes file.
+
+  o Directory authority changes:
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Minor features (circuit padding logging):
+    - Demote noisy client-side warn logs about circuit padding to
+      protocol warnings. Add additional log messages and circuit ID
+      fields to help with bug 30992 and any other future issues.
+
+  o Minor bugfixes (circuit padding negotiation):
+    - Bump the circuit padding protocol version to explicitly signify
+      that the HS setup machine support is finalized in 0.4.1.x-stable.
+      This also means that 0.4.1.x-alpha clients will not negotiate
+      padding with 0.4.1.x-stable relays, and 0.4.1.x-stable clients
+      will not negotiate padding with 0.4.1.x-alpha relays (or 0.4.0.x
+      relays). Fixes bug 31356; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (circuit padding):
+    - Ignore non-padding cells on padding circuits. This addresses
+      various warning messages from subsystems that were not expecting
+      padding circuits. Fixes bug 30942; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (clock skew detection):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compatibility, standards compliance):
+    - Fix a bug that would invoke undefined behavior on certain
+      operating systems when trying to asprintf() a string exactly
+      INT_MAX bytes long. We don't believe this is exploitable, but it's
+      better to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha.
+      Found and fixed by Tobias Stoeckmann.
+
+  o Minor bugfixes (compilation warning):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (distribution):
+    - Do not ship any temporary files found in the
+      scripts/maint/practracker directory. Fixes bug 31311; bugfix
+      on 0.4.1.1-alpha.
+
+  o Testing (continuous integration):
+    - In Travis, make stem log a controller trace to the console, and
+      tail stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary. Closes
+      ticket 30694.
+
+
+Changes in version 0.4.1.4-rc - 2019-07-25
+  Tor 0.4.1.4-rc fixes a few bugs from previous versions of Tor, and
+  updates to a new list of fallback directories. If no new bugs are
+  found, the next release in the 0.4.1.x serious should be stable.
+
+  o Major bugfixes (circuit build, guard):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (continuous integration):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+
+  o Minor features (fallback directory list):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor bugfixes (circuit padding):
+    - On relays, properly check that a padding machine is absent before
+      logging a warning about it being absent. Fixes bug 30649; bugfix
+      on 0.4.0.1-alpha.
+    - Add two NULL checks in unreachable places to silence Coverity (CID
+      144729 and 1447291) and better future-proof ourselves. Fixes bug
+      31024; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (crash on exit):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (logging):
+    - Fix a conflict between the flag used for messaging-domain log
+      messages, and the LD_NO_MOCK testing flag. Fixes bug 31080; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a trivial memory leak when parsing an invalid value from a
+      download schedule in the configuration. Fixes bug 30894; bugfix
+      on 0.3.4.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove some dead code from circpad_machine_remove_token() to fix
+      some Coverity warnings (CID 1447298). Fixes bug 31027; bugfix
+      on 0.4.1.1-alpha.
+
+
+Changes in version 0.4.1.3-alpha - 2019-06-25
+  Tor 0.4.1.3-alpha resolves numerous bugs left over from the previous
+  alpha, most of them from earlier release series.
+
+  o Major bugfixes (Onion service reachability):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30852.
+
+  o Minor features (logging):
+    - Give a more useful assertion failure message if we think we have
+      minherit() but we fail to make a region non-inheritable. Give a
+      compile-time warning if our support for minherit() is incomplete.
+      Closes ticket 30686.
+
+  o Minor bugfixes (circuit isolation):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (continuous integration):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - When running as a bridge with pluggable transports, always publish
+      pluggable transport information in our extrainfo descriptor, even
+      if ExtraInfoStatistics is 0. This information is needed by
+      BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha.
+
+  o Documentation:
+    - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md.
+      Closes ticket 30630.
+
+
+Changes in version 0.4.1.2-alpha - 2019-06-06
+  Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the
+  previous alpha, and some much older. It also contains minor testing
+  improvements, and an improvement to the security of our authenticated
+  SENDME implementation.
+
+  o Major bugfixes (bridges):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (flow control, SENDME):
+    - Decrement the stream-level package window after packaging a cell.
+      Previously, it was done inside a log_debug() call, meaning that if
+      debug logs were not enabled, the decrement would never happen, and
+      thus the window would be out of sync with the other end point.
+      Fixes bug 30628; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (onion service reachability):
+    - Properly clean up the introduction point map and associated state
+      when circuits change purpose from onion service circuits to
+      pathbias, measurement, or other circuit types. This may fix some
+      instances of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (authenticated SENDME):
+    - Ensure that there is enough randomness on every circuit to prevent
+      an attacker from successfully predicting the hashes they will need
+      to include in authenticated SENDME cells. At a random interval, if
+      we have not sent randomness already, we now leave some extra space
+      at the end of a cell that we can fill with random bytes. Closes
+      ticket 26846.
+
+  o Minor features (continuous integration):
+    - When running coverage builds on Travis, we now set
+      TOR_TEST_RNG_SEED, to avoid RNG-based coverage differences. Part
+      of ticket 28878.
+
+  o Minor features (maintenance):
+    - Add a new "make autostyle" target that developers can use to apply
+      all automatic Tor style and consistency conversions to the
+      codebase. Closes ticket 30539.
+
+  o Minor features (testing):
+    - The circuitpadding tests now use a reproducible RNG implementation,
+      so that if a test fails, we can learn why. Part of ticket 28878.
+    - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED,
+      to set the RNG seed for tests that use a reproducible RNG. Part of
+      ticket 28878.
+    - When running tests in coverage mode, take additional care to make
+      our coverage deterministic, so that we can accurately track
+      changes in code coverage. Closes ticket 30519.
+
+  o Minor bugfixes (configuration, proxies):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (controller):
+    - POSTDESCRIPTOR requests should work again. Previously, they were
+      broken if a "purpose=" flag was specified. Fixes bug 30580; bugfix
+      on 0.4.1.1-alpha.
+    - Repair the HSFETCH command so that it works again. Previously, it
+      expected a body when it shouldn't have. Fixes bug 30646; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (developer tooling):
+    - Fix pre-push hook to allow fixup and squash commits when pushing
+      to non-upstream git remote. Fixes bug 30286; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (directory authority):
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (NetBSD):
+    - Fix usage of minherit() on NetBSD and other platforms that define
+      MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug
+      30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell.
+
+  o Minor bugfixes (out-of-memory handler):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (shutdown, libevent, memory safety):
+    - Avoid use-after-free bugs when shutting down, by making sure that
+      we shut down libevent only after shutting down all of its users.
+      We believe these are harmless in practice, since they only occur
+      on the shutdown path, and do not involve any attacker-controlled
+      data. Fixes bug 30629; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (static analysis):
+    - Fix several spurious Coverity warnings about the unit tests, to
+      lower our chances of missing real warnings in the future. Fixes
+      bug 30150; bugfix on 0.3.5.1-alpha and various other Tor versions.
+
+  o Testing:
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+
+Changes in version 0.4.1.1-alpha - 2019-05-22
+  This is the first alpha in the 0.4.1.x series. It introduces
+  lightweight circuit padding to make some onion-service circuits harder
+  to distinguish, includes a new "authenticated SENDME" feature to make
+  certain denial-of-service attacks more difficult, and improves
+  performance in several areas.
+
+  o Major features (circuit padding):
+    - Onion service clients now add padding cells at the start of their
+      INTRODUCE and RENDEZVOUS circuits, to make those circuits' traffic
+      look more like general purpose Exit traffic. The overhead for this
+      is 2 extra cells in each direction for RENDEZVOUS circuits, and 1
+      extra upstream cell and 10 downstream cells for INTRODUCE
+      circuits. This feature is only enabled when also supported by the
+      circuit's middle node. (Clients may specify fixed middle nodes
+      with the MiddleNodes option, and may force-disable this feature
+      with the CircuitPadding option.) Closes ticket 28634.
+
+  o Major features (code organization):
+    - Tor now includes a generic publish-subscribe message-passing
+      subsystem that we can use to organize intermodule dependencies. We
+      hope to use this to reduce dependencies between modules that don't
+      need to be related, and to generally simplify our codebase. Closes
+      ticket 28226.
+
+  o Major features (controller protocol):
+    - Controller commands are now parsed using a generalized parsing
+      subsystem. Previously, each controller command was responsible for
+      parsing its own input, which led to strange inconsistencies.
+      Closes ticket 30091.
+
+  o Major features (flow control):
+    - Implement authenticated SENDMEs as detailed in proposal 289. A
+      SENDME cell now includes the digest of the traffic that it
+      acknowledges, so that once an end point receives the SENDME, it
+      can confirm the other side's knowledge of the previous cells that
+      were sent, and prevent certain types of denial-of-service attacks.
+      This behavior is controlled by two new consensus parameters: see
+      the proposal for more details. Fixes ticket 26288.
+
+  o Major features (performance):
+    - Our node selection algorithm now excludes nodes in linear time.
+      Previously, the algorithm was quadratic, which could slow down
+      heavily used onion services. Closes ticket 30307.
+
+  o Major features (performance, RNG):
+    - Tor now constructs a fast secure pseudorandom number generator for
+      each thread, to use when performance is critical. This PRNG is
+      based on AES-CTR, using a buffering construction similar to
+      libottery and the (newer) OpenBSD arc4random() code. It
+      outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for
+      small outputs. Although we believe it to be cryptographically
+      strong, we are only using it when necessary for performance.
+      Implements tickets 29023 and 29536.
+
+  o Major bugfixes (onion service v3):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (circuit padding):
+    - We now use a fast PRNG when scheduling circuit padding. Part of
+      ticket 28636.
+    - Allow the padding machine designer to pick the edges of their
+      histogram instead of trying to compute them automatically using an
+      exponential formula. Resolves some undefined behavior in the case
+      of small histograms and allows greater flexibility on machine
+      design. Closes ticket 29298; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to hold a circuit open until they
+      are done padding it. Closes ticket 28780.
+
+  o Minor features (compile-time modules):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration):
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Minor features (controller):
+    - Add onion service version 3 support to the HSFETCH command.
+      Previously, only version 2 onion services were supported. Closes
+      ticket 25417. Patch by Neel Chauhan.
+
+  o Minor features (debugging):
+    - Introduce tor_assertf() and tor_assertf_nonfatal() to enable
+      logging of additional information during assert failure. Now we
+      can use format strings to include information for trouble
+      shooting. Resolves ticket 29662.
+
+  o Minor features (defense in depth):
+    - In smartlist_remove_keeporder(), set unused pointers to NULL, in
+      case a bug causes them to be used later. Closes ticket 30176.
+      Patch from Tobias Stoeckmann.
+    - Tor now uses a cryptographically strong PRNG even for decisions
+      that we do not believe are security-sensitive. Previously, for
+      performance reasons, we had used a trivially predictable linear
+      congruential generator algorithm for certain load-balancing and
+      statistical sampling decisions. Now we use our fast RNG in those
+      cases. Closes ticket 29542.
+
+  o Minor features (developer tools):
+    - Tor's "practracker" test script now checks for files and functions
+      that seem too long and complicated. Existing overlong functions
+      and files are accepted for now, but should eventually be
+      refactored. Closes ticket 29221.
+    - Add some scripts used for git maintenance to scripts/git. Closes
+      ticket 29391.
+    - Call practracker from pre-push and pre-commit git hooks to let
+      developers know if they made any code style violations. Closes
+      ticket 30051.
+    - Add a script to check that each header has a well-formed and
+      unique guard macro. Closes ticket 29756.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30522.
+
+  o Minor features (HTTP tunnel):
+    - Return an informative web page when the HTTPTunnelPort is used as
+      an HTTP proxy. Closes ticket 27821, patch by "eighthave".
+
+  o Minor features (IPv6, v3 onion services):
+    - Make v3 onion services put IPv6 addresses in service descriptors.
+      Before this change, service descriptors only contained IPv4
+      addresses. Implements 26992.
+
+  o Minor features (modularity):
+    - The "--disable-module-dirauth" compile-time option now disables
+      even more dirauth-only code. Closes ticket 30345.
+
+  o Minor features (performance):
+    - Use OpenSSL's implementations of SHA3 when available (in OpenSSL
+      1.1.1 and later), since they tend to be faster than tiny-keccak.
+      Closes ticket 28837.
+
+  o Minor features (testing):
+    - Tor's unit test code now contains helper functions to replace the
+      PRNG with a deterministic or reproducible version for testing.
+      Previously, various tests implemented this in various ways.
+      Implements ticket 29732.
+    - We now have a script, cov-test-determinism.sh, to identify places
+      where our unit test coverage has become nondeterministic. Closes
+      ticket 29436.
+    - Check that representative subsets of values of `int` and `unsigned
+      int` can be represented by `void *`. Resolves issue 29537.
+
+  o Minor bugfixes (bridge authority):
+    - Bridge authorities now set bridges as running or non-running when
+      about to dump their status to a file. Previously, they set bridges
+      as running in response to a GETINFO command, but those shouldn't
+      modify data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (channel padding statistics):
+    - Channel padding write totals and padding-enabled totals are now
+      counted properly in relay extrainfo descriptors. Fixes bug 29231;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (circuit padding):
+    - Add a "CircuitPadding" torrc option to disable circuit padding.
+      Fixes bug 28693; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to specify that they do not
+      contribute much overhead, and provide consensus flags and torrc
+      options to force clients to only use these low overhead machines.
+      Fixes bug 29203; bugfix on 0.4.0.1-alpha.
+    - Provide a consensus parameter to fully disable circuit padding, to
+      be used in emergency network overload situations. Fixes bug 30173;
+      bugfix on 0.4.0.1-alpha.
+    - The circuit padding subsystem will no longer schedule padding if
+      dormant mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha.
+    - Inspect a circuit-level cell queue before sending padding, to
+      avoid sending padding while too much data is already queued. Fixes
+      bug 29204; bugfix on 0.4.0.1-alpha.
+    - Avoid calling monotime_absolute_usec() in circuit padding machines
+      that do not use token removal or circuit RTT estimation. Fixes bug
+      29085; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Teach the controller parser to distinguish an object preceded by
+      an argument list from one without. Previously, it couldn't
+      distinguish an argument list from the first line of a multiline
+      object. Fixes bug 29984; bugfix on 0.2.3.8-alpha.
+
+  o Minor bugfixes (directory authority, ipv6):
+    - Directory authorities with IPv6 support now always mark themselves
+      as reachable via IPv6. Fixes bug 24338; bugfix on 0.2.4.1-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (documentation):
+    - Improve the documentation for using MapAddress with ".exit". Fixes
+      bug 30109; bugfix on 0.1.0.1-rc.
+    - Improve the monotonic time module and function documentation to
+      explain what "monotonic" actually means, and document some results
+      that have surprised people. Fixes bug 29640; bugfix
+      on 0.2.9.1-alpha.
+    - Use proper formatting when providing an example on quoting options
+      that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc.
+
+  o Minor bugfixes (logging):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+    - Warn operators when the MyFamily option is set but ContactInfo is
+      missing, as the latter should be set too. Fixes bug 25110; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+    - When refusing to launch a controller's HSFETCH request because of
+      rate-limiting, respond to the controller with a new response,
+      "QUERY_RATE_LIMITED". Previously, we would log QUERY_NO_HSDIR for
+      this case. Fixes bug 28269; bugfix on 0.3.1.1-alpha. Patch by
+      Neel Chauhan.
+    - When relaunching a circuit to a rendezvous service, mark the
+      circuit as needing high-uptime routers as appropriate. Fixes bug
+      17357; bugfix on 0.1.0.1-rc. Patch by Neel Chauhan.
+    - Stop ignoring IPv6 link specifiers sent to v3 onion services.
+      (IPv6 support for v3 onion services is still incomplete: see
+      ticket 23493 for details.) Fixes bug 23588; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (onion services, performance):
+    - When building circuits to onion services, call tor_addr_parse()
+      less often. Previously, we called tor_addr_parse() in
+      circuit_is_acceptable() even if its output wasn't used. This
+      change should improve performance when building circuits. Fixes
+      bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan.
+
+  o Minor bugfixes (performance):
+    - When checking whether a node is a bridge, use a fast check to make
+      sure that its identity is set. Previously, we used a constant-time
+      check, which is not necessary in this case. Fixes bug 30308;
+      bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as
+      well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (probability distributions):
+    - Refactor and improve parts of the probability distribution code
+      that made Coverity complain. Fixes bug 29805; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (python):
+    - Stop assuming that /usr/bin/python3 exists. For scripts that work
+      with python2, use /usr/bin/python. Otherwise, use /usr/bin/env
+      python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha.
+
+  o Minor bugfixes (relay):
+    - When running as a relay, if IPv6Exit is set to 1 while ExitRelay
+      is auto, act as if ExitRelay is 1. Previously, we would ignore
+      IPv6Exit if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on
+      0.3.5.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including bandwidth usage
+      statistics, GeoIPFile hashes, ServerTransportPlugin lines, and
+      bridge statistics by country in extra-info documents. Fixes bug
+      29018; bugfix on 0.2.4.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Call setrlimit() to disable core dumps in test_bt_cl.c. Previously
+      we used `ulimit -c` in test_bt.sh, which violates POSIX shell
+      compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha.
+    - Fix some incorrect code in the v3 onion service unit tests. Fixes
+      bug 29243; bugfix on 0.3.2.1-alpha.
+    - In the "routerkeys/*" tests, check the return values of mkdir()
+      for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha.
+      Found by Coverity as CID 1444254.
+    - Split test_utils_general() into several smaller test functions.
+      This makes it easier to perform resource deallocation on assert
+      failure, and fixes Coverity warnings CID 1444117 and CID 1444118.
+      Fixes bug 29823; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - Fix a memory leak in tor-resolve that could happen if Tor gave it
+      a malformed SOCKS response. (Memory leaks in tor-resolve don't
+      actually matter, but it's good to fix them anyway.) Fixes bug
+      30151; bugfix on 0.4.0.1-alpha.
+
+  o Code simplification and refactoring:
+    - Abstract out the low-level formatting of replies on the control
+      port. Implements ticket 30007.
+    - Add several assertions in an attempt to fix some Coverity
+      warnings. Closes ticket 30149.
+    - Introduce a connection_dir_buf_add() helper function that checks
+      for compress_state of dir_connection_t and automatically writes a
+      string to directory connection with or without compression.
+      Resolves issue 28816.
+    - Make the base32_decode() API return the number of bytes written,
+      for consistency with base64_decode(). Closes ticket 28913.
+    - Move most relay-only periodic events out of mainloop.c into the
+      relay subsystem. Closes ticket 30414.
+    - Refactor and encapsulate parts of the codebase that manipulate
+      crypt_path_t objects. Resolves issue 30236.
+    - Refactor several places in our code that Coverity incorrectly
+      believed might have memory leaks. Closes ticket 30147.
+    - Remove redundant return values in crypto_format, and the
+      associated return value checks elsewhere in the code. Make the
+      implementations in crypto_format consistent, and remove redundant
+      code. Resolves ticket 29660.
+    - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that
+      it is not a constant-time function. Closes ticket 30309.
+    - Replace hs_desc_link_specifier_t with link_specifier_t, and remove
+      all hs_desc_link_specifier_t-specific code. Fixes bug 22781;
+      bugfix on 0.3.2.1-alpha.
+    - Simplify v3 onion service link specifier handling code. Fixes bug
+      23576; bugfix on 0.3.2.1-alpha.
+    - Split crypto_digest.c into NSS code, OpenSSL code, and shared
+      code. Resolves ticket 29108.
+    - Split control.c into several submodules, in preparation for
+      distributing its current responsibilities throughout the codebase.
+      Closes ticket 29894.
+    - Start to move responsibility for knowing about periodic events to
+      the appropriate subsystems, so that the mainloop doesn't need to
+      know all the periodic events in the rest of the codebase.
+      Implements tickets 30293 and 30294.
+
+  o Documentation:
+    - Document how to find git commits and tags for bug fixes in
+      CodingStandards.md. Update some file documentation. Closes
+      ticket 30261.
+
+  o Removed features:
+    - Remove the linux-tor-prio.sh script from contrib/operator-tools
+      directory. Resolves issue 29434.
+    - Remove the obsolete OpenSUSE initscript. Resolves issue 30076.
+    - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves
+      issue 30075.
+
+  o Code simplification and refactoring (shell scripts):
+    - Clean up many of our shell scripts to fix shellcheck warnings.
+      These include autogen.sh (ticket 26069), test_keygen.sh (ticket
+      29062), test_switch_id.sh (ticket 29065), test_rebind.sh (ticket
+      29063), src/test/fuzz/minimize.sh (ticket 30079), test_rust.sh
+      (ticket 29064), torify (ticket 29070), asciidoc-helper.sh (29926),
+      fuzz_multi.sh (30077), fuzz_static_testcases.sh (ticket 29059),
+      nagios-check-tor-authority-cert (ticket 29071),
+      src/test/fuzz/fixup_filenames.sh (ticket 30078), test-network.sh
+      (ticket 29060), test_key_expiration.sh (ticket 30002),
+      zero_length_keys.sh (ticket 29068), and test_workqueue_*.sh
+      (ticket 29067).
+
+  o Testing (chutney):
+    - In "make test-network-all", test IPv6-only v3 single onion
+      services, using the chutney network single-onion-v23-ipv6-md.
+      Closes ticket 27251.
+
+
+Changes in version 0.4.0.5 - 2019-05-02
+  This is the first stable release in the 0.4.0.x series. It contains
+  improvements for power management and bootstrap reporting, as well as
+  preliminary backend support for circuit padding to prevent some kinds
+  of traffic analysis. It also continues our work in refactoring Tor for
+  long-term maintainability.
+
+  Per our support policy, we will support the 0.4.0.x series for nine
+  months, or until three months after the release of a stable 0.4.1.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.0.4-rc. For a complete list of changes
+  since 0.3.5.7, see the ReleaseNotes file.
+
+  o Minor features (continuous integration):
+    - In Travis, tell timelimit to use stem's backtrace signals, and
+      launch python directly from timelimit, so python receives the
+      signals from timelimit, rather than make. Closes ticket 30117.
+
+  o Minor features (diagnostic):
+    - Add more diagnostic log messages in an attempt to solve the issue
+      of NUL bytes appearing in a microdescriptor cache. Related to
+      ticket 28223.
+
+  o Minor features (testing):
+    - Use the approx_time() function when setting the "Expires" header
+      in directory replies, to make them more testable. Needed for
+      ticket 30001.
+
+  o Minor bugfixes (rust):
+    - Abort on panic in all build profiles, instead of potentially
+      unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (shellcheck):
+    - Look for scripts in their correct locations during "make
+      shellcheck". Previously we had looked in the wrong place during
+      out-of-tree builds. Fixes bug 30263; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Check the time in the "Expires" header using approx_time(). Fixes
+      bug 30001; bugfix on 0.4.0.4-rc.
+
+  o Minor bugfixes (UI):
+    - Lower log level of unlink() errors during bootstrap. Fixes bug
+      29930; bugfix on 0.4.0.1-alpha.
+
+
+Changes in version 0.4.0.4-rc - 2019-04-11
+  Tor 0.4.0.4-rc is the first release candidate in its series; it fixes
+  several bugs from earlier versions, including some that had affected
+  stability, and one that prevented relays from working with NSS.
+
+  o Major bugfixes (NSS, relay):
+    - When running with NSS, disable TLS 1.2 ciphersuites that use
+      SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for
+      these ciphersuites don't work -- which caused relays to fail to
+      handshake with one another when these ciphersuites were enabled.
+      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
+
+  o Minor features (bandwidth authority):
+    - Make bandwidth authorities ignore relays that are reported in the
+      bandwidth file with the flag "vote=0". This change allows us to
+      report unmeasured relays for diagnostic reasons without including
+      their bandwidth in the bandwidth authorities' vote. Closes
+      ticket 29806.
+    - When a directory authority is using a bandwidth file to obtain the
+      bandwidth values that will be included in the next vote, serve
+      this bandwidth file at /tor/status-vote/next/bandwidth. Closes
+      ticket 21377.
+
+  o Minor features (circuit padding):
+    - Stop warning about undefined behavior in the probability
+      distribution tests. Float division by zero may technically be
+      undefined behavior in C, but it's well defined in IEEE 754.
+      Partial backport of 29298. Closes ticket 29527; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor features (continuous integration):
+    - On Travis Rust builds, cleanup Rust registry and refrain from
+      caching the "target/" directory to speed up builds. Resolves
+      issue 29962.
+
+  o Minor features (dormant mode):
+    - Add a DormantCanceledByStartup option to tell Tor that it should
+      treat a startup event as cancelling any previous dormant state.
+      Integrators should use this option with caution: it should only be
+      used if Tor is being started because of something that the user
+      did, and not if Tor is being automatically started in the
+      background. Closes ticket 29357.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29992.
+
+  o Minor features (NSS, diagnostic):
+    - Try to log an error from NSS (if there is any) and a more useful
+      description of our situation if we are using NSS and a call to
+      SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241.
+
+  o Minor bugfixes (security):
+    - Fix a potential double free bug when reading huge bandwidth files.
+      The issue is not exploitable in the current Tor network because
+      the vulnerable code is only reached when directory authorities
+      read bandwidth files, but bandwidth files come from a trusted
+      source (usually the authorities themselves). Furthermore, the
+      issue is only exploitable in rare (non-POSIX) 32-bit architectures,
+      which are not used by any of the current authorities. Fixes bug
+      30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by
+      Tobias Stoeckmann.
+    - Verify in more places that we are not about to create a buffer
+      with more than INT_MAX bytes, to avoid possible OOB access in the
+      event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and
+      fixed by Tobias Stoeckmann.
+
+  o Minor bugfix (continuous integration):
+    - Reset coverage state on disk after Travis CI has finished. This
+      should prevent future coverage merge errors from causing the test
+      suite for the "process" subsystem to fail. The process subsystem
+      was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix
+      on 0.2.9.15.
+    - Terminate test-stem if it takes more than 9.5 minutes to run.
+      (Travis terminates the job after 10 minutes of no output.)
+      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (bootstrap reporting):
+    - During bootstrap reporting, correctly distinguish pluggable
+      transports from plain proxies. Fixes bug 28925; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug
+      29824; bugfix on 0.3.1.1-alpha. This is Coverity warning
+      CID 1444119.
+
+  o Minor bugfixes (circuitpadding testing):
+    - Minor tweaks to avoid rare test failures related to timers and
+      monotonic time. Fixes bug 29500; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Actually include the bandwidth-file-digest line in directory
+      authority votes. Fixes bug 29959; bugfix on 0.4.0.2-alpha.
+
+  o Minor bugfixes (logging):
+    - On Windows, when errors cause us to reload a consensus from disk,
+      tell the user that we are retrying at log level "notice".
+      Previously we only logged this information at "info", which was
+      confusing because the errors themselves were logged at "warning".
+      Improves previous fix for 28614. Fixes bug 30004; bugfix
+      on 0.4.0.2-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Restore old behavior when it comes to discovering the path of a
+      given Pluggable Transport executable file. A change in
+      0.4.0.1-alpha had broken this behavior on paths containing a
+      space. Fixes bug 29874; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a
+      recent test-network.sh to use new chutney features in CI. Fixes
+      bug 29703; bugfix on 0.2.9.1-alpha.
+    - Fix a test failure on Windows caused by an unexpected "BUG"
+      warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (TLS protocol):
+    - When classifying a client's selection of TLS ciphers, if the
+      client ciphers are not yet available, do not cache the result.
+      Previously, we had cached the unavailability of the cipher list
+      and never looked again, which in turn led us to assume that the
+      client only supported the ancient V1 link protocol. This, in turn,
+      was causing Stem integration tests to stall in some cases. Fixes
+      bug 30021; bugfix on 0.2.4.8-alpha.
+
+  o Code simplification and refactoring:
+    - Introduce a connection_dir_buf_add() helper function that detects
+      whether compression is in use, and adds a string accordingly.
+      Resolves issue 28816.
+    - Refactor handle_get_next_bandwidth() to use
+      connection_dir_buf_add(). Implements ticket 29897.
+
+  o Documentation:
+    - Clarify that Tor performs stream isolation among *Port listeners
+      by default. Resolves issue 29121.
+
+
+Changes in version 0.4.0.3-alpha - 2019-03-22
+  Tor 0.4.0.3-alpha is the third in its series; it fixes several small
+  bugs from earlier versions.
+
+  o Minor features (address selection):
+    - Treat the subnet 100.64.0.0/10 as public for some purposes;
+      private for others. This subnet is the RFC 6598 (Carrier Grade
+      NAT) IP range, and is deployed by many ISPs as an alternative to
+      RFC 1918 that does not break existing internal networks. Tor now
+      blocks SOCKS and control ports on these addresses and warns users
+      if client ports or ExtORPorts are listening on a RFC 6598 address.
+      Closes ticket 28525. Patch by Neel Chauhan.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the March 4 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29666.
+
+  o Minor bugfixes (circuitpadding):
+    - Inspect the circuit-level cell queue before sending padding, to
+      avoid sending padding when too much data is queued. Fixes bug
+      29204; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Correct a misleading error message when IPv4Only or IPv6Only is
+      used but the resolved address can not be interpreted as an address
+      of the specified IP version. Fixes bug 13221; bugfix on
+      0.2.3.9-alpha. Patch from Kris Katterjohn.
+    - Log the correct port number for listening sockets when "auto" is
+      used to let Tor pick the port number. Previously, port 0 was
+      logged instead of the actual port number. Fixes bug 29144; bugfix
+      on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+    - Stop logging a BUG() warning when Tor is waiting for exit
+      descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (memory management):
+    - Refactor the shared random state's memory management so that it
+      actually takes ownership of the shared random value pointers.
+      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (memory management, testing):
+    - Stop leaking parts of the shared random state in the shared-random
+      unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Fix an assertion failure crash bug when a pluggable transport is
+      terminated during the bootstrap phase. Fixes bug 29562; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (Rust, protover):
+    - Add a missing "Padding" value to the Rust implementation of
+      protover. Fixes bug 29631; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (single onion services):
+    - Allow connections to single onion services to remain idle without
+      being disconnected. Previously, relays acting as rendezvous points
+      for single onion services were mistakenly closing idle rendezvous
+      circuits after 60 seconds, thinking that they were unused
+      directory-fetching circuits that had served their purpose. Fixes
+      bug 29665; bugfix on 0.2.1.26.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Downgrade some LOG_ERR messages in the address/* tests to
+      warnings. The LOG_ERR messages were occurring when we had no
+      configured network. We were failing the unit tests, because we
+      backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug
+      29530; bugfix on 0.3.5.8.
+    - Fix our gcov wrapper script to look for object files at the
+      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
+    - Decrease the false positive rate of stochastic probability
+      distribution tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (Windows, CI):
+    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit
+      Windows Server 2012 R2 job. The remaining 2 jobs still provide
+      coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set
+      fast_finish, so failed jobs terminate the build immediately. Fixes
+      bug 29601; bugfix on 0.3.5.4-alpha.
+
+
+Changes in version 0.3.5.8 - 2019-02-21
+  Tor 0.3.5.8 backports several fixes from later releases, including fixes
+  for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x
+  releases.
+
+  It also includes a fix for a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking, backport from 0.4.0.2-alpha):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth message and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Minor features (compilation, backport from 0.4.0.2-alpha):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor features (testing, backport from 0.4.0.2-alpha):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+
+  o Minor bugfixes (onion service v3, client, backport from 0.4.0.1-alpha):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (IPv6, backport from 0.4.0.1-alpha):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (client, clock skew, backport from 0.4.0.1-alpha):
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.0.1-alpha):
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (documentation, backport from 0.4.0.2-alpha):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (logging, backport from 0.4.0.1-alpha):
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (logging, backport from 0.4.0.2-alpha):
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (misc, backport from 0.4.0.2-alpha):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (tests, directory clients, backport from 0.4.0.1-alpha):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (tests, backport from 0.4.0.2-alpha):
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (usability, backport from 0.4.0.1-alpha):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.11 - 2019-02-21
+  Tor 0.3.4.11 is the third stable release in its series.  It includes
+  a fix for a medium-severity security bug affecting Tor 0.3.2.1-alpha and
+  later. All Tor instances running an affected release should upgrade to
+  0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.3.3.12 - 2019-02-21
+  Tor 0.3.3.12 fixes a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  This release marks the end of support for the Tor 0.3.3.x series. We
+  recommend that users switch to either the Tor 0.3.4 series (supported
+  until at least 10 June 2019), or the Tor 0.3.5 series, which will
+  receive long-term support until at least 1 Feb 2022.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.4.0.2-alpha - 2019-02-21
+  Tor 0.4.0.2-alpha is the second alpha in its series; it fixes several
+  bugs from earlier versions, including several that had broken
+  backward compatibility.
+
+  It also includes a fix for a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth messsage and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (windows, startup):
+    - When reading a consensus file from disk, detect whether it was
+      written in text mode, and re-read it in text mode if so. Always
+      write consensus files in binary mode so that we can map them into
+      memory later. Previously, we had written in text mode, which
+      confused us when we tried to map the file on windows. Fixes bug
+      28614; bugfix on 0.4.0.1-alpha.
+
+  o Minor features (compilation):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (developer tooling):
+    - Check that bugfix versions in changes files look like Tor versions
+      from the versions spec. Warn when bugfixes claim to be on a future
+      release. Closes ticket 27761.
+    - Provide a git pre-commit hook that disallows committing if we have
+      any failures in our code and changelog formatting checks. It is
+      now available in scripts/maint/pre-commit.git-hook. Implements
+      feature 28976.
+
+  o Minor features (directory authority):
+    - When a directory authority is using a bandwidth file to obtain
+      bandwidth values, include the digest of that file in the vote.
+      Closes ticket 26698.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor features (testing):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+
+  o Minor bugfixes (build, compatibility, rust):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix compilation warnings in test_circuitpadding.c. Fixes bug
+      29169; bugfix on 0.4.0.1-alpha.
+    - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug
+      29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (documentation):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (linux seccomp sandbox):
+    - Fix startup crash when experimental sandbox support is enabled.
+      Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber.
+
+  o Minor bugfixes (logging):
+    - Avoid logging that we are relaxing a circuit timeout when that
+      timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha.
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (misc):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (onion services):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (scheduler):
+    - When re-adding channels to the pending list, check the correct
+      channel's sched_heap_idx. This issue has had no effect in mainline
+      Tor, but could have led to bugs down the road in improved versions
+      of our circuit scheduling code. Fixes bug 29508; bugfix
+      on 0.3.2.10.
+
+  o Minor bugfixes (tests):
+    - Fix intermittent failures on an adaptive padding test. Fixes one
+      case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Disable an unstable circuit-padding test that was failing
+      intermittently because of an ill-defined small histogram. Such
+      histograms will be allowed again after 29298 is implemented. Fixes
+      a second case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+
+  o Documentation:
+    - In the manpage entry describing MapAddress torrc setting, use
+      example IP addresses from ranges specified for use in documentation
+      by RFC 5737. Resolves issue 28623.
+
+  o Removed features:
+    - Remove the old check-tor script. Resolves issue 29072.
+
+
+Changes in version 0.4.0.1-alpha - 2019-01-18
+  Tor 0.4.0.1-alpha is the first release in the new 0.4.0.x series. It
+  introduces improved features for power and bandwidth conservation,
+  more accurate reporting of bootstrap progress for user interfaces, and
+  an experimental backend for an exciting new adaptive padding feature.
+  There is also the usual assortment of bugfixes and minor features, all
+  described below.
+
+  o Major features (battery management, client, dormant mode):
+    - When Tor is running as a client, and it is unused for a long time,
+      it can now enter a "dormant" state. When Tor is dormant, it avoids
+      network and CPU activity until it is reawoken either by a user
+      request or by a controller command. For more information, see the
+      configuration options starting with "Dormant". Implements tickets
+      2149 and 28335.
+    - The client's memory of whether it is "dormant", and how long it
+      has spent idle, persists across invocations. Implements
+      ticket 28624.
+    - There is a DormantOnFirstStartup option that integrators can use
+      if they expect that in many cases, Tor will be installed but
+      not used.
+
+  o Major features (bootstrap reporting):
+    - When reporting bootstrap progress, report the first connection
+      uniformly, regardless of whether it's a connection for building
+      application circuits. This allows finer-grained reporting of early
+      progress than previously possible, with the improvements of ticket
+      27169. Closes tickets 27167 and 27103. Addresses ticket 27308.
+    - When reporting bootstrap progress, treat connecting to a proxy or
+      pluggable transport as separate from having successfully used that
+      proxy or pluggable transport to connect to a relay. Closes tickets
+      27100 and 28884.
+
+  o Major features (circuit padding):
+    - Implement preliminary support for the circuit padding portion of
+      Proposal 254. The implementation supports Adaptive Padding (aka
+      WTF-PAD) state machines for use between experimental clients and
+      relays. Support is also provided for APE-style state machines that
+      use probability distributions instead of histograms to specify
+      inter-packet delay. At the moment, Tor does not provide any
+      padding state machines that are used in normal operation: for now,
+      this feature exists solely for experimentation. Closes
+      ticket 28142.
+
+  o Major features (refactoring):
+    - Tor now uses an explicit list of its own subsystems when
+      initializing and shutting down. Previously, these systems were
+      managed implicitly in various places throughout the codebase.
+      (There may still be some subsystems using the old system.) Closes
+      ticket 28330.
+
+  o Minor features (bootstrap reporting):
+    - When reporting bootstrap progress, stop distinguishing between
+      situations where only internal paths are available and situations
+      where external paths are available. Previously, Tor would often
+      erroneously report that it had only internal paths. Closes
+      ticket 27402.
+
+  o Minor features (continuous integration):
+    - Log Python version during each Travis CI job. Resolves
+      issue 28551.
+
+  o Minor features (controller):
+    - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP.
+      Implements ticket 28843.
+
+  o Minor features (developer tooling):
+    - Provide a git hook script to prevent "fixup!" and "squash!"
+      commits from ending up in the master branch, as scripts/main/pre-
+      push.git-hook. Closes ticket 27993.
+
+  o Minor features (directory authority):
+    - Directory authorities support a new consensus algorithm, under
+      which the family lines in microdescriptors are encoded in a
+      canonical form. This change makes family lines more compressible
+      in transit, and on the client. Closes ticket 28266; implements
+      proposal 298.
+
+  o Minor features (directory authority, relay):
+    - Authorities now vote on a "StaleDesc" flag to indicate that a
+      relay's descriptor is so old that the relay should upload again
+      soon. Relays treat this flag as a signal to upload a new
+      descriptor. This flag will eventually let us remove the
+      'published' date from routerstatus entries, and make our consensus
+      diffs much smaller. Closes ticket 26770; implements proposal 293.
+
+  o Minor features (fallback directory mirrors):
+    - Update the fallback whitelist based on operator opt-ins and opt-
+      outs. Closes ticket 24805, patch by Phoul.
+
+  o Minor features (FreeBSD):
+    - On FreeBSD-based systems, warn relay operators if the
+      "net.inet.ip.random_id" sysctl (IP ID randomization) is disabled.
+      Closes ticket 28518.
+
+  o Minor features (HTTP standards compliance):
+    - Stop sending the header "Content-type: application/octet-stream"
+      along with transparently compressed documents: this confused
+      browsers. Closes ticket 28100.
+
+  o Minor features (IPv6):
+    - We add an option ClientAutoIPv6ORPort, to make clients randomly
+      prefer a node's IPv4 or IPv6 ORPort. The random preference is set
+      every time a node is loaded from a new consensus or bridge config.
+      We expect that this option will enable clients to bootstrap more
+      quickly without having to determine whether they support IPv4,
+      IPv6, or both. Closes ticket 27490. Patch by Neel Chauhan.
+    - When using addrs_in_same_network_family(), avoid choosing circuit
+      paths that pass through the same IPv6 subnet more than once.
+      Previously, we only checked IPv4 subnets. Closes ticket 24393.
+      Patch by Neel Chauhan.
+
+  o Minor features (log messages):
+    - Improve log message in v3 onion services that could print out
+      negative revision counters. Closes ticket 27707. Patch
+      by "ffmancera".
+
+  o Minor features (memory usage):
+    - Save memory by storing microdescriptor family lists with a more
+      compact representation. Closes ticket 27359.
+    - Tor clients now use mmap() to read consensus files from disk, so
+      that they no longer need keep the full text of a consensus in
+      memory when parsing it or applying a diff. Closes ticket 27244.
+
+  o Minor features (parsing):
+    - Directory authorities now validate that router descriptors and
+      ExtraInfo documents are in a valid subset of UTF-8, and reject
+      them if they are not. Closes ticket 27367.
+
+  o Minor features (performance):
+    - Cache the results of summarize_protocol_flags(), so that we don't
+      have to parse the same protocol-versions string over and over.
+      This should save us a huge number of malloc calls on startup, and
+      may reduce memory fragmentation with some allocators. Closes
+      ticket 27225.
+    - Remove a needless memset() call from get_token_arguments, thereby
+      speeding up the tokenization of directory objects by about 20%.
+      Closes ticket 28852.
+    - Replace parse_short_policy() with a faster implementation, to
+      improve microdescriptor parsing time. Closes ticket 28853.
+    - Speed up directory parsing a little by avoiding use of the non-
+      inlined strcmp_len() function. Closes ticket 28856.
+    - Speed up microdescriptor parsing by about 30%, to help improve
+      startup time. Closes ticket 28839.
+
+  o Minor features (pluggable transports):
+    - Add support for emitting STATUS updates to Tor's control port from
+      a pluggable transport process. Closes ticket 28846.
+    - Add support for logging to Tor's logging subsystem from a
+      pluggable transport process. Closes ticket 28180.
+
+  o Minor features (process management):
+    - Add a new process API for handling child processes. This new API
+      allows Tor to have bi-directional communication with child
+      processes on both Unix and Windows. Closes ticket 28179.
+    - Use the subsystem manager to initialize and shut down the process
+      module. Closes ticket 28847.
+
+  o Minor features (relay):
+    - When listing relay families, list them in canonical form including
+      the relay's own identity, and try to give a more useful set of
+      warnings. Part of ticket 28266 and proposal 298.
+
+  o Minor features (required protocols):
+    - Before exiting because of a missing required protocol, Tor will
+      now check the publication time of the consensus, and not exit
+      unless the consensus is newer than the Tor program's own release
+      date. Previously, Tor would not check the consensus publication
+      time, and so might exit because of a missing protocol that might
+      no longer be required in a current consensus. Implements proposal
+      297; closes ticket 27735.
+
+  o Minor features (testing):
+    - Allow a HeartbeatPeriod of less than 30 minutes in testing Tor
+      networks. Closes ticket 28840. Patch by Rob Jansen.
+
+  o Minor bugfixes (client, clock skew):
+    - Bootstrap successfully even when Tor's clock is behind the clocks
+      on the authorities. Fixes bug 28591; bugfix on 0.2.0.9-alpha.
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (directory clients):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Even when a directory mirror's clock is behind the clocks on the
+      authorities, we now allow the mirror to serve "future"
+      consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (DNS):
+    - Gracefully handle an empty or absent resolve.conf file by falling
+      back to using "localhost" as a DNS server (and hoping it works).
+      Previously, we would just stop running as an exit. Fixes bug
+      21900; bugfix on 0.2.1.10-alpha.
+
+  o Minor bugfixes (guards):
+    - In count_acceptable_nodes(), the minimum number is now one bridge
+      or guard node, and two non-guard nodes for a circuit. Previously,
+      we had added up the sum of all nodes with a descriptor, but that
+      could cause us to build failing circuits when we had either too
+      many bridges or not enough guard nodes. Fixes bug 25885; bugfix on
+      0.2.3.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (IPv6):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (logging):
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (networking):
+    - Introduce additional checks into tor_addr_parse() to reject
+      certain incorrect inputs that previously were not detected. Fixes
+      bug 23082; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (onion service v3, client):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (periodic events):
+    - Refrain from calling routerlist_remove_old_routers() from
+      check_descriptor_callback(). Instead, create a new hourly periodic
+      event. Fixes bug 27929; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Make sure that data is continously read from standard output and
+      standard error pipes of a pluggable transport child-process, to
+      avoid deadlocking when a pipe's buffer is full. Fixes bug 26360;
+      bugfix on 0.2.3.6-alpha.
+
+  o Minor bugfixes (unit tests):
+    - Instead of relying on hs_free_all() to clean up all onion service
+      objects in test_build_descriptors(), we now deallocate them one by
+      one. This lets Coverity know that we are not leaking memory there
+      and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (usability):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+  o Code simplification and refactoring:
+    - Reimplement NETINFO cell parsing and generation to rely on
+      trunnel-generated wire format handling code. Closes ticket 27325.
+    - Remove unnecessary unsafe code from the Rust macro "cstr!". Closes
+      ticket 28077.
+    - Rework SOCKS wire format handling to rely on trunnel-generated
+      parsing/generation code. Resolves ticket 27620.
+    - Split out bootstrap progress reporting from control.c into a
+      separate file. Part of ticket 27402.
+    - The .may_include files that we use to describe our directory-by-
+      directory dependency structure now describe a noncircular
+      dependency graph over the directories that they cover. Our
+      checkIncludes.py tool now enforces this noncircularity. Closes
+      ticket 28362.
+
+  o Documentation:
+    - Mention that you cannot add a new onion service if Tor is already
+      running with Sandbox enabled. Closes ticket 28560.
+    - Improve ControlPort documentation. Mention that it accepts
+      address:port pairs, and can be used multiple times. Closes
+      ticket 28805.
+    - Document the exact output of "tor --version". Closes ticket 28889.
+
+  o Removed features:
+    - Stop responding to the 'GETINFO status/version/num-concurring' and
+      'GETINFO status/version/num-versioning' control port commands, as
+      those were deprecated back in 0.2.0.30. Also stop listing them in
+      output of 'GETINFO info/names'. Resolves ticket 28757.
+    - The scripts used to generate and maintain the list of fallback
+      directories have been extracted into a new "fallback-scripts"
+      repository. Closes ticket 27914.
+
+  o Testing:
+    - Run shellcheck for scripts in the in scripts/ directory. Closes
+      ticket 28058.
+    - Add unit tests for tokenize_string() and get_next_token()
+      functions. Resolves ticket 27625.
+
+  o Code simplification and refactoring (onion service v3):
+    - Consolidate the authorized client descriptor cookie computation
+      code from client and service into one function. Closes
+      ticket 27549.
+
+  o Code simplification and refactoring (shell scripts):
+    - Cleanup scan-build.sh to silence shellcheck warnings. Closes
+      ticket 28007.
+    - Fix issues that shellcheck found in chutney-git-bisect.sh.
+      Resolves ticket 28006.
+    - Fix issues that shellcheck found in updateRustDependencies.sh.
+      Resolves ticket 28012.
+    - Fix shellcheck warnings in cov-diff script. Resolves issue 28009.
+    - Fix shellcheck warnings in run_calltool.sh. Resolves ticket 28011.
+    - Fix shellcheck warnings in run_trunnel.sh. Resolves issue 28010.
+    - Fix shellcheck warnings in scripts/test/coverage. Resolves
+      issue 28008.
+
+
+Changes in version 0.3.3.11 - 2019-01-07
+  Tor 0.3.3.11 backports numerous fixes from later versions of Tor.
+  numerous fixes, including an important fix for anyone using OpenSSL
+  1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade
+  to this version, or to a later series.
+
+  As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019.
+  We anticipate that this will be the last release of Tor 0.3.3, unless
+  some major bug is before then. Some time between now and then, users
+  should switch to either the Tor 0.3.4 series (supported until at least
+  10 June 2019), or the Tor 0.3.5 series, which will receive long-term
+  support until at least 1 Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.3.5.1-alpha):
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, backport from 0.3.5.3-alpha):
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha):
+    - Avoid undefined behavior in an end-of-string check when parsing
+      the BEGIN line in a directory object. Fixes bug 28202; bugfix
+      on 0.2.0.3-alpha.
+
+  o Minor bugfixes (code safety, backport from 0.3.5.3-alpha):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha):
+    - Log additional info when we get a relay that shares an ed25519 ID
+      with a different relay, instead making a BUG() warning. Fixes bug
+      27800; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha):
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to
+      an HTTP request we couldn't handle. Fixes bug 26470; bugfix on
+      0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (protover, backport from 0.3.5.3-alpha):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.1-alpha):
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.2-alpha):
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (rust, backport from 0.3.5.4-alpha):
+    - Fix a potential null dereference in protover_all_supported(). Add
+      a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
+    - Return a string that can be safely freed by C code, not one
+      created by the rust allocator, in protover_all_supported(). Fixes
+      bug 27740; bugfix on 0.3.3.1-alpha.
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
+
+  o Minor bugfixes (testing, backport from 0.3.5.1-alpha):
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.4-alpha):
+    - Treat backtrace test failures as expected on BSD-derived systems
+      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
+      (FreeBSD failures have been treated as expected since 18204 in
+      0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.10 - 2019-01-07
+  Tor 0.3.4.9 is the second stable release in its series; it backports
+  numerous fixes, including an important fix for relays, and for anyone
+  using OpenSSL 1.1.1. Anyone running an  earlier version of Tor 0.3.4
+  should upgrade.
+
+  As a reminder, the Tor 0.3.4 series will be supported until 10 June
+  2019. Some time between now and then, users should switch to the Tor
+  0.3.5 series, which will receive long-term support until at least 1
+  Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (relay, directory, backport from 0.3.5.7):
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+
+  o Minor features (controller, backport from 0.3.5.1-alpha):
+    - For purposes of CIRC_BW-based dropped cell detection, track half-
+      closed stream ids, and allow their ENDs, SENDMEs, DATA and path
+      bias check cells to arrive without counting it as dropped until
+      either the END arrives, or the windows are empty. Closes
+      ticket 25573.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28395.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.5.7 - 2019-01-07
+  Tor 0.3.5.7 is the first stable release in its series; it includes
+  compilation and portability fixes, and a fix for a severe problem
+  affecting directory caches.
+
+  The Tor 0.3.5 series includes several new features and performance
+  improvements, including client authorization for v3 onion services,
+  cleanups to bootstrap reporting, support for improved bandwidth-
+  measurement tools, experimental support for NSS in place of OpenSSL,
+  and much more. It also begins a full reorganization of Tor's code
+  layout, for improved modularity and maintainability in the future.
+  Finally, there is the usual set of performance improvements and
+  bugfixes that we try to do in every release series.
+
+  There are a couple of changes in the 0.3.5 that may affect
+  compatibility. First, the default version for newly created onion
+  services is now v3. Use the HiddenServiceVersion option if you want to
+  override this. Second, some log messages related to bootstrapping have
+  changed; if you use stem, you may need to update to the latest version
+  so it will recognize them.
+
+  We have designated 0.3.5 as a "long-term support" (LTS) series: we
+  will continue to patch major bugs in typical configurations of 0.3.5
+  until at least 1 Feb 2022. (We do not plan to provide long-term
+  support for embedding, Rust support, NSS support, running a directory
+  authority, or unsupported platforms. For these, you will need to stick
+  with the latest stable release.)
+
+  Below are the changes since 0.3.5.6-rc. For a complete list of changes
+  since 0.3.4.9, see the ReleaseNotes file.
+
+  o Major bugfixes (relay, directory):
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Minor features (compilation):
+    - When possible, place our warning flags in a separate file, to
+      avoid flooding verbose build logs. Closes ticket 28924.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (OpenSSL bug workaround):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor features (performance):
+    - Remove about 96% of the work from the function that we run at
+      startup to test our curve25519_basepoint implementation. Since
+      this function has yet to find an actual failure, we now only run
+      it for 8 iterations instead of 200. Based on our profile
+      information, this change should save around 8% of our startup time
+      on typical desktops, and may have a similar effect on other
+      platforms. Closes ticket 28838.
+    - Stop re-validating our hardcoded Diffie-Hellman parameters on
+      every startup. Doing this wasted time and cycles, especially on
+      low-powered devices. Closes ticket 28851.
+
+  o Minor bugfixes (compilation):
+    - Fix compilation for Android by adding a missing header to
+      freespace.c. Fixes bug 28974; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (correctness):
+    - Fix an unreached code path where we checked the value of
+      "hostname" inside send_resolved_hostname_cell(). Previously, we
+      used it before checking it; now we check it first. Fixes bug
+      28879; bugfix on 0.1.2.7-alpha.
+
+  o Minor bugfixes (testing):
+    - Make sure that test_rebind.py actually obeys its timeout, even
+      when it receives a large number of log messages. Fixes bug 28883;
+      bugfix on 0.3.5.4-alpha.
+    - Stop running stem's unit tests as part of "make test-stem", but
+      continue to run stem's unit and online tests during "make test-
+      stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (windows services):
+    - Make Tor start correctly as an NT service again: previously it was
+      broken by refactoring. Fixes bug 28612; bugfix on 0.3.5.3-alpha.
+
+  o Code simplification and refactoring:
+    - When parsing a port configuration, make it more obvious to static
+      analyzer tools that we always initialize the address. Closes
+      ticket 28881.
+
+
+Changes in version 0.3.5.6-rc - 2018-12-18
+  Tor 0.3.5.6-rc fixes numerous small bugs in earlier versions of Tor.
+  It is the first release candidate in the 0.3.5.x series; if no further
+  huge bugs are found, our next release may be the stable 0.3.5.x.
+
+  o Minor features (continuous integration, Windows):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+
+  o Minor features (fallback directory list):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28744.
+
+  o Minor bugfixes (compilation):
+    - Add missing dependency on libgdi32.dll for tor-print-ed-signing-
+      cert.exe on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows):
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+
+  o Minor bugfixes (onion service v3):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (restart-in-process, boostrap):
+    - Add missing resets of bootstrap tracking state when shutting down
+      (regression caused by ticket 27169). Fixes bug 28524; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Use a separate DataDirectory for the test_rebind script.
+      Previously, this script would run using the default DataDirectory,
+      and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha.
+      Patch from Taylor R Campbell.
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (Windows):
+    - Correctly identify Windows 8.1, Windows 10, and Windows Server
+      2008 and later from their NT versions. Fixes bug 28096; bugfix on
+      0.2.2.34; reported by Keifer Bly.
+    - On recent Windows versions, the GetVersionEx() function may report
+      an earlier Windows version than the running OS. To avoid user
+      confusion, add "[or later]" to Tor's version string on affected
+      versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported
+      by Keifer Bly.
+    - Remove Windows versions that were never supported by the
+      GetVersionEx() function. Stop duplicating the latest Windows
+      version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34;
+      reported by Keifer Bly.
+
+  o Testing:
+    - Increase logging and tag all log entries with timestamps in
+      test_rebind.py. Provides diagnostics for issue 28229.
+
+  o Code simplification and refactoring (shared random, dirauth):
+    - Change many tor_assert() to use BUG() instead. The idea is to not
+      crash a dirauth but rather scream loudly with a stacktrace and let
+      it continue run. The shared random subsystem is very resilient and
+      if anything wrong happens with it, at worst a non coherent value
+      will be put in the vote and discarded by the other authorities.
+      Closes ticket 19566.
+
+  o Documentation (onion services):
+    - Document in the man page that changing ClientOnionAuthDir value or
+      adding a new file in the directory will not work at runtime upon
+      sending a HUP if Sandbox 1. Closes ticket 28128.
+    - Note in the man page that the only real way to fully revoke an
+      onion service v3 client authorization is by restarting the tor
+      process. Closes ticket 28275.
+
+
+Changes in version 0.3.5.5-alpha - 2018-11-16
+  Tor 0.3.5.5-alpha includes numerous bugfixes on earlier releases,
+  including several that we hope to backport to older release series in
+  the future.
+
+  o Major bugfixes (OpenSSL, portability):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28395.
+
+  o Minor bugfixes (compilation):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection, relay):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Make Doxygen work again after the code movement in the 0.3.5
+      source tree. Fixes bug 28435; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Permit the "shutdown()" system call, which is apparently used by
+      OpenSSL under some circumstances. Fixes bug 28183; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Stop talking about the Named flag in log messages. Clients have
+      ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (onion services):
+    - On an intro point for a version 3 onion service, stop closing
+      introduction circuits on an NACK. This lets the client decide
+      whether to reuse the circuit or discard it. Previously, we closed
+      intro circuits when sending NACKs. Fixes bug 27841; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chaunan.
+    - When replacing a descriptor in the client cache, make sure to
+      close all client introduction circuits for the old descriptor, so
+      we don't end up with unusable leftover circuits. Fixes bug 27471;
+      bugfix on 0.3.2.1-alpha.
+
+
 Changes in version 0.3.5.4-alpha - 2018-11-08
   Tor 0.3.5.4-alpha includes numerous bugfixes on earlier versions and
   improves our continuous integration support. It continues our attempts
@@ -2580,7 +7625,7 @@ Changes in version 0.3.4.1-alpha - 2018-05-17
     - Remove duplicate code in parse_{c,s}method_line and bootstrap
       their functionalities into a single function. Fixes bug 6236;
       bugfix on 0.2.3.6-alpha.
-    - We remove the PortForwsrding and PortForwardingHelper options,
+    - We remove the PortForwarding and PortForwardingHelper options,
       related functions, and the port_forwarding tests. These options
       were used by the now-deprecated Vidalia to help ordinary users
       become Tor relays or bridges. Closes ticket 25409. Patch by
@@ -9182,7 +14227,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
       HiddenServiceNonAnonymousMode options. When both are set to 1,
       every hidden service on a Tor instance becomes a non-anonymous
       Single Onion Service. Single Onions make one-hop (direct)
-      connections to their introduction and renzedvous points. One-hop
+      connections to their introduction and rendezvous points. One-hop
       circuits make Single Onion servers easily locatable, but clients
       remain location-anonymous. This is compatible with the existing
       hidden service implementation, and works on the current tor
@@ -9685,13 +14730,12 @@ Changes in version 0.2.9.1-alpha - 2016-08-08
       needless crash bugs. Closes ticket 18613.
 
   o Minor features (performance):
-    - Changer the "optimistic data" extension from "off by default" to
+    - Change the "optimistic data" extension from "off by default" to
       "on by default". The default was ordinarily overridden by a
       consensus option, but when clients were bootstrapping for the
       first time, they would not have a consensus to get the option
-      from. Changing this default When fetching a consensus for the
-      first time, use optimistic data. This saves a round-trip during
-      startup. Closes ticket 18815.
+      from. Changing this default saves a round-trip during startup.
+      Closes ticket 18815.
 
   o Minor features (relay, usability):
     - When the directory authorities refuse a bad relay's descriptor,
diff --git a/Doxyfile.in b/Doxyfile.in
index 4caf421097..503c1302db 100644
--- a/Doxyfile.in
+++ b/Doxyfile.in
@@ -1,96 +1,150 @@
-# Doxyfile 1.5.6
+# Doxyfile 1.8.15
+
+# (Tor's Doxyfile is automatically generated from "Doxyfile.in". Don't
+# edit Doxyfile; edit Doxyfile.in.)
 
 # This file describes the settings to be used by the documentation system
-# doxygen (www.doxygen.org) for a project
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
 #
-# All text after a hash (#) is considered a comment and will be ignored
+# All text after a single hash (#) is considered a comment and will be ignored.
 # The format is:
-#       TAG = value [value, ...]
-# For lists items can also be appended using:
-#       TAG += value [value, ...]
-# Values that contain spaces should be placed between quotes (" ")
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
 
 #---------------------------------------------------------------------------
 # Project related configuration options
 #---------------------------------------------------------------------------
 
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all
+# This tag specifies the encoding used for all characters in the configuration
+# file that follow. The default is UTF-8 which is also the encoding used for all
 # text before the first occurrence of this tag. Doxygen uses libiconv (or the
 # iconv built into libc) for the transcoding. See
-# http://www.gnu.org/software/libiconv for the list of possible encodings.
+# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
+# The default value is: UTF-8.
 
 DOXYFILE_ENCODING      = UTF-8
 
-# The PROJECT_NAME tag is a single word (or a sequence of words surrounded
-# by quotes) that should identify the project.
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
 
-PROJECT_NAME           = tor
+PROJECT_NAME           = Tor
 
-# The PROJECT_NUMBER tag can be used to enter a project or revision number.
-# This could be handy for archiving the generated documentation or
-# if some version control system is used.
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
 
 PROJECT_NUMBER         = @VERSION@
 
-# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
-# base path where the generated documentation will be put.
-# If a relative path is entered, it will be relative to the location
-# where doxygen was started. If left blank the current directory will be used.
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          =
+
+# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
+# in the documentation. The maximum height of the logo should not exceed 55
+# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
+# the logo to the output directory.
 
-OUTPUT_DIRECTORY       = @top_builddir@/doc/doxygen
+PROJECT_LOGO           =
 
-# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
-# 4096 sub-directories (in 2 levels) under the output directory of each output
-# format and will distribute the generated files over these directories.
-# Enabling this option can be useful when feeding doxygen a huge amount of
-# source files, where putting all generated files in the same directory would
-# otherwise cause performance problems for the file system.
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       = @abs_top_builddir@/doc/doxygen
+
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
 
 CREATE_SUBDIRS         = NO
 
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES    = NO
+
 # The OUTPUT_LANGUAGE tag is used to specify the language in which all
 # documentation generated by doxygen is written. Doxygen will use this
 # information to generate all constant output in the proper language.
-# The default language is English, other supported languages are:
-# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional,
-# Croatian, Czech, Danish, Dutch, Farsi, Finnish, French, German, Greek,
-# Hungarian, Italian, Japanese, Japanese-en (Japanese with English messages),
-# Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, Polish,
-# Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish,
-# and Ukrainian.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
 
 OUTPUT_LANGUAGE        = English
 
-# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will
-# include brief member descriptions after the members that are listed in
-# the file and class documentation (similar to JavaDoc).
-# Set to NO to disable this.
+# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all generated output in the proper direction.
+# Possible values are: None, LTR, RTL and Context.
+# The default value is: None.
+
+OUTPUT_TEXT_DIRECTION  = None
 
-BRIEF_MEMBER_DESC      = NO
+# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
 
-# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend
-# the brief description of a member or function before the detailed description.
-# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
 # brief descriptions will be completely suppressed.
+# The default value is: YES.
 
 REPEAT_BRIEF           = YES
 
-# This tag implements a quasi-intelligent brief description abbreviator
-# that is used to form the text in various listings. Each string
-# in this list, if found as the leading text of the brief description, will be
-# stripped from the text and the result after processing the whole list, is
-# used as the annotated text. Otherwise, the brief description is used as-is.
-# If left blank, the following values are used ("$name" is automatically
-# replaced with the name of the entity): "The $name class" "The $name widget"
-# "The $name file" "is" "provides" "specifies" "contains"
-# "represents" "a" "an" "the"
-
-ABBREVIATE_BRIEF       =
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF       = "The $name class" \
+                         "The $name widget" \
+                         "The $name file" \
+                         is \
+                         provides \
+                         specifies \
+                         contains \
+                         represents \
+                         a \
+                         an \
+                         the
 
 # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
-# Doxygen will generate a detailed section even if there is only a brief
+# doxygen will generate a detailed section even if there is only a brief
 # description.
+# The default value is: NO.
 
 ALWAYS_DETAILED_SEC    = NO
 
@@ -98,482 +152,735 @@ ALWAYS_DETAILED_SEC    = NO
 # inherited members of a class in the documentation of that class as if those
 # members were ordinary class members. Constructors, destructors and assignment
 # operators of the base classes will not be shown.
+# The default value is: NO.
 
 INLINE_INHERITED_MEMB  = NO
 
-# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full
-# path before files name in the file list and in the header files. If set
-# to NO the shortest path that makes the file name unique will be used.
+# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
 
-FULL_PATH_NAMES        = NO
+FULL_PATH_NAMES        = YES
 
-# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag
-# can be used to strip a user-defined part of the path. Stripping is
-# only done if one of the specified strings matches the left-hand part of
-# the path. The tag can be used to show relative paths in the file list.
-# If left blank the directory from which doxygen is run is used as the
-# path to strip.
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
 
-STRIP_FROM_PATH        =
+STRIP_FROM_PATH        = ./src
 
-# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of
-# the path mentioned in the documentation of a class, which tells
-# the reader which header file to include in order to use a class.
-# If left blank only the name of the header file containing the class
-# definition is used. Otherwise one should specify the include paths that
-# are normally passed to the compiler using the -I flag.
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
 
 STRIP_FROM_INC_PATH    =
 
-# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter
-# (but less readable) file names. This can be useful is your file systems
-# doesn't support long names like on DOS, Mac, or CD-ROM.
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
 
 SHORT_NAMES            = NO
 
-# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen
-# will interpret the first line (until the first dot) of a JavaDoc-style
-# comment as the brief description. If set to NO, the JavaDoc
-# comments will behave just like regular Qt-style comments
-# (thus requiring an explicit @brief command for a brief description.)
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
 
 JAVADOC_AUTOBRIEF      = NO
 
-# If the QT_AUTOBRIEF tag is set to YES then Doxygen will
-# interpret the first line (until the first dot) of a Qt-style
-# comment as the brief description. If set to NO, the comments
-# will behave just like regular Qt-style comments (thus requiring
-# an explicit \brief command for a brief description.)
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
 
 QT_AUTOBRIEF           = NO
 
-# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen
-# treat a multi-line C++ special comment block (i.e. a block of //! or ///
-# comments) as a brief description. This used to be the default behaviour.
-# The new default is to treat a multi-line C++ comment block as a detailed
-# description. Set this tag to YES if you prefer the old behaviour instead.
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
 
 MULTILINE_CPP_IS_BRIEF = NO
 
-# If the DETAILS_AT_TOP tag is set to YES then Doxygen
-# will output the detailed description near the top, like JavaDoc.
-# If set to NO, the detailed description appears after the member
-# documentation.
-
-# DETAILS_AT_TOP         = NO
-
-# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
-# member inherits the documentation from any documented member that it
-# re-implements.
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
 
 INHERIT_DOCS           = YES
 
-# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce
-# a new page for each member. If set to NO, the documentation of a member will
-# be part of the file/class/namespace that contains it.
+# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
+# page for each member. If set to NO, the documentation of a member will be part
+# of the file/class/namespace that contains it.
+# The default value is: NO.
 
 SEPARATE_MEMBER_PAGES  = NO
 
-# The TAB_SIZE tag can be used to set the number of spaces in a tab.
-# Doxygen uses this value to replace tabs by spaces in code fragments.
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
 
 TAB_SIZE               = 8
 
-# This tag can be used to specify a number of aliases that acts
-# as commands in the documentation. An alias has the form "name=value".
-# For example adding "sideeffect=\par Side Effects:\n" will allow you to
-# put the command \sideeffect (or @sideeffect) in the documentation, which
-# will result in a user-defined paragraph with heading "Side Effects:".
-# You can put \n's in the value part of an alias to insert newlines.
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines (in the resulting output). You can put ^^ in the value part of an
+# alias to insert a newline as if a physical newline was in the original file.
+# When you need a literal { or } or , in the value part of an alias you have to
+# escape them by means of a backslash (\), this can lead to conflicts with the
+# commands \{ and \} for these it is advised to use the version @{ and @} or use
+# a double escape (\\{ and \\})
 
 ALIASES                =
 
-# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C
-# sources only. Doxygen will then generate output that is more tailored for C.
-# For instance, some of the names that are used will be different. The list
-# of all members will be omitted, etc.
+ALIASES += refdir{1}="\ref src/\1 \"\1\""
+
+ALIASES += ticket{1}="[ticket \1](https://bugs.torproject.org/\1)"
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
 
 OPTIMIZE_OUTPUT_FOR_C  = YES
 
-# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java
-# sources only. Doxygen will then generate output that is more tailored for
-# Java. For instance, namespaces will be presented as packages, qualified
-# scopes will look different, etc.
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
 
 OPTIMIZE_OUTPUT_JAVA   = NO
 
 # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
-# sources only. Doxygen will then generate output that is more tailored for
-# Fortran.
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
 
 OPTIMIZE_FOR_FORTRAN   = NO
 
 # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
-# sources. Doxygen will then generate output that is tailored for
-# VHDL.
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
 
 OPTIMIZE_OUTPUT_VHDL   = NO
 
+# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
+# sources only. Doxygen will then generate output that is more tailored for that
+# language. For instance, namespaces will be presented as modules, types will be
+# separated into more groups, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_SLICE  = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
+# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
+# tries to guess whether the code is fixed or free formatted code, this is the
+# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat
+# .inc files as Fortran files (default is PHP), and .f files as C (default is
+# Fortran), use: inc=Fortran f=C.
+#
+# Note: For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING      = dox=md h=C c=C inc=C md=md
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See https://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up
+# to that level are automatically included in the table of contents, even if
+# they do not have an id attribute.
+# Note: This feature currently applies only to Markdown headings.
+# Minimum value: 0, maximum value: 99, default value: 0.
+# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
+
+TOC_INCLUDE_HEADINGS   = 0
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
 # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
-# to include (a tag file for) the STL sources as input, then you should
-# set this tag to YES in order to let doxygen match functions declarations and
-# definitions whose arguments contain STL classes (e.g. func(std::string); v.s.
-# func(std::string) {}). This also make the inheritance and collaboration
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
 # diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
 
 BUILTIN_STL_SUPPORT    = NO
 
 # If you use Microsoft's C++/CLI language, you should set this option to YES to
 # enable parsing support.
+# The default value is: NO.
 
 CPP_CLI_SUPPORT        = NO
 
-# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only.
-# Doxygen will parse them like normal C++ but will assume all classes use public
-# instead of private inheritance when no explicit protection keyword is present.
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
 
 SIP_SUPPORT            = NO
 
-# For Microsoft's IDL there are propget and propput attributes to indicate getter
-# and setter methods for a property. Setting this option to YES (the default)
-# will make doxygen to replace the get and set methods by a property in the
-# documentation. This will only work if the methods are indeed getting or
-# setting a simple type. If this is not the case, or you want to show the
-# methods anyway, you should set this option to NO.
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
 
 IDL_PROPERTY_SUPPORT   = NO
 
 # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
-# tag is set to YES, then doxygen will reuse the documentation of the first
+# tag is set to YES then doxygen will reuse the documentation of the first
 # member in the group (if any) for the other members of the group. By default
 # all members of a group must be documented explicitly.
+# The default value is: NO.
 
 DISTRIBUTE_GROUP_DOC   = NO
 
-# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
-# the same type (for instance a group of public functions) to be put as a
-# subgroup of that type (e.g. under the Public Functions section). Set it to
-# NO to prevent subgrouping. Alternatively, this can be done per class using
-# the \nosubgrouping command.
+# If one adds a struct or class to a group and this option is enabled, then also
+# any nested class or struct is added to the same group. By default this option
+# is disabled and one has to add nested compounds explicitly via \ingroup.
+# The default value is: NO.
+
+GROUP_NESTED_COMPOUNDS = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
 
 SUBGROUPING            = YES
 
-# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum
-# is documented as struct, union, or enum with the name of the typedef. So
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS  = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
 # typedef struct TypeS {} TypeT, will appear in the documentation as a struct
 # with name TypeT. When disabled the typedef will appear as a member of a file,
-# namespace, or class. And the struct will be named TypeS. This can typically
-# be useful for C code in case the coding convention dictates that all compound
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
 # types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT   = YES
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
 
-TYPEDEF_HIDES_STRUCT   = NO
+LOOKUP_CACHE_SIZE      = 0
 
 #---------------------------------------------------------------------------
 # Build related configuration options
 #---------------------------------------------------------------------------
 
-# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
-# documentation are documented, even if no documentation was available.
-# Private class members and static file members will be hidden unless
-# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
 
 EXTRACT_ALL            = NO
 
-# If the EXTRACT_PRIVATE tag is set to YES all private members of a class
-# will be included in the documentation.
+# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
 
 EXTRACT_PRIVATE        = NO
 
-# If the EXTRACT_STATIC tag is set to YES all static members of a file
-# will be included in the documentation.
+# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
 
 EXTRACT_STATIC         = YES
 
-# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs)
-# defined locally in source files will be included in the documentation.
-# If set to NO only classes defined in header files are included.
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO,
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
 
 EXTRACT_LOCAL_CLASSES  = YES
 
-# This flag is only useful for Objective-C code. When set to YES local
-# methods, which are defined in the implementation section but not in
-# the interface are included in the documentation.
-# If set to NO (the default) only methods in the interface are included.
+# This flag is only useful for Objective-C code. If set to YES, local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO, only methods in the interface are
+# included.
+# The default value is: NO.
 
 EXTRACT_LOCAL_METHODS  = NO
 
 # If this flag is set to YES, the members of anonymous namespaces will be
 # extracted and appear in the documentation as a namespace called
-# 'anonymous_namespace{file}', where file will be replaced with the base
-# name of the file that contains the anonymous namespace. By default
-# anonymous namespace are hidden.
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
 
 EXTRACT_ANON_NSPACES   = NO
 
-# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all
-# undocumented members of documented classes, files or namespaces.
-# If set to NO (the default) these members will be included in the
-# various overviews, but no documentation section is generated.
-# This option has no effect if EXTRACT_ALL is enabled.
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
 
 HIDE_UNDOC_MEMBERS     = NO
 
-# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all
-# undocumented classes that are normally visible in the class hierarchy.
-# If set to NO (the default) these classes will be included in the various
-# overviews. This option has no effect if EXTRACT_ALL is enabled.
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
 
 HIDE_UNDOC_CLASSES     = NO
 
-# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all
-# friend (class|struct|union) declarations.
-# If set to NO (the default) these declarations will be included in the
-# documentation.
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO, these declarations will be
+# included in the documentation.
+# The default value is: NO.
 
 HIDE_FRIEND_COMPOUNDS  = NO
 
-# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any
-# documentation blocks found inside the body of a function.
-# If set to NO (the default) these blocks will be appended to the
-# function's detailed documentation block.
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO, these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
 
 HIDE_IN_BODY_DOCS      = NO
 
-# The INTERNAL_DOCS tag determines if documentation
-# that is typed after a \internal command is included. If the tag is set
-# to NO (the default) then the documentation will be excluded.
-# Set it to YES to include the internal documentation.
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
 
 INTERNAL_DOCS          = NO
 
-# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate
-# file names in lower-case letters. If set to YES upper-case letters are also
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES, upper-case letters are also
 # allowed. This is useful if you have classes or files whose names only differ
 # in case and if your file system supports case sensitive file names. Windows
 # and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
 
 CASE_SENSE_NAMES       = YES
 
-# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen
-# will show members with their full class and namespace scopes in the
-# documentation. If set to YES the scope will be hidden.
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES, the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES       = YES
 
-HIDE_SCOPE_NAMES       = NO
+# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
+# append additional text to a page's title, such as Class Reference. If set to
+# YES the compound reference will be hidden.
+# The default value is: NO.
 
-# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen
-# will put a list of the files that are included by a file in the documentation
-# of that file.
+HIDE_COMPOUND_REFERENCE= NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
 
 SHOW_INCLUDE_FILES     = YES
 
-# If the INLINE_INFO tag is set to YES (the default) then a tag [inline]
-# is inserted in the documentation for inline members.
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC  = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES   = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
 
 INLINE_INFO            = YES
 
-# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen
-# will sort the (detailed) documentation of file and class members
-# alphabetically by member name. If set to NO the members will appear in
-# declaration order.
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order.
+# The default value is: YES.
 
 SORT_MEMBER_DOCS       = YES
 
-# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the
-# brief documentation of file, namespace and class members alphabetically
-# by member name. If set to NO (the default) the members will appear in
-# declaration order.
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
 
 SORT_BRIEF_DOCS        = NO
 
-# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the
-# hierarchy of group names into alphabetical order. If set to NO (the default)
-# the group names will appear in their defined order.
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
 
 SORT_GROUP_NAMES       = NO
 
-# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be
-# sorted by fully-qualified names, including namespaces. If set to
-# NO (the default), the class list will be sorted only by class name,
-# not including the namespace part.
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
 # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
-# Note: This option applies only to the class list, not to the
-# alphabetical list.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
 
 SORT_BY_SCOPE_NAME     = NO
 
-# The GENERATE_TODOLIST tag can be used to enable (YES) or
-# disable (NO) the todo list. This list is created by putting \todo
-# commands in the documentation.
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING  = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
+# list. This list is created by putting \todo commands in the documentation.
+# The default value is: YES.
 
 GENERATE_TODOLIST      = YES
 
-# The GENERATE_TESTLIST tag can be used to enable (YES) or
-# disable (NO) the test list. This list is created by putting \test
-# commands in the documentation.
+# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
+# list. This list is created by putting \test commands in the documentation.
+# The default value is: YES.
 
 GENERATE_TESTLIST      = YES
 
-# The GENERATE_BUGLIST tag can be used to enable (YES) or
-# disable (NO) the bug list. This list is created by putting \bug
-# commands in the documentation.
+# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
 
 GENERATE_BUGLIST       = YES
 
-# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or
-# disable (NO) the deprecated list. This list is created by putting
-# \deprecated commands in the documentation.
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
 
 GENERATE_DEPRECATEDLIST= YES
 
-# The ENABLED_SECTIONS tag can be used to enable conditional
-# documentation sections, marked by \if sectionname ... \endif.
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
 
 ENABLED_SECTIONS       =
 
-# The MAX_INITIALIZER_LINES tag determines the maximum number of lines
-# the initial value of a variable or define consists of for it to appear in
-# the documentation. If the initializer consists of more lines than specified
-# here it will be hidden. Use a value of 0 to hide initializers completely.
-# The appearance of the initializer of individual variables and defines in the
-# documentation can be controlled using \showinitializer or \hideinitializer
-# command in the documentation regardless of this setting.
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
 
 MAX_INITIALIZER_LINES  = 30
 
-# Set the SHOW_USED_FILES tag to NO to disable the list of files generated
-# at the bottom of the documentation of classes and structs. If set to YES the
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES, the
 # list will mention the files that were used to generate the documentation.
+# The default value is: YES.
 
 SHOW_USED_FILES        = YES
 
-# Set the SHOW_FILES tag to NO to disable the generation of the Files page.
-# This will remove the Files entry from the Quick Index and from the
-# Folder Tree View (if specified). The default is YES.
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
 
 SHOW_FILES             = YES
 
-# Set the SHOW_NAMESPACES tag to NO to disable the generation of the
-# Namespaces page.  This will remove the Namespaces entry from the Quick Index
-# and from the Folder Tree View (if specified). The default is YES.
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
 
 SHOW_NAMESPACES        = YES
 
 # The FILE_VERSION_FILTER tag can be used to specify a program or script that
 # doxygen should invoke to get the current version for each file (typically from
 # the version control system). Doxygen will invoke the program by executing (via
-# popen()) the command <command> <input-file>, where <command> is the value of
-# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file
-# provided by doxygen. Whatever the program writes to standard output
-# is used as the file version. See the manual for examples.
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
 
 FILE_VERSION_FILTER    =
 
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE            =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES         =
+
 #---------------------------------------------------------------------------
-# configuration options related to warning and progress messages
+# Configuration options related to warning and progress messages
 #---------------------------------------------------------------------------
 
-# The QUIET tag can be used to turn on/off the messages that are generated
-# by doxygen. Possible values are YES and NO. If left blank NO is used.
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
 
 QUIET                  = NO
 
 # The WARNINGS tag can be used to turn on/off the warning messages that are
-# generated by doxygen. Possible values are YES and NO. If left blank
-# NO is used.
+# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
 
 WARNINGS               = YES
 
-# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings
-# for undocumented members. If EXTRACT_ALL is set to YES then this flag will
-# automatically be disabled.
+# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
 
-WARN_IF_UNDOCUMENTED   = YES
+WARN_IF_UNDOCUMENTED   = @DOXYGEN_WARN_ON_MISSING@
 
-# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for
-# potential errors in the documentation, such as not documenting some
-# parameters in a documented function, or documenting parameters that
-# don't exist or using markup commands wrongly.
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
 
 WARN_IF_DOC_ERROR      = YES
 
-# This WARN_NO_PARAMDOC option can be abled to get warnings for
-# functions that are documented, but have no documentation for their parameters
-# or return value. If set to NO (the default) doxygen will only warn about
-# wrong or incomplete parameter documentation, but not about the absence of
-# documentation.
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO, doxygen will only warn about wrong or incomplete
+# parameter documentation, but not about the absence of documentation. If
+# EXTRACT_ALL is set to YES then this flag will automatically be disabled.
+# The default value is: NO.
 
 WARN_NO_PARAMDOC       = NO
 
-# The WARN_FORMAT tag determines the format of the warning messages that
-# doxygen can produce. The string should contain the $file, $line, and $text
-# tags, which will be replaced by the file and line number from which the
-# warning originated and the warning text. Optionally the format may contain
-# $version, which will be replaced by the version of the file (if it could
-# be obtained via FILE_VERSION_FILTER)
+# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
+# a warning is encountered.
+# The default value is: NO.
+
+WARN_AS_ERROR          = @DOXYGEN_FATAL_WARNINGS@
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
 
 WARN_FORMAT            = "$file:$line: $text"
 
-# The WARN_LOGFILE tag can be used to specify a file to which warning
-# and error messages should be written. If left blank the output is written
-# to stderr.
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
 
 WARN_LOGFILE           =
 
 #---------------------------------------------------------------------------
-# configuration options related to the input files
+# Configuration options related to the input files
 #---------------------------------------------------------------------------
 
-# The INPUT tag can be used to specify the files and/or directories that contain
-# documented source files. You may enter file names like "myfile.cpp" or
-# directories like "/usr/src/myproject". Separate the files or directories
-# with spaces.
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
 
-INPUT                  = @top_srcdir@/src/lib \
-		         @top_srcdir@/src/core \
- 		         @top_srcdir@/src/feature \
- 		         @top_srcdir@/src/app
+INPUT                  = ./src/
 
 # This tag can be used to specify the character encoding of the source files
-# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
-# also the default input encoding. Doxygen uses libiconv (or the iconv built
-# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for
-# the list of possible encodings.
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
+# possible encodings.
+# The default value is: UTF-8.
 
 INPUT_ENCODING         = UTF-8
 
 # If the value of the INPUT tag contains directories, you can use the
-# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
-# and *.h) to filter out the source-files in the directories. If left
-# blank the following patterns are tested:
-# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx
-# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08,
+# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice.
 
 FILE_PATTERNS          = *.c \
-                         *.h
+                         *.h \
+                         *.inc \
+                         *.md
 
-# The RECURSIVE tag can be used to turn specify whether or not subdirectories
-# should be searched for input files as well. Possible values are YES and NO.
-# If left blank NO is used.
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
 
 RECURSIVE              = YES
 
-# The EXCLUDE tag can be used to specify files and/or directories that should
+# The EXCLUDE tag can be used to specify files and/or directories that should be
 # excluded from the INPUT source files. This way you can easily exclude a
 # subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
 
-EXCLUDE                = tree.h
+EXCLUDE                = ./src/ext \
+                         ./src/trunnel \
+                         ./src/test \
+                         ./src/rust/registry
 
-# The EXCLUDE_SYMLINKS tag can be used select whether or not files or
-# directories that are symbolic links (a Unix filesystem feature) are excluded
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
 # from the input.
+# The default value is: NO.
 
 EXCLUDE_SYMLINKS       = NO
 
 # If the value of the INPUT tag contains directories, you can use the
 # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
-# certain files from those directories. Note that the wildcards are matched
-# against the file with absolute path, so to exclude all test directories
-# for example use the pattern */test/*
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
 
 EXCLUDE_PATTERNS       =
 
@@ -582,510 +889,1169 @@ EXCLUDE_PATTERNS       =
 # output. The symbol name can be a fully qualified name, a word, or if the
 # wildcard * is used, a substring. Examples: ANamespace, AClass,
 # AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
 
 EXCLUDE_SYMBOLS        =
 
-# The EXAMPLE_PATH tag can be used to specify one or more files or
-# directories that contain example code fragments that are included (see
-# the \include command).
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
 
 EXAMPLE_PATH           =
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
-# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
-# and *.h) to filter out the source-files in the directories. If left
-# blank all files are included.
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
 
-EXAMPLE_PATTERNS       =
+EXAMPLE_PATTERNS       = *
 
 # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
-# searched for input files to be used with the \include or \dontinclude
-# commands irrespective of the value of the RECURSIVE tag.
-# Possible values are YES and NO. If left blank NO is used.
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
 
 EXAMPLE_RECURSIVE      = NO
 
-# The IMAGE_PATH tag can be used to specify one or more files or
-# directories that contain image that are included in the documentation (see
-# the \image command).
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
 
 IMAGE_PATH             =
 
 # The INPUT_FILTER tag can be used to specify a program that doxygen should
 # invoke to filter for each input file. Doxygen will invoke the filter program
-# by executing (via popen()) the command <filter> <input-file>, where <filter>
-# is the value of the INPUT_FILTER tag, and <input-file> is the name of an
-# input file. Doxygen will then use the output that the filter program writes
-# to standard output.  If FILTER_PATTERNS is specified, this tag will be
-# ignored.
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
 
 INPUT_FILTER           =
 
 # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
-# basis.  Doxygen will compare the file name with each pattern and apply the
-# filter if there is a match.  The filters are a list of the form:
-# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further
-# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER
-# is applied to all files.
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
 
 FILTER_PATTERNS        =
 
 # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
-# INPUT_FILTER) will be used to filter the input files when producing source
-# files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# INPUT_FILTER) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
 
 FILTER_SOURCE_FILES    = NO
 
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
 #---------------------------------------------------------------------------
-# configuration options related to source browsing
+# Configuration options related to source browsing
 #---------------------------------------------------------------------------
 
-# If the SOURCE_BROWSER tag is set to YES then a list of source files will
-# be generated. Documented entities will be cross-referenced with these sources.
-# Note: To get rid of all source code in the generated output, make sure also
-# VERBATIM_HEADERS is set to NO.
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
 
 SOURCE_BROWSER         = YES
 
-# Setting the INLINE_SOURCES tag to YES will include the body
-# of functions and classes directly in the documentation.
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
 
 INLINE_SOURCES         = NO
 
-# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct
-# doxygen to hide any special comment blocks from generated source code
-# fragments. Normal C and C++ comments will always remain visible.
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
 
-STRIP_CODE_COMMENTS    = YES
+STRIP_CODE_COMMENTS    = NO
 
-# If the REFERENCED_BY_RELATION tag is set to YES
-# then for each documented function all documented
-# functions referencing it will be listed.
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# entity all documented functions referencing it will be listed.
+# The default value is: NO.
 
 REFERENCED_BY_RELATION = YES
 
-# If the REFERENCES_RELATION tag is set to YES
-# then for each documented function all documented entities
-# called/used by that function will be listed.
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
 
-REFERENCES_RELATION    = YES
+REFERENCES_RELATION    = NO
 
-# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
-# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
-# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
-# link to the source code.  Otherwise they will link to the documentstion.
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
 
 REFERENCES_LINK_SOURCE = YES
 
-# If the USE_HTAGS tag is set to YES then the references to source code
-# will point to the HTML generated by the htags(1) tool instead of doxygen
-# built-in source browser. The htags tool is part of GNU's global source
-# tagging system (see http://www.gnu.org/software/global/global.html). You
-# will need version 4.8.6 or higher.
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS        = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see https://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
 
 USE_HTAGS              = NO
 
-# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen
-# will generate a verbatim copy of the header file for each class for
-# which an include is specified. Set to NO to disable this.
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
 
 VERBATIM_HEADERS       = YES
 
 #---------------------------------------------------------------------------
-# configuration options related to the alphabetical class index
+# Configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
 
-# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index
-# of all compounds will be generated. Enable this if the project
-# contains a lot of classes, structs, unions or interfaces.
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
 
-ALPHABETICAL_INDEX     = NO
+ALPHABETICAL_INDEX     = YES
 
-# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then
-# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns
-# in which this list will be split (can be a number in the range [1..20])
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
 
 COLS_IN_ALPHA_INDEX    = 5
 
-# In case all classes in a project start with a common prefix, all
-# classes will be put under the same header in the alphabetical index.
-# The IGNORE_PREFIX tag can be used to specify one or more prefixes that
-# should be ignored while generating the index headers.
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
 
 IGNORE_PREFIX          =
 
 #---------------------------------------------------------------------------
-# configuration options related to the HTML output
+# Configuration options related to the HTML output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_HTML tag is set to YES (the default) Doxygen will
-# generate HTML output.
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
 
 GENERATE_HTML          = YES
 
-# The HTML_OUTPUT tag is used to specify where the HTML docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `html' will be used as the default path.
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 HTML_OUTPUT            = html
 
-# The HTML_FILE_EXTENSION tag can be used to specify the file extension for
-# each generated HTML page (for example: .htm,.php,.asp). If it is left blank
-# doxygen will generate files with .html extension.
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 HTML_FILE_EXTENSION    = .html
 
-# The HTML_HEADER tag can be used to specify a personal HTML header for
-# each generated HTML page. If it is left blank doxygen will generate a
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
 # standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 HTML_HEADER            =
 
-# The HTML_FOOTER tag can be used to specify a personal HTML footer for
-# each generated HTML page. If it is left blank doxygen will generate a
-# standard footer.
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 HTML_FOOTER            =
 
-# The HTML_STYLESHEET tag can be used to specify a user-defined cascading
-# style sheet that is used by each HTML page. It can be used to
-# fine-tune the look of the HTML output. If the tag is left blank doxygen
-# will generate a default style sheet. Note that doxygen will try to copy
-# the style sheet file to the HTML output directory, so don't put your own
-# stylesheet in the HTML output directory as well, or it will be erased!
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 HTML_STYLESHEET        =
 
-# If the GENERATE_HTMLHELP tag is set to YES, additional index files
-# will be generated that can be used as input for tools like the
-# Microsoft HTML help workshop to generate a compiled HTML help file (.chm)
-# of the generated HTML documentation.
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET  = doc/tor-doxygen.css
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES       =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the style sheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE    = 150
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT    = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA  = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to YES can help to show when doxygen was last run and thus if the
+# documentation is up to date.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP         = NO
+
+# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
+# documentation will contain a main index with vertical navigation menus that
+# are dynamically created via Javascript. If disabled, the navigation index will
+# consists of multiple levels of tabs that are statically embedded in every HTML
+# page. Disable this option to support browsers that do not have Javascript,
+# like the Qt help browser.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_MENUS     = YES
 
-GENERATE_HTMLHELP      = NO
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS  = NO
 
-# If the GENERATE_DOCSET tag is set to YES, additional index files
-# will be generated that can be used as input for Apple's Xcode 3
-# integrated development environment, introduced with OSX 10.5 (Leopard).
-# To create a documentation set, doxygen will generate a Makefile in the
-# HTML output directory. Running make will produce the docset in that
-# directory and running "make install" will install the docset in
-# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find
-# it at startup.
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: https://developer.apple.com/xcode/), introduced with OSX
+# 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
+# genXcode/_index.html for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 GENERATE_DOCSET        = NO
 
-# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the
-# feed. A documentation feed provides an umbrella under which multiple
-# documentation sets from a single provider (such as a company or product suite)
-# can be grouped.
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
 
-DOCSET_FEEDNAME        = "Doxygen generated docs for Tor"
+DOCSET_FEEDNAME        = "Doxygen generated docs"
 
-# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that
-# should uniquely identify the documentation set bundle. This should be a
-# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen
-# will append .docset to the name.
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
 
-DOCSET_BUNDLE_ID       = org.torproject.Tor
+DOCSET_BUNDLE_ID       = org.doxygen.Project
 
-# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
-# documentation will contain sections that can be hidden and shown after the
-# page has loaded. For this to work a browser that supports
-# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox
-# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari).
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
 
-HTML_DYNAMIC_SECTIONS  = NO
+DOCSET_PUBLISHER_ID    = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME  = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP      = NO
 
-# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can
-# be used to specify the file name of the resulting .chm file. You
-# can add a path in front of the file if the result should not be
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
 # written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
 CHM_FILE               =
 
-# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can
-# be used to specify the location (absolute path including file name) of
-# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run
-# the HTML help compiler on the generated index.hhp.
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler (hhc.exe). If non-empty,
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
 HHC_LOCATION           =
 
-# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag
-# controls if a separate .chi index file is generated (YES) or that
-# it should be included in the master .chm file (NO).
+# The GENERATE_CHI flag controls if a separate .chi index file is generated
+# (YES) or that it should be included in the master .chm file (NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
 GENERATE_CHI           = NO
 
-# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING
-# is used to encode HtmlHelp index (hhk), content (hhc) and project file
-# content.
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
 CHM_INDEX_ENCODING     =
 
-# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag
-# controls whether a binary table of contents is generated (YES) or a
-# normal table of contents (NO) in the .chm file.
+# The BINARY_TOC flag controls whether a binary table of contents is generated
+# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
 BINARY_TOC             = NO
 
-# The TOC_EXPAND flag can be set to YES to add extra items for group members
-# to the contents of the HTML help documentation and to the tree view.
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
 TOC_EXPAND             = NO
 
-# The DISABLE_INDEX tag can be used to turn on/off the condensed index at
-# top of each HTML page. The value NO (the default) enables the index and
-# the value YES disables it.
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP           = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE               =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE          = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER     = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME   =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS  =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS  =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION           =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP   = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID         = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 DISABLE_INDEX          = NO
 
-# This tag can be used to set the number of enum values (range [1..20])
-# that doxygen will group on one line in the generated HTML documentation.
-
-ENUM_VALUES_PER_LINE   = 4
-
 # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
-# structure should be generated to display hierarchical information.
-# If the tag value is set to FRAME, a side panel will be generated
-# containing a tree-like index structure (just like the one that
-# is generated for HTML Help). For this to work a browser that supports
-# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+,
-# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are
-# probably better off using the HTML help feature. Other possible values
-# for this tag are: HIERARCHIES, which will generate the Groups, Directories,
-# and Class Hiererachy pages using a tree view instead of an ordered list;
-# ALL, which combines the behavior of FRAME and HIERARCHIES; and NONE, which
-# disables this behavior completely. For backwards compatibility with previous
-# releases of Doxygen, the values YES and NO are equivalent to FRAME and NONE
-# respectively.
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 GENERATE_TREEVIEW      = NO
 
-# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be
-# used to set the initial width (in pixels) of the frame in which the tree
-# is shown.
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 TREEVIEW_WIDTH         = 250
 
-# Use this tag to change the font size of Latex formulas included
-# as images in the HTML documentation. The default is 10. Note that
-# when you change the font size after a successful doxygen run you need
-# to manually remove any form_*.png images from the HTML output directory
-# to force them to be regenerated.
+# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW    = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
 
 FORMULA_FONTSIZE       = 10
 
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT    = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# https://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX            = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT         = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from https://www.mathjax.org before deployment.
+# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS     =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE       =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE           = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH    = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: https://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH        = NO
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: https://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL       =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE        = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID     =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS  =
+
 #---------------------------------------------------------------------------
-# configuration options related to the LaTeX output
+# Configuration options related to the LaTeX output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will
-# generate Latex output.
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
 
-GENERATE_LATEX         = YES
+GENERATE_LATEX         = NO
 
-# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `latex' will be used as the default path.
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_OUTPUT           = latex
 
 # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
-# invoked. If left blank `latex' will be used as the default command name.
-
-LATEX_CMD_NAME         = latex
-
-# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to
-# generate index for LaTeX. If left blank `makeindex' will be used as the
-# default command name.
+# invoked.
+#
+# Note that when not enabling USE_PDFLATEX the default is latex when enabling
+# USE_PDFLATEX the default is pdflatex and when in the later case latex is
+# chosen this is overwritten by pdflatex. For specific output languages the
+# default can have been set differently, this depends on the implementation of
+# the output language.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME         =
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# Note: This tag is used in the Makefile / make.bat.
+# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
+# (.tex).
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
 MAKEINDEX_CMD_NAME     = makeindex
 
-# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact
-# LaTeX documents. This may be useful for small projects and may help to
-# save some trees in general.
+# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
+# generate index for LaTeX.
+# Note: This tag is used in the generated output file (.tex).
+# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
+# The default value is: \makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
-COMPACT_LATEX          = NO
+LATEX_MAKEINDEX_CMD    = \makeindex
 
-# The PAPER_TYPE tag can be used to set the paper type that is used
-# by the printer. Possible values are: a4, a4wide, letter, legal and
-# executive. If left blank a4wide will be used.
+# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
-PAPER_TYPE             = a4wide
+COMPACT_LATEX          = NO
 
-# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX
-# packages that should be included in the LaTeX output.
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE             = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. The package can be specified just
+# by its name or with the correct syntax as to be used with the LaTeX
+# \usepackage command. To get the times font for instance you can specify :
+# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
+# To use the option intlimits with the amsmath package you can specify:
+# EXTRA_PACKAGES=[intlimits]{amsmath}
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
 EXTRA_PACKAGES         =
 
-# The LATEX_HEADER tag can be used to specify a personal LaTeX header for
-# the generated latex document. The header should contain everything until
-# the first chapter. If it is left blank doxygen will generate a
-# standard header. Notice: only use this tag if you know what you are doing!
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
+# string, for the replacement values of the other commands the user is referred
+# to HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_HEADER           =
 
-# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated
-# is prepared for conversion to pdf (using ps2pdf). The pdf file will
-# contain links (just like the HTML output) instead of page references
-# This makes the output suitable for online browsing using a pdf viewer.
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
-PDF_HYPERLINKS         = NO
+LATEX_FOOTER           =
 
-# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of
-# plain latex in the generated Makefile. Set this option to YES to get a
+# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# LaTeX style sheets that are included after the standard style sheets created
+# by doxygen. Using this option one can overrule certain style aspects. Doxygen
+# will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_STYLESHEET =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES      =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS         = YES
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES, to get a
 # higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
-USE_PDFLATEX           = NO
+USE_PDFLATEX           = YES
 
-# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode.
-# command to the generated LaTeX files. This will instruct LaTeX to keep
-# running if errors occur, instead of asking the user for help.
-# This option is also used when generating formulas in HTML.
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_BATCHMODE        = NO
 
-# If LATEX_HIDE_INDICES is set to YES then doxygen will not
-# include the index chapters (such as File Index, Compound Index, etc.)
-# in the output.
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_HIDE_INDICES     = NO
 
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE      = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE        = plain
+
+# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_TIMESTAMP        = NO
+
+# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
+# path from which the emoji images will be read. If a relative path is entered,
+# it will be relative to the LATEX_OUTPUT directory. If left blank the
+# LATEX_OUTPUT directory will be used.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EMOJI_DIRECTORY  =
+
 #---------------------------------------------------------------------------
-# configuration options related to the RTF output
+# Configuration options related to the RTF output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output
-# The RTF output is optimized for Word 97 and may not look very pretty with
-# other RTF readers or editors.
+# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
 
 GENERATE_RTF           = NO
 
-# The RTF_OUTPUT tag is used to specify where the RTF docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `rtf' will be used as the default path.
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_OUTPUT             = rtf
 
-# If the COMPACT_RTF tag is set to YES Doxygen generates more compact
-# RTF documents. This may be useful for small projects and may help to
-# save some trees in general.
+# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
 
 COMPACT_RTF            = NO
 
-# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated
-# will contain hyperlink fields. The RTF file will
-# contain links (just like the HTML output) instead of page references.
-# This makes the output suitable for online browsing using WORD or other
-# programs which support those fields.
-# Note: wordpad (write) and others do not support links.
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_HYPERLINKS         = NO
 
 # Load stylesheet definitions from file. Syntax is similar to doxygen's
-# config file, i.e. a series of assignments. You only have to provide
+# configuration file, i.e. a series of assignments. You only have to provide
 # replacements, missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_STYLESHEET_FILE    =
 
-# Set optional variables used in the generation of an rtf document.
-# Syntax is similar to doxygen's config file.
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's configuration file. A template extensions file can be
+# generated using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_EXTENSIONS_FILE    =
 
+# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
+# with syntax highlighting in the RTF output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_SOURCE_CODE        = NO
+
 #---------------------------------------------------------------------------
-# configuration options related to the man page output
+# Configuration options related to the man page output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_MAN tag is set to YES (the default) Doxygen will
-# generate man pages
+# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
 
 GENERATE_MAN           = NO
 
-# The MAN_OUTPUT tag is used to specify where the man pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `man' will be used as the default path.
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
 
 MAN_OUTPUT             = man
 
-# The MAN_EXTENSION tag determines the extension that is added to
-# the generated man pages (default is the subroutine's section .3)
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
 
 MAN_EXTENSION          = .3
 
-# If the MAN_LINKS tag is set to YES and Doxygen generates man output,
-# then it will generate one additional man file for each entity
-# documented in the real man page(s). These additional files
-# only source the real man page, but without them the man command
-# would be unable to find the correct page. The default is NO.
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR             =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
 
 MAN_LINKS              = NO
 
 #---------------------------------------------------------------------------
-# configuration options related to the XML output
+# Configuration options related to the XML output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_XML tag is set to YES Doxygen will
-# generate an XML file that captures the structure of
-# the code including all documentation.
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
 
 GENERATE_XML           = NO
 
-# The XML_OUTPUT tag is used to specify where the XML pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `xml' will be used as the default path.
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
 
 XML_OUTPUT             = xml
 
-# If the XML_PROGRAMLISTING tag is set to YES Doxygen will
-# dump the program listings (including syntax highlighting
-# and cross-referencing information) to the XML output. Note that
-# enabling this will significantly increase the size of the XML output.
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
 
 XML_PROGRAMLISTING     = YES
 
+# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
+# namespace members in file scope as well, matching the HTML output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_NS_MEMB_FILE_SCOPE = NO
+
 #---------------------------------------------------------------------------
-# configuration options for the AutoGen Definitions output
+# Configuration options related to the DOCBOOK output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will
-# generate an AutoGen Definitions (see autogen.sf.net) file
-# that captures the structure of the code including all
-# documentation. Note that this feature is still experimental
-# and incomplete at the moment.
+# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK       = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT         = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
+# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# the structure of the code including all documentation. Note that this feature
+# is still experimental and incomplete at the moment.
+# The default value is: NO.
 
 GENERATE_AUTOGEN_DEF   = NO
 
 #---------------------------------------------------------------------------
-# configuration options related to the Perl module output
+# Configuration options related to the Perl module output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_PERLMOD tag is set to YES Doxygen will
-# generate a Perl module file that captures the structure of
-# the code including all documentation. Note that this
-# feature is still experimental and incomplete at the
-# moment.
+# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
 
 GENERATE_PERLMOD       = NO
 
-# If the PERLMOD_LATEX tag is set to YES Doxygen will generate
-# the necessary Makefile rules, Perl scripts and LaTeX code to be able
-# to generate PDF and DVI output from the Perl module output.
+# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
 
 PERLMOD_LATEX          = NO
 
-# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be
-# nicely formatted so it can be parsed by a human reader.  This is useful
-# if you want to understand what is going on.  On the other hand, if this
-# tag is set to NO the size of the Perl module output will be much smaller
-# and Perl will parse it just the same.
+# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO, the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
 
 PERLMOD_PRETTY         = YES
 
-# The names of the make variables in the generated doxyrules.make file
-# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX.
-# This is useful so different doxyrules.make files included by the same
-# Makefile don't overwrite each other's variables.
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
 
 PERLMOD_MAKEVAR_PREFIX =
 
@@ -1093,108 +2059,136 @@ PERLMOD_MAKEVAR_PREFIX =
 # Configuration options related to the preprocessor
 #---------------------------------------------------------------------------
 
-# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will
-# evaluate all C-preprocessor directives found in the sources and include
-# files.
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
 
 ENABLE_PREPROCESSING   = YES
 
-# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro
-# names in the source code. If set to NO (the default) only conditional
-# compilation will be performed. Macro expansion can be done in a controlled
-# way by setting EXPAND_ONLY_PREDEF to YES.
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-MACRO_EXPANSION        = NO
+MACRO_EXPANSION        = YES
 
-# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES
-# then the macro expansion is limited to the macros specified with the
-# PREDEFINED and EXPAND_AS_DEFINED tags.
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_ONLY_PREDEF     = NO
+EXPAND_ONLY_PREDEF     = YES
 
-# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files
-# in the INCLUDE_PATH (see below) will be search if a #include is found.
+# If the SEARCH_INCLUDES tag is set to YES, the include files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
 SEARCH_INCLUDES        = YES
 
 # The INCLUDE_PATH tag can be used to specify one or more directories that
-# contain include files that are not input files but should be processed by
-# the preprocessor.
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
 INCLUDE_PATH           =
 
 # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
 # patterns (like *.h and *.hpp) to filter out the header-files in the
-# directories. If left blank, the patterns specified with FILE_PATTERNS will
-# be used.
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
 INCLUDE_FILE_PATTERNS  =
 
-# The PREDEFINED tag can be used to specify one or more macro names that
-# are defined before the preprocessor is started (similar to the -D option of
-# gcc). The argument of the tag is a list of macros of the form: name
-# or name=definition (no spaces). If the definition and the = are
-# omitted =1 is assumed. To prevent a macro definition from being
-# undefined via #undef or recursively expanded use the := operator
-# instead of the = operator.
-
-PREDEFINED             =
-
-# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then
-# this tag can be used to specify a list of macro names that should be expanded.
-# The macro definition that is found in the sources will be used.
-# Use the PREDEFINED tag if you want to use a different macro definition.
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED             = "MOCK_IMPL(a,b,c)=a b c" \
+                         "MOCK_DECL(a,b,c)=a b c" \
+                         __attribute__(x)= \
+                         "BEGIN_CONF_STRUCT(x)=struct x {" \
+                         "END_CONF_STRUCT(x)=};" \
+                         "CONF_VAR(a,b,c,d)=b a;" \
+                         "CHECK_PRINTF(a, b)="
+
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
 EXPAND_AS_DEFINED      =
 
-# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then
-# doxygen's preprocessor will remove all function-like macros that are alone
-# on a line, have an all uppercase name, and do not end with a semicolon. Such
-# function macros are typically used for boiler-plate code, and will confuse
-# the parser if not removed.
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
 SKIP_FUNCTION_MACROS   = YES
 
 #---------------------------------------------------------------------------
-# Configuration::additions related to external references
+# Configuration options related to external references
 #---------------------------------------------------------------------------
 
-# The TAGFILES option can be used to specify one or more tagfiles.
-# Optionally an initial location of the external documentation
-# can be added for each tagfile. The format of a tag file without
-# this location is as follows:
-#   TAGFILES = file1 file2 ...
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
 # Adding location for the tag files is done as follows:
-#   TAGFILES = file1=loc1 "file2 = loc2" ...
-# where "loc1" and "loc2" can be relative or absolute paths or
-# URLs. If a location is present for each tag, the installdox tool
-# does not have to be run to correct the links.
-# Note that each tag file must have a unique name
-# (where the name does NOT include the path)
-# If a tag file is not located in the directory in which doxygen
-# is run, you must also specify the path to the tagfile here.
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
 
 TAGFILES               =
 
-# When a file name is specified after GENERATE_TAGFILE, doxygen will create
-# a tag file that is based on the input files it reads.
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
 
 GENERATE_TAGFILE       =
 
-# If the ALLEXTERNALS tag is set to YES all external classes will be listed
-# in the class index. If set to NO only the inherited external classes
-# will be listed.
+# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
+# the class index. If set to NO, only the inherited external classes will be
+# listed.
+# The default value is: NO.
 
 ALLEXTERNALS           = NO
 
-# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed
-# in the modules index. If set to NO, only the current project's groups will
-# be listed.
+# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
 
 EXTERNAL_GROUPS        = YES
 
+# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES         = YES
+
 # The PERL_PATH should be the absolute path and name of the perl script
-# interpreter (i.e. the result of `which perl').
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
 
 PERL_PATH              = /usr/bin/perl
 
@@ -1202,196 +2196,317 @@ PERL_PATH              = /usr/bin/perl
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------
 
-# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will
-# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base
-# or super classes. Setting the tag to NO turns the diagrams off. Note that
-# this option is superseded by the HAVE_DOT option below. This is only a
-# fallback. It is recommended to install and use dot, since it yields more
+# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
 # powerful graphs.
+# The default value is: YES.
 
 CLASS_DIAGRAMS         = YES
 
 # You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see
-# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
 # documentation. The MSCGEN_PATH tag allows you to specify the directory where
 # the mscgen tool resides. If left empty the tool is assumed to be found in the
 # default search path.
 
 MSCGEN_PATH            =
 
-# If set to YES, the inheritance and collaboration graphs will hide
-# inheritance and usage relations if the target is undocumented
-# or is not a class.
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH               =
+
+# If set to YES the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
 
 HIDE_UNDOC_RELATIONS   = YES
 
 # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
-# available from the path. This tool is part of Graphviz, a graph visualization
-# toolkit from AT&T and Lucent Bell Labs. The other options in this section
-# have no effect if this option is set to NO (the default)
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: NO.
 
 HAVE_DOT               = NO
 
-# By default doxygen will write a font called FreeSans.ttf to the output
-# directory and reference it in all dot files that doxygen generates. This
-# font does not include all possible unicode characters however, so when you need
-# these (or just want a differently looking font) you can specify the font name
-# using DOT_FONTNAME. You need need to make sure dot is able to find the font,
-# which can be done by putting it in a standard location or by setting the
-# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory
-# containing the font.
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS        = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_FONTNAME           =
+DOT_FONTNAME           = Helvetica
 
-# By default doxygen will tell dot to use the output directory to look for the
-# FreeSans.ttf font (which doxygen will put there itself). If you specify a
-# different font using DOT_FONTNAME you can set the path where dot
-# can find it using this tag.
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE           = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_FONTPATH           =
 
-# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for each documented class showing the direct and
-# indirect inheritance relations. Setting this tag to YES will force the
-# the CLASS_DIAGRAMS tag to NO.
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 CLASS_GRAPH            = YES
 
-# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for each documented class showing the direct and
-# indirect implementation dependencies (inheritance, containment, and
-# class references variables) of the class with other documented classes.
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 COLLABORATION_GRAPH    = YES
 
-# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for groups, showing the direct groups dependencies
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 GROUP_GRAPHS           = YES
 
-# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
+# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
 # collaboration diagrams in a style similar to the OMG's Unified Modeling
 # Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 UML_LOOK               = NO
 
-# If set to YES, the inheritance and collaboration graphs will show the
-# relations between templates and their instances.
-
-TEMPLATE_RELATIONS     = YES
-
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT
-# tags are set to YES then doxygen will generate a graph for each documented
-# file showing the direct and indirect include dependencies of the file with
-# other documented files.
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS   = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS     = NO
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 INCLUDE_GRAPH          = YES
 
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and
-# HAVE_DOT tags are set to YES then doxygen will generate a graph for each
-# documented header file showing the documented files that directly or
-# indirectly include this file.
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 INCLUDED_BY_GRAPH      = YES
 
-# If the CALL_GRAPH and HAVE_DOT options are set to YES then
-# doxygen will generate a call dependency graph for every global function
-# or class method. Note that enabling this option will significantly increase
-# the time of a run. So in most cases it will be better to enable call graphs
-# for selected functions only using the \callgraph command.
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command. Disabling a call graph can be
+# accomplished by means of the command \hidecallgraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 CALL_GRAPH             = NO
 
-# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then
-# doxygen will generate a caller dependency graph for every global function
-# or class method. Note that enabling this option will significantly increase
-# the time of a run. So in most cases it will be better to enable caller
-# graphs for selected functions only using the \callergraph command.
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command. Disabling a caller graph can be
+# accomplished by means of the command \hidecallergraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 CALLER_GRAPH           = NO
 
-# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen
-# will graphical hierarchy of all classes instead of a textual one.
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 GRAPHICAL_HIERARCHY    = YES
 
-# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES
-# then doxygen will show the dependencies a directory has on other directories
-# in a graphical way. The dependency relations are determined by the #include
-# relations between the files in the directories.
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DIRECTORY_GRAPH        = YES
 
 # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
-# generated by dot. Possible values are png, jpg, or gif
-# If left blank png will be used.
+# generated by dot. For an explanation of the image formats see the section
+# output formats in the documentation of the dot tool (Graphviz (see:
+# http://www.graphviz.org/)).
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo,
+# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
+# png:gdiplus:gdiplus.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_IMAGE_FORMAT       = png
 
-# The tag DOT_PATH can be used to specify the path where the dot tool can be
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG        = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
 # found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_PATH               =
 
 # The DOTFILE_DIRS tag can be used to specify one or more directories that
-# contain dot files that are included in the documentation (see the
-# \dotfile command).
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOTFILE_DIRS           =
 
-# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of
-# nodes that will be shown in the graph. If the number of nodes in a graph
-# becomes larger than this value, doxygen will truncate the graph, which is
-# visualized by representing a node as a red box. Note that doxygen if the
-# number of direct children of the root node in a graph is already larger than
-# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note
-# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS           =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS           =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+
+PLANTUML_JAR_PATH      =
+
+# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
+# configuration file for plantuml.
+
+PLANTUML_CFG_FILE      =
+
+# When using plantuml, the specified paths are searched for files specified by
+# the !include statement in a plantuml block.
+
+PLANTUML_INCLUDE_PATH  =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_GRAPH_MAX_NODES    = 50
 
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the
-# graphs generated by dot. A depth value of 3 means that only nodes reachable
-# from the root by following a path via at most 3 edges will be shown. Nodes
-# that lay further from the root node will be omitted. Note that setting this
-# option to 1 or 2 may greatly reduce the computation time needed for large
-# code bases. Also note that the size of a graph can be further restricted by
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
 # DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 MAX_DOT_GRAPH_DEPTH    = 0
 
 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
-# background. This is enabled by default, which results in a transparent
-# background. Warning: Depending on the platform used, enabling this option
-# may lead to badly anti-aliased labels on the edges of a graph (i.e. they
-# become hard to read).
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_TRANSPARENT        = NO
 
-# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
+# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
 # files in one run (i.e. multiple -o and -T options on the command line). This
-# makes dot run faster, but since only newer versions of dot (>1.8.10)
-# support this, this feature is disabled by default.
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_MULTI_TARGETS      = NO
 
-# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will
-# generate a legend page explaining the meaning of the various boxes and
-# arrows in the dot generated graphs.
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 GENERATE_LEGEND        = YES
 
-# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will
-# remove the intermediate dot files that are used to generate
-# the various graphs.
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_CLEANUP            = YES
-
-#---------------------------------------------------------------------------
-# Configuration::additions related to the search engine
-#---------------------------------------------------------------------------
-
-# The SEARCHENGINE tag specifies whether or not a search engine should be
-# used. If set to NO the values of all tags below this one will be ignored.
-
-SEARCHENGINE           = NO
diff --git a/LICENSE b/LICENSE
index 897f529b7b..abab8e33fe 100644
--- a/LICENSE
+++ b/LICENSE
@@ -9,7 +9,8 @@
         there may be other license terms that you should be aware of.
 
 ===============================================================================
-Tor is distributed under this license:
+Tor is distributed under the "3-clause BSD" license, a commonly used
+software license that means Tor is both free software and open source:
 
 Copyright (c) 2001-2004, Roger Dingledine
 Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
diff --git a/Makefile.am b/Makefile.am
index c7bc514287..a212ae19c6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -21,7 +21,12 @@ AM_CPPFLAGS=\
 	-I$(top_srcdir)/src/ext/trunnel \
 	-I$(top_srcdir)/src/trunnel
 
-AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ @TOR_ZSTD_CFLAGS@
+AM_CFLAGS=					\
+	@TOR_SYSTEMD_CFLAGS@			\
+	@CFLAGS_BUGTRAP@			\
+	@TOR_LZMA_CFLAGS@			\
+	@TOR_ZSTD_CFLAGS@
+
 SHELL=@SHELL@
 
 if COVERAGE_ENABLED
@@ -31,9 +36,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/app/tor$(EXEEXT)
 endif
 
 if USE_RUST
-## this MUST be $(), otherwise am__DEPENDENCIES will not track it
-rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) \
-	$(TOR_RUST_EXTRA_LIBS)
+rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH)
 else
 rust_ldadd=
 endif
@@ -42,6 +45,10 @@ endif
 TOR_UTIL_LIBS = \
         src/lib/libtor-geoip.a \
 	src/lib/libtor-process.a \
+        src/lib/libtor-buf.a \
+	src/lib/libtor-confmgt.a \
+	src/lib/libtor-pubsub.a \
+	src/lib/libtor-dispatch.a \
 	src/lib/libtor-time.a \
 	src/lib/libtor-fs.a \
 	src/lib/libtor-encoding.a \
@@ -62,6 +69,8 @@ TOR_UTIL_LIBS = \
 	src/lib/libtor-malloc.a \
 	src/lib/libtor-wallclock.a \
 	src/lib/libtor-err.a \
+	src/lib/libtor-version.a \
+	src/lib/libtor-llharden.a \
 	src/lib/libtor-intmath.a \
 	src/lib/libtor-ctime.a
 
@@ -71,6 +80,10 @@ if UNITTESTS_ENABLED
 TOR_UTIL_TESTING_LIBS = \
         src/lib/libtor-geoip-testing.a \
 	src/lib/libtor-process-testing.a \
+        src/lib/libtor-buf-testing.a \
+	src/lib/libtor-confmgt-testing.a \
+	src/lib/libtor-pubsub-testing.a \
+	src/lib/libtor-dispatch-testing.a \
 	src/lib/libtor-time-testing.a \
 	src/lib/libtor-fs-testing.a \
 	src/lib/libtor-encoding-testing.a \
@@ -91,6 +104,8 @@ TOR_UTIL_TESTING_LIBS = \
 	src/lib/libtor-malloc-testing.a \
 	src/lib/libtor-wallclock-testing.a \
 	src/lib/libtor-err-testing.a \
+	src/lib/libtor-version-testing.a \
+	src/lib/libtor-llharden-testing.a \
 	src/lib/libtor-intmath.a \
 	src/lib/libtor-ctime-testing.a
 endif
@@ -152,16 +167,52 @@ include contrib/include.am
 manpages: $(nodist_man1_MANS)
 
 EXTRA_DIST+= \
-	ChangeLog					\
-	CONTRIBUTING					\
-	CODE_OF_CONDUCT                                 \
-	INSTALL						\
-	LICENSE						\
-	Makefile.nmake					\
-	README						\
-	ReleaseNotes					\
-	scripts/maint/checkIncludes.py                  \
-	scripts/maint/checkSpace.pl
+	ChangeLog							\
+	CONTRIBUTING							\
+	CODE_OF_CONDUCT							\
+	INSTALL								\
+	LICENSE								\
+	Makefile.nmake							\
+	README								\
+	ReleaseNotes							\
+	scripts/maint/checkIncludes.py					\
+	scripts/maint/checkSpace.pl					\
+	scripts/maint/checkSpaceTest.sh					\
+	scripts/maint/checkspace_tests/dubious.c			\
+	scripts/maint/checkspace_tests/dubious.h			\
+	scripts/maint/checkspace_tests/expected.txt			\
+	scripts/maint/checkspace_tests/good_guard.h			\
+	scripts/maint/checkspace_tests/same_guard.h			\
+	scripts/maint/checkspace_tests/subdir/dubious.c			\
+	scripts/maint/checkShellScripts.sh				\
+	scripts/maint/practracker/README				\
+	scripts/maint/practracker/exceptions.txt			\
+	scripts/maint/practracker/includes.py				\
+	scripts/maint/practracker/metrics.py				\
+	scripts/maint/practracker/practracker.py			\
+	scripts/maint/practracker/practracker_tests.py			\
+	scripts/maint/practracker/problem.py				\
+	scripts/maint/practracker/testdata/.may_include			\
+	scripts/maint/practracker/testdata/a.c				\
+	scripts/maint/practracker/testdata/b.c				\
+	scripts/maint/practracker/testdata/ex0-expected.txt		\
+	scripts/maint/practracker/testdata/ex0.txt			\
+	scripts/maint/practracker/testdata/ex1-expected.txt		\
+	scripts/maint/practracker/testdata/ex1.txt			\
+	scripts/maint/practracker/testdata/ex1-overbroad-expected.txt	\
+	scripts/maint/practracker/testdata/ex1-regen-expected.txt	\
+	scripts/maint/practracker/testdata/ex1-regen-overbroad-expected.txt \
+	scripts/maint/practracker/testdata/ex.txt			\
+	scripts/maint/practracker/testdata/header.h			\
+	scripts/maint/practracker/testdata/not_c_file			\
+	scripts/maint/practracker/test_practracker.sh			\
+	scripts/maint/practracker/util.py				\
+	scripts/coccinelle/apply.sh					\
+	scripts/coccinelle/check_cocci_parse.sh				\
+	scripts/coccinelle/exceptions.txt				\
+	scripts/coccinelle/test-operator-cleanup			\
+	scripts/coccinelle/tor-coccinelle.h				\
+	scripts/coccinelle/try_parse.sh
 
 ## This tells etags how to find mockable function definitions.
 AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
@@ -179,7 +230,7 @@ TEST_CFLAGS=
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
 TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
-TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings
+TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS=--quiet --only-warnings
 
 if LIBFUZZER_ENABLED
 TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div
@@ -209,65 +260,203 @@ dist-rpm: dist-gzip
 	echo "RPM build finished";						\
 	#end of dist-rpm
 
-doxygen:
-	doxygen && cd doc/doxygen/latex && make
+.PHONY: doxygen
+doxygen: Doxyfile
+	mkdir -p doc/doxygen
+	(cd "$(top_srcdir)" && doxygen "$(abs_top_builddir)/Doxyfile")
 
 test: all
 	$(top_builddir)/src/test/test
 
-check-local: check-spaces check-changes check-includes
+shellcheck:
+	$(top_srcdir)/scripts/maint/checkShellScripts.sh
+
+check-local:					\
+	check-spaces				\
+	check-changes				\
+	check-includes				\
+	shellcheck
 
+# test-network requires a copy of Chutney in $CHUTNEY_PATH.
+# Chutney can be cloned from https://git.torproject.org/chutney.git .
+.PHONY: need-chutney-path
 need-chutney-path:
 	@if test ! -d "$$CHUTNEY_PATH"; then \
 		echo '$$CHUTNEY_PATH was not set.'; \
-		if test -d $(top_srcdir)/../chutney -a -x $(top_srcdir)/../chutney/chutney; then \
-			echo "Assuming test-network.sh will find" $(top_srcdir)/../chutney; \
+		if test -d "$(top_srcdir)/../chutney" && \
+		   test -x "$(top_srcdir)/../chutney/chutney"; then \
+			echo "Assuming test-network.sh will find" \
+			  "$(top_srcdir)/../chutney"; \
 		else \
 			echo; \
-			echo "To run these tests, git clone https://git.torproject.org/chutney.git ; export CHUTNEY_PATH=\`pwd\`/chutney"; \
+			echo "To run these tests," \
+			  "git clone" \
+			  "https://git.torproject.org/chutney.git ;" \
+			  "export CHUTNEY_PATH=\`pwd\`/chutney"; \
 			exit 1; \
 		fi \
 	fi
 
-# Note that test-network requires a copy of Chutney in $CHUTNEY_PATH.
-# Chutney can be cloned from https://git.torproject.org/chutney.git .
-test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert
-	$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS)
+# Run some basic tests using automake's test-driver
+.PHONY: test-network
+# Hide directory path logs from submakes using $(MAKE) -s
+test-network:
+	@$(MAKE) -s test-network-mkdir
+	@$(MAKE) -s test-network-clean
+	@$(MAKE) -s test-network-run \
+	  ipv4_flavors="$(TEST_CHUTNEY_FLAVOR_QUICK)" \
+	  ipv6_flavors="$(TEST_CHUTNEY_FLAVOR_QUICK_IPV6)"
+	@$(MAKE) -s test-network-results
 
 # Run all available tests using automake's test-driver
-# only run IPv6 tests if we can ping6 ::1 (localhost)
-# only run IPv6 tests if we can ping ::1 (localhost)
-# some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
-# only run mixed tests if we have a tor-stable binary
-# Try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
-# because they're incompatible
-test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
-	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
-	rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
-	@flavors="$(TEST_CHUTNEY_FLAVORS)"; \
-	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || ping -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \
-		echo "ping6 ::1 or ping ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
-		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
-	else \
-		echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
-		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
+.PHONY: test-network-all
+# Hide directory path logs from submakes using $(MAKE) -s
+test-network-all:
+	@$(MAKE) -s test-network-mkdir
+	@$(MAKE) -s test-network-clean
+	@$(MAKE) -s test-network-run \
+	  ipv4_flavors="$(TEST_CHUTNEY_FLAVORS)" \
+	  mixed_flavors="$(TEST_CHUTNEY_FLAVORS_MIXED)" \
+	  ipv6_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6)" \
+	  ipv6_mixed_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6_MIXED)"
+	@$(MAKE) -s test-network-results
+
+# Run IPv4 and mixed tests using automake's test-driver
+.PHONY: test-network-ipv4
+# Hide directory path logs from submakes using $(MAKE) -s
+test-network-ipv4:
+	@$(MAKE) -s test-network-mkdir
+	@$(MAKE) -s test-network-clean
+	@$(MAKE) -s test-network-run \
+	  ipv4_flavors="$(TEST_CHUTNEY_FLAVORS)" \
+	  mixed_flavors="$(TEST_CHUTNEY_FLAVORS_MIXED)"
+	@$(MAKE) -s test-network-results
+
+# Run IPv6 tests using automake's test-driver
+.PHONY: test-network-ipv6
+# Hide directory path logs from submakes using $(MAKE) -s
+test-network-ipv6:
+	@$(MAKE) -s test-network-mkdir
+	@$(MAKE) -s test-network-clean
+	@$(MAKE) -s test-network-run \
+	  ipv6_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6)" \
+	  ipv6_mixed_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6_MIXED)"
+	@$(MAKE) -s test-network-results
+
+# Make the test network log directory, if it does not exist
+.PHONY: test-network-mkdir
+test-network-mkdir:
+	@mkdir -p "$(TEST_NETWORK_ALL_LOG_DIR)"
+
+# Clean the test network log directory
+.PHONY: test-network-clean
+# We need to remove all matching files, so we can't quote the glob part of the
+# rm arguments
+test-network-clean:
+	rm -f "$(TEST_NETWORK_ALL_LOG_DIR)"/*.log \
+	      "$(TEST_NETWORK_ALL_LOG_DIR)"/*.trs
+
+# Run tests using automake's test-driver
+# When checking if a set of test can be run, log the type of test, and the
+# list of tests that will be run (or skipped).
+#
+# Run the IPv4 tests in $(ipv4_flavors), unconditionally
+#   - tor relays and directory authorities require IPv4.
+# Run the IPv6 tests in $(ipv6_flavors), if IPv6 is available
+#   - only run IPv6 tests if we can ping6 or ping -6 ::1 (localhost)
+#     we try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
+#     because they're incompatible
+#   - some IPv6 tests may fail without an IPv6 DNS server
+#     (see #16971 and #17011)
+# Run the mixed tests in $(mixed_flavors), if a tor-stable binary is available
+#   - only run mixed tests if we have a tor-stable binary
+# Run the IPv6 mixed tests in $(ipv6_mixed_flavors), if IPv6 and mixed are run
+#   - see above for details about IPv6 and mixed
+.PHONY: test-network-run
+# We need the word splitting in the "for" lines, so we can't quote
+# $(skip_flavors) or $(flavors)
+test-network-run: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
+	@flavors=""; \
+	skip_flavors=""; \
+	if test -n "$(ipv4_flavors)"; then \
+		echo "Running IPv4 flavors: $(ipv4_flavors)."; \
+		flavors="$$flavors $(ipv4_flavors)"; \
 	fi; \
-	if command -v tor-stable >/dev/null 2>&1; then \
-		echo "tor-stable found, running mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
-		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
-	else \
-		echo "tor-stable not found, skipping mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
-		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+	test_network_ipv6=false; \
+	if test -n "$(ipv6_flavors)" || \
+	   test -n "$(ipv6_mixed_flavors)"; then \
+		if ping6 -q -c 1 -o   ::1 >/dev/null 2>&1 || \
+		   ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || \
+		   ping  -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \
+			test_network_ipv6=true; \
+		fi; \
+	fi; \
+	if test -n "$(ipv6_flavors)"; then \
+		if test "$$test_network_ipv6" = "true"; then \
+			echo "ping6 ::1 or ping ::1 succeeded, running IPv6" \
+			  "flavors: $(ipv6_flavors)."; \
+			flavors="$$flavors $(ipv6_flavors)"; \
+		else \
+			echo "ping6 ::1 and ping ::1 failed, skipping IPv6" \
+			  "flavors: $(ipv6_flavors)."; \
+			skip_flavors="$$skip_flavors $(ipv6_flavors)"; \
+		fi; \
+	fi; \
+	test_network_mixed=false; \
+	if test -n "$(mixed_flavors)" || \
+	   test -n "$(ipv6_mixed_flavors)"; then \
+		if command -v tor-stable >/dev/null 2>&1; then \
+			test_network_mixed=true; \
+		fi; \
+	fi; \
+	if test -n "$(mixed_flavors)"; then \
+		if test "$$test_network_mixed" = "true"; then \
+			echo "tor-stable found, running mixed flavors:" \
+			  "$(mixed_flavors)."; \
+			flavors="$$flavors $(mixed_flavors)"; \
+		else \
+			echo "tor-stable not found, skipping mixed flavors:" \
+			  "$(mixed_flavors)."; \
+			skip_flavors="$$skip_flavors $(mixed_flavors)"; \
+		fi; \
+	fi; \
+	if test -n "$(ipv6_mixed_flavors)"; then \
+		if test "$$test_network_ipv6" = "true" && \
+		   test "$$test_network_mixed" = "true"; then \
+			echo "Running IPv6 mixed flavors:" \
+			  "$(ipv6_mixed_flavors)."; \
+			flavors="$$flavors $(ipv6_mixed_flavors)"; \
+		else \
+			echo "Skipping IPv6 mixed flavors:" \
+			  "$(ipv6_mixed_flavors)."; \
+			skip_flavors="$$skip_flavors $(ipv6_mixed_flavors)"; \
+		fi; \
 	fi; \
 	for f in $$skip_flavors; do \
 		echo "SKIP: $$f"; \
 	done; \
 	for f in $$flavors; do \
-		$(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
-		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \
-	done; \
-	echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \
-	! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
+		$(SHELL) "$(top_srcdir)/test-driver" --test-name "$$f" \
+		  --log-file "$(TEST_NETWORK_ALL_LOG_DIR)/$$f.log" \
+		  --trs-file "$(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs" \
+		  $(TEST_NETWORK_ALL_DRIVER_FLAGS) \
+		  "$(top_srcdir)/src/test/test-network.sh" \
+		    --flavor "$$f" $(TEST_NETWORK_FLAGS); \
+		"$(top_srcdir)/src/test/test-network.sh" \
+		  $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \
+	done
+
+# Print the results from automake's test-driver
+#   - show tor warnings on the console after each network run
+#     (otherwise, warnings go to the logs, and people don't see them unless
+#     there is a network failure)
+.PHONY: test-network-results
+# We need to grep all matching files, so we can't quote the glob part of the
+# grep arguments
+test-network-results:
+	@echo \
+	  "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."
+	@! grep -q FAIL "$(TEST_NETWORK_ALL_LOG_DIR)"/*.trs
 
 need-stem-path:
 	@if test ! -d "$$STEM_SOURCE_DIR"; then \
@@ -282,9 +471,19 @@ test-stem: need-stem-path $(TESTING_TOR_BINARY)
 test-stem-full: need-stem-path $(TESTING_TOR_BINARY)
 	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --all --log notice --target RUN_ALL,ONLINE -v;
 
-test-full: need-stem-path need-chutney-path check test-network test-stem
+test-full:					\
+	need-stem-path				\
+	need-chutney-path			\
+	check					\
+	test-network				\
+	test-stem
 
-test-full-online: need-stem-path need-chutney-path check test-network test-stem-full
+test-full-online:				\
+	need-stem-path				\
+	need-chutney-path			\
+	check					\
+	test-network				\
+	test-stem-full
 
 # We can't delete the gcno files, because they are created when tor is compiled
 reset-gcov:
@@ -319,11 +518,12 @@ coverage-html-full: all
 	lcov --remove "$(HTML_COVER_DIR)/lcov.tmp" --rc lcov_branch_coverage=1 'test/*' 'ext/tinytest*' '/usr/*' --output-file "$(HTML_COVER_DIR)/lcov.info"
 	genhtml --branch-coverage -o "$(HTML_COVER_DIR)" "$(HTML_COVER_DIR)/lcov.info"
 
-# Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c,
-# tinytest*.[ch]
-check-spaces:
-if USE_PERL
-	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+# For scripts: avoid src/ext and src/trunnel.
+# Keep these lists consistent:
+#   - OWNED_TOR_C_FILES in Makefile.am
+#   - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook
+#   - try_parse in check_cocci_parse.sh
+OWNED_TOR_C_FILES=\
 		$(top_srcdir)/src/lib/*/*.[ch] \
 		$(top_srcdir)/src/core/*/*.[ch] \
 		$(top_srcdir)/src/feature/*/*.[ch] \
@@ -331,13 +531,29 @@ if USE_PERL
 		$(top_srcdir)/src/test/*.[ch] \
 		$(top_srcdir)/src/test/*/*.[ch] \
 		$(top_srcdir)/src/tools/*.[ch]
+
+check-spaces:
+if USE_PERL
+	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+		$(OWNED_TOR_C_FILES)
 endif
 
 check-includes:
 if USEPYTHON
-	$(PYTHON) $(top_srcdir)/scripts/maint/checkIncludes.py
+	$(PYTHON) $(top_srcdir)/scripts/maint/practracker/includes.py $(top_srcdir)
+endif
+
+check-best-practices:
+if USEPYTHON
+	@$(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py $(top_srcdir) $(TOR_PRACTRACKER_OPTIONS)
 endif
 
+check-cocci:
+	VERBOSE=1 $(top_srcdir)/scripts/coccinelle/check_cocci_parse.sh $(OWNED_TOR_C_FILES)
+
+practracker-regen:
+	$(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py --regen $(top_srcdir)
+
 check-docs: all
 	$(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl
 
@@ -414,17 +630,17 @@ endif
 check-changes:
 if USEPYTHON
 	@if test -d "$(top_srcdir)/changes"; then \
-		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \
+		PACKAGE_VERSION=$(PACKAGE_VERSION) $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \
 		fi
 endif
 
 .PHONY: update-versions
 update-versions:
-	$(PERL) $(top_builddir)/scripts/maint/updateVersions.pl
+	abs_top_srcdir="$(abs_top_srcdir)" $(PYTHON) $(top_srcdir)/scripts/maint/update_versions.py
 
 .PHONY: callgraph
 callgraph:
-	$(top_builddir)/scripts/maint/run_calltool.sh
+	cd $(top_builddir); $(abs_top_srcdir)/scripts/maint/run_calltool.sh
 
 version:
 	@echo "Tor @VERSION@"
@@ -433,6 +649,25 @@ version:
 	   (cd "$(top_srcdir)" && git rev-parse --short=16 HEAD); \
 	fi
 
+.PHONY: autostyle-ifdefs
+autostyle-ifdefs:
+	$(PYTHON) $(top_srcdir)/scripts/maint/annotate_ifdef_directives.py $(OWNED_TOR_C_FILES)
+
+.PHONY: autostyle-ifdefs
+autostyle-operators:
+	$(PERL) $(top_srcdir)/scripts/coccinelle/test-operator-cleanup $(OWNED_TOR_C_FILES)
+
+.PHONY: rectify-includes
+rectify-includes:
+	cd $(top_srcdir); $(PYTHON) $(abs_top_srcdir)/scripts/maint/rectify_include_paths.py
+
+.PHONY: update-copyright
+update-copyright:
+	$(PERL) $(top_srcdir)/scripts/maint/updateCopyright.pl $(OWNED_TOR_C_FILES)
+
+.PHONY: autostyle
+autostyle: update-versions rustfmt autostyle-ifdefs rectify-includes
+
 mostlyclean-local:
 	rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no}
 	rm -rf $(HTML_COVER_DIR)
diff --git a/README b/README
index 9e0f4b8241..9bbf027d70 100644
--- a/README
+++ b/README
@@ -24,9 +24,8 @@ Making applications work with Tor:
 Frequently Asked Questions:
         https://www.torproject.org/docs/faq.html
 
+Release timeline:
+         https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/CoreTorReleases
 
 To get started working on Tor development:
         See the doc/HACKING directory.
-
-Release timeline:
-         https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/CoreTorReleases
diff --git a/ReleaseNotes b/ReleaseNotes
index e0a25a74b7..a572293c07 100644
--- a/ReleaseNotes
+++ b/ReleaseNotes
@@ -2,6 +2,5041 @@ This document summarizes new features and bugfixes in each stable
 release of Tor. If you want to see more detailed descriptions of the
 changes in each development snapshot, see the ChangeLog file.
 
+
+Changes in version 0.4.3.5 - 2020-05-15
+  Tor 0.4.3.5 is the first stable release in the 0.4.3.x series. This
+  series adds support for building without relay code enabled, and
+  implements functionality needed for OnionBalance with v3 onion
+  services. It includes significant refactoring of our configuration and
+  controller functionality, and fixes numerous smaller bugs and
+  performance issues.
+
+  Per our support policy, we support each stable release series for nine
+  months after its first stable release, or three months after the first
+  stable release of the next series: whichever is longer. This means
+  that 0.4.3.x will be supported until around February 2021--later, if
+  0.4.4.x is later than anticipated.
+
+  Note also that support for 0.4.1.x is about to end on May 20 of this
+  year; 0.4.2.x will be supported until September 15. We still plan to
+  continue supporting 0.3.5.x, our long-term stable series, until
+  Feb 2022.
+
+  Below are the changes since 0.4.2.6. For a list of only the changes
+  since 0.4.3.4-rc, see the ChangeLog file.
+
+  o New system requirements:
+    - When building Tor, you now need to have Python 3 in order to run
+      the integration tests. (Python 2 is officially unsupported
+      upstream, as of 1 Jan 2020.) Closes ticket 32608.
+
+  o Major features (build system):
+    - The relay code can now be disabled using the --disable-module-relay
+      configure option. When this option is set, we also disable the
+      dirauth module. Closes ticket 32123.
+    - When Tor is compiled --disable-module-relay, we also omit the code
+      used to act as a directory cache. Closes ticket 32487.
+
+  o Major features (directory authority, ed25519):
+    - Add support for banning a relay's ed25519 keys in the approved-
+      routers file. This will help us migrate away from RSA keys in the
+      future. Previously, only RSA keys could be banned in approved-
+      routers. Resolves ticket 22029. Patch by Neel Chauhan.
+
+  o Major features (onion services):
+    - New control port commands to manage client-side onion service
+      authorization credentials. The ONION_CLIENT_AUTH_ADD command adds
+      a credential, ONION_CLIENT_AUTH_REMOVE deletes a credential, and
+      ONION_CLIENT_AUTH_VIEW lists the credentials. Closes ticket 30381.
+    - Introduce a new SocksPort flag, ExtendedErrors, to support more
+      detailed error codes in information for applications that support
+      them. Closes ticket 30382; implements proposal 304.
+
+  o Major features (proxy):
+    - In addition to its current supported proxy types (HTTP CONNECT,
+      SOCKS4, and SOCKS5), Tor can now make its OR connections through a
+      HAProxy server. A new torrc option was added to specify the
+      address/port of the server: TCPProxy <protocol> <host>:<port>.
+      Currently the only supported protocol for the option is haproxy.
+      Closes ticket 31518. Patch done by Suphanat Chunhapanya (haxxpop).
+
+  o Major bugfixes (security, denial-of-service):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (circuit padding, memory leak):
+    - Avoid a remotely triggered memory leak in the case that a circuit
+      padding machine is somehow negotiated twice on the same circuit.
+      Fixes bug 33619; bugfix on 0.4.0.1-alpha. Found by Tobias Pulls.
+      This is also tracked as TROVE-2020-004 and CVE-2020-10593.
+
+  o Major bugfixes (directory authority):
+    - Directory authorities will now send a 503 (not enough bandwidth)
+      code to clients when under bandwidth pressure. Known relays and
+      other authorities will always be answered regardless of the
+      bandwidth situation. Fixes bug 33029; bugfix on 0.1.2.5-alpha.
+
+  o Major bugfixes (DoS defenses, bridges, pluggable transport):
+    - Fix a bug that was preventing DoS defenses from running on bridges
+      with a pluggable transport. Previously, the DoS subsystem was not
+      given the transport name of the client connection, thus failed to
+      find the GeoIP cache entry for that client address. Fixes bug
+      33491; bugfix on 0.3.3.2-alpha.
+
+  o Major bugfixes (networking):
+    - Correctly handle IPv6 addresses in SOCKS5 RESOLVE_PTR requests,
+      and accept strings as well as binary addresses. Fixes bug 32315;
+      bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (onion service):
+    - Report HS circuit failure back into the HS subsystem so we take
+      appropriate action with regards to the client introduction point
+      failure cache. This improves reachability of onion services, since
+      now clients notice failing introduction circuits properly. Fixes
+      bug 32020; bugfix on 0.3.2.1-alpha.
+
+  o Minor feature (heartbeat, onion service):
+    - Add the DoS INTRODUCE2 defenses counter to the heartbeat DoS
+      message. Closes ticket 31371.
+
+  o Minor feature (sendme, flow control):
+    - Default to sending SENDME version 1 cells. (Clients are already
+      sending these, because of a consensus parameter telling them to do
+      so: this change only affects what clients would do if the
+      consensus didn't contain a recommendation.) Closes ticket 33623.
+
+  o Minor features (best practices tracker):
+    - Practracker now supports a --regen-overbroad option to regenerate
+      the exceptions file, but only to revise exceptions to be _less_
+      tolerant of best-practices violations. Closes ticket 32372.
+
+  o Minor features (configuration validation):
+    - Configuration validation can now be done by per-module callbacks,
+      rather than a global validation function. This will let us reduce
+      the size of config.c and some of its more cumbersome functions.
+      Closes ticket 31241.
+
+  o Minor features (configuration):
+    - If a configured hardware crypto accelerator in AccelName is
+      prefixed with "!", Tor now exits when it cannot be found. Closes
+      ticket 32406.
+    - We now use flag-driven logic to warn about obsolete configuration
+      fields, so that we can include their names. In 0.4.2, we used a
+      special type, which prevented us from generating good warnings.
+      Implements ticket 32404.
+
+  o Minor features (configure, build system):
+    - Output a list of enabled/disabled features at the end of the
+      configure process in a pleasing way. Closes ticket 31373.
+
+  o Minor features (continuous integration):
+    - Run Doxygen Makefile target on Travis, so we can learn about
+      regressions in our internal documentation. Closes ticket 32455.
+    - Stop allowing failures on the Travis CI stem tests job. It looks
+      like all the stem hangs we were seeing before are now fixed.
+      Closes ticket 33075.
+
+  o Minor features (controller):
+    - Add stream isolation data to STREAM event. Closes ticket 19859.
+    - Implement a new GETINFO command to fetch microdescriptor
+      consensus. Closes ticket 31684.
+
+  o Minor features (debugging, directory system):
+    - Don't crash when we find a non-guard with a guard-fraction value
+      set. Instead, log a bug warning, in an attempt to figure out how
+      this happened. Diagnostic for ticket 32868.
+
+  o Minor features (defense in depth):
+    - Add additional checks around tor_vasprintf() usage, in case the
+      function returns an error. Patch by Tobias Stoeckmann. Fixes
+      ticket 31147.
+
+  o Minor features (developer tools):
+    - Remove the 0.2.9.x series branches from git scripts (git-merge-
+      forward.sh, git-pull-all.sh, git-push-all.sh, git-setup-dirs.sh).
+      Closes ticket 32772.
+    - Add a check_cocci_parse.sh script that checks that new code is
+      parseable by Coccinelle. Add an exceptions file for unparseable
+      files, and run the script from travis CI. Closes ticket 31919.
+    - Call the check_cocci_parse.sh script from a 'check-cocci' Makefile
+      target. Closes ticket 31919.
+    - Add a rename_c_identifiers.py tool to rename a bunch of C
+      identifiers at once, and generate a well-formed commit message
+      describing the change. This should help with refactoring. Closes
+      ticket 32237.
+    - Add some scripts in "scripts/coccinelle" to invoke the Coccinelle
+      semantic patching tool with the correct flags. These flags are
+      fairly easy to forget, and these scripts should help us use
+      Coccinelle more effectively in the future. Closes ticket 31705.
+
+  o Minor features (diagnostic):
+    - Improve assertions and add some memory-poisoning code to try to
+      track down possible causes of a rare crash (32564) in the EWMA
+      code. Closes ticket 33290.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors from relays running
+      Tor versions from the 0.2.9 and 0.4.0 series. The 0.3.5 series is
+      still allowed. Resolves ticket 32672. Patch by Neel Chauhan.
+
+  o Minor features (Doxygen):
+    - Update Doxygen configuration file to a more recent template (from
+      1.8.15). Closes ticket 32110.
+    - "make doxygen" now works with out-of-tree builds. Closes
+      ticket 32113.
+    - Make sure that doxygen outputs documentation for all of our C
+      files. Previously, some were missing @file declarations, causing
+      them to be ignored. Closes ticket 32307.
+    - Our "make doxygen" target now respects --enable-fatal-warnings by
+      default, and does not warn about items that are missing
+      documentation. To warn about missing documentation, run configure
+      with the "--enable-missing-doc-warnings" flag: doing so suspends
+      fatal warnings for doxygen. Closes ticket 32385.
+
+  o Minor features (git scripts):
+    - Add TOR_EXTRA_CLONE_ARGS to git-setup-dirs.sh for git clone
+      customisation. Closes ticket 32347.
+    - Add git-setup-dirs.sh, which sets up an upstream git repository
+      and worktrees for tor maintainers. Closes ticket 29603.
+    - Add TOR_EXTRA_REMOTE_* to git-setup-dirs.sh for a custom extra
+      remote. Closes ticket 32347.
+    - Call the check_cocci_parse.sh script from the git commit and push
+      hooks. Closes ticket 31919.
+    - Make git-push-all.sh skip unchanged branches when pushing to
+      upstream. The script already skipped unchanged test branches.
+      Closes ticket 32216.
+    - Make git-setup-dirs.sh create a master symlink in the worktree
+      directory. Closes ticket 32347.
+    - Skip unmodified source files when doing some existing git hook
+      checks. Related to ticket 31919.
+
+  o Minor features (IPv6, client):
+    - Make Tor clients tell dual-stack exits that they prefer IPv6
+      connections. This change is equivalent to setting the PreferIPv6
+      flag on SOCKSPorts (and most other listener ports). Tor Browser
+      has been setting this flag for some time, and we want to remove a
+      client distinguisher at exits. Closes ticket 32637.
+
+  o Minor features (portability, android):
+    - When building for Android, disable some tests that depend on $HOME
+      and/or pwdb, which Android doesn't have. Closes ticket 32825.
+      Patch from Hans-Christoph Steiner.
+
+  o Minor features (relay modularity):
+    - Split the relay and server pluggable transport config code into
+      separate files in the relay module. Disable this code when the
+      relay module is disabled. Closes part of ticket 32213.
+    - When the relay module is disabled, reject attempts to set the
+      ORPort, DirPort, DirCache, BridgeRelay, ExtORPort, or
+      ServerTransport* options, rather than ignoring the values of these
+      options. Closes part of ticket 32213.
+    - When the relay module is disabled, change the default config so
+      that DirCache is 0, and ClientOnly is 1. Closes ticket 32410.
+
+  o Minor features (release tools):
+    - Port our ChangeLog formatting and sorting tools to Python 3.
+      Closes ticket 32704.
+
+  o Minor features (testing):
+    - The unit tests now support a "TOR_SKIP_TESTCASES" environment
+      variable to specify a list of space-separated test cases that
+      should not be executed. We will use this to disable certain tests
+      that are failing on Appveyor because of mismatched OpenSSL
+      libraries. Part of ticket 33643.
+    - Detect some common failure cases for test_parseconf.sh in
+      src/test/conf_failures. Closes ticket 32451.
+    - Allow test_parseconf.sh to test expected log outputs for successful
+      configs, as well as failed configs. Closes ticket 32451.
+    - The test_parseconf.sh script now supports result variants for any
+      combination of the optional libraries lzma, nss, and zstd. Closes
+      ticket 32397.
+    - When running the unit tests on Android, create temporary files in
+      a subdirectory of /data/local/tmp. Closes ticket 32172. Based on a
+      patch from Hans-Christoph Steiner.
+
+  o Minor features (usability):
+    - Include more information when failing to parse a configuration
+      value. This should make it easier to tell what's going wrong when
+      a configuration file doesn't parse. Closes ticket 33460.
+
+  o Minor bugfix (relay, configuration):
+    - Warn if the ContactInfo field is not set, and tell the relay
+      operator that not having a ContactInfo field set might cause their
+      relay to get rejected in the future. Fixes bug 33361; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (bridges):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (build system):
+    - Fix "make autostyle" for out-of-tree builds. Fixes bug 32370;
+      bugfix on 0.4.1.2-alpha.
+
+  o Minor bugfixes (compiler compatibility):
+    - Avoid compiler warnings from Clang 10 related to the use of GCC-
+      style "/* falls through */" comments. Both Clang and GCC allow
+      __attribute__((fallthrough)) instead, so that's what we're using
+      now. Fixes bug 34078; bugfix on 0.3.1.3-alpha.
+    - Fix compilation warnings with GCC 10.0.1. Fixes bug 34077; bugfix
+      on 0.4.0.3-alpha.
+
+  o Minor bugfixes (configuration handling):
+    - Make control_event_conf_changed() take in a config_line_t instead
+      of a smartlist of alternating key/value entries. Fixes bug 31531;
+      bugfix on 0.2.3.3-alpha. Patch by Neel Chauhan.
+    - Check for multiplication overflow when parsing memory units inside
+      configuration. Fixes bug 30920; bugfix on 0.0.9rc1.
+    - When dumping the configuration, stop adding a trailing space after
+      the option name when there is no option value. This issue only
+      affects options that accept an empty value or list. (Most options
+      reject empty values, or delete the entire line from the dumped
+      options.) Fixes bug 32352; bugfix on 0.0.9pre6.
+    - Avoid changing the user's value of HardwareAccel as stored by
+      SAVECONF, when AccelName is set but HardwareAccel is not. Fixes
+      bug 32382; bugfix on 0.2.2.1-alpha.
+    - When creating a KeyDirectory with the same location as the
+      DataDirectory (not recommended), respect the DataDirectory's
+      group-readable setting if one has not been set for the
+      KeyDirectory. Fixes bug 27992; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (continuous integration):
+    - Remove the buggy and unused mirroring job. Fixes bug 33213; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - When receiving "ACTIVE" or "DORMANT" signals on the control port,
+      report them as SIGNAL events. Previously we would log a bug
+      warning. Fixes bug 33104; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (controller):
+    - In routerstatus_has_changed(), check all the fields that are
+      output over the control port. Fixes bug 20218; bugfix
+      on 0.1.1.11-alpha.
+
+  o Minor bugfixes (developer tools):
+    - Allow paths starting with ./ in scripts/add_c_file.py. Fixes bug
+      31336; bugfix on 0.4.1.2-alpha.
+
+  o Minor bugfixes (dirauth module):
+    - Split the dirauth config code into a separate file in the dirauth
+      module. Disable this code when the dirauth module is disabled.
+      Closes ticket 32213.
+    - When the dirauth module is disabled, reject attempts to set the
+      AuthoritativeDir option, rather than ignoring the value of the
+      option. Fixes bug 32213; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (embedded Tor):
+    - When starting Tor any time after the first time in a process,
+      register the thread in which it is running as the main thread.
+      Previously, we only did this on Windows, which could lead to bugs
+      like 23081 on non-Windows platforms. Fixes bug 32884; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (git scripts):
+    - Avoid sleeping before the last push in git-push-all.sh. Closes
+      ticket 32216.
+    - Forward all unrecognised arguments in git-push-all.sh to git push.
+      Closes ticket 32216.
+
+  o Minor bugfixes (key portability):
+    - When reading PEM-encoded key data, tolerate CRLF line-endings even
+      if we are not running on Windows. Previously, non-Windows hosts
+      would reject these line-endings in certain positions, making
+      certain key files hard to move from one host to another. Fixes bug
+      33032; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Stop truncating IPv6 addresses and ports in channel and connection
+      logs. Fixes bug 33918; bugfix on 0.2.4.4-alpha.
+    - Flush stderr, stdout, and file logs during shutdown, if supported
+      by the OS. This change helps make sure that any final logs are
+      recorded. Fixes bug 33087; bugfix on 0.4.1.6.
+    - Stop closing stderr and stdout during shutdown. Closing these file
+      descriptors can hide sanitiser logs. Fixes bug 33087; bugfix
+      on 0.4.1.6.
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+    - When logging a bug, do not say "Future instances of this warning
+      will be silenced" unless we are actually going to silence them.
+      Previously we would say this whenever a BUG() check failed in the
+      code. Fixes bug 33095; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (onion services v2):
+    - Move a series of v2 onion service warnings to protocol-warning
+      level because they can all be triggered remotely by a malformed
+      request. Fixes bug 32706; bugfix on 0.1.1.14-alpha.
+    - When sending the INTRO cell for a v2 Onion Service, look at the
+      failure cache alongside timeout values to check if the intro point
+      is marked as failed. Previously, we only looked at the relay
+      timeout values. Fixes bug 25568; bugfix on 0.2.7.3-rc. Patch by
+      Neel Chauhan.
+
+  o Minor bugfixes (onion services v3):
+    - Remove a BUG() warning that would cause a stack trace if an onion
+      service descriptor was freed while we were waiting for a
+      rendezvous circuit to complete. Fixes bug 28992; bugfix
+      on 0.3.2.1-alpha.
+    - Relax severity of a log message that can appear naturally when
+      decoding onion service descriptors as a relay. Also add some
+      diagnostics to debug any future bugs in that area. Fixes bug
+      31669; bugfix on 0.3.0.1-alpha.
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+    - Properly handle the client rendezvous circuit timeout. Previously
+      Tor would sometimes timeout a rendezvous circuit awaiting the
+      introduction ACK, and find itself unable to re-establish all
+      circuits because the rendezvous circuit timed out too early. Fixes
+      bug 32021; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Do not rely on a "circuit established" flag for intro circuits but
+      instead always query the HS circuit map. This is to avoid sync
+      issue with that flag and the map. Fixes bug 32094; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services, all):
+    - In cancel_descriptor_fetches(), use
+      connection_list_by_type_purpose() instead of
+      connection_list_by_type_state(). Fixes bug 32639; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (pluggable transports):
+    - When receiving a message on standard error from a pluggable
+      transport, log it at info level, rather than as a warning. Fixes
+      bug 33005; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (rust, build):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (scripts):
+    - Fix update_versions.py for out-of-tree builds. Fixes bug 32371;
+      bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Use the same code to find the tor binary in all of our test
+      scripts. This change makes sure we are always using the coverage
+      binary when coverage is enabled. Fixes bug 32368; bugfix
+      on 0.2.7.3-rc.
+    - Stop ignoring "tor --dump-config" errors in test_parseconf.sh.
+      Fixes bug 32468; bugfix on 0.4.2.1-alpha.
+    - Our option-validation tests no longer depend on specially
+      configured non-default, non-passing sets of options. Previously,
+      the tests had been written to assume that options would _not_ be
+      set to their defaults, which led to needless complexity and
+      verbosity. Fixes bug 32175; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (TLS bug handling):
+    - When encountering a bug in buf_read_from_tls(), return a "MISC"
+      error code rather than "WANTWRITE". This change might help avoid
+      some CPU-wasting loops if the bug is ever triggered. Bug reported
+      by opara. Fixes bug 32673; bugfix on 0.3.0.4-alpha.
+
+  o Deprecated features:
+    - Deprecate the ClientAutoIPv6ORPort option. This option was not
+      true "Happy Eyeballs", and often failed on connections that
+      weren't reliably dual-stack. Closes ticket 32942. Patch by
+      Neel Chauhan.
+
+  o Documentation:
+    - Provide a quickstart guide for a Circuit Padding Framework, and
+      documentation for researchers to implement and study circuit
+      padding machines. Closes ticket 28804.
+    - Add documentation in 'HelpfulTools.md' to describe how to build a
+      tag file. Closes ticket 32779.
+    - Create a high-level description of the long-term software
+      architecture goals. Closes ticket 32206.
+    - Describe the --dump-config command in the manual page. Closes
+      ticket 32467.
+    - Unite coding advice from this_not_that.md in torguts repo into our
+      coding standards document. Resolves ticket 31853.
+
+  o Removed features:
+    - Our Doxygen configuration no longer generates LaTeX output. The
+      reference manual produced by doing this was over 4000 pages long,
+      and generally unusable. Closes ticket 32099.
+    - The option "TestingEstimatedDescriptorPropagationTime" is now
+      marked as obsolete. It has had no effect since 0.3.0.7, when
+      clients stopped rejecting consensuses "from the future". Closes
+      ticket 32807.
+    - We no longer support consensus methods before method 28; these
+      methods were only used by authorities running versions of Tor that
+      are now at end-of-life. In effect, this means that clients,
+      relays, and authorities now assume that authorities will be
+      running version 0.3.5.x or later. Closes ticket 32695.
+
+  o Testing:
+    - Avoid conflicts between the fake sockets in tor's unit tests, and
+      real file descriptors. Resolves issues running unit tests with
+      GitHub Actions, where the process that embeds or launches the
+      tests has already opened a large number of file descriptors. Fixes
+      bug 33782; bugfix on 0.2.8.1-alpha. Found and fixed by
+      Putta Khunchalee.
+    - Add more test cases for tor's UTF-8 validation function. Also,
+      check the arguments passed to the function for consistency. Closes
+      ticket 32845.
+    - Improve test coverage for relay and dirauth config code, focusing
+      on option validation and normalization. Closes ticket 32213.
+    - Improve the consistency of test_parseconf.sh output, and run all
+      the tests, even if one fails. Closes ticket 32213.
+    - Run the practracker unit tests in the pre-commit git hook. Closes
+      ticket 32609.
+
+  o Code simplification and refactoring (channel):
+    - Channel layer had a variable length cell handler that was not used
+      and thus removed. Closes ticket 32892.
+
+  o Code simplification and refactoring (configuration):
+    - Immutability is now implemented as a flag on individual
+      configuration options rather than as part of the option-transition
+      checking code. Closes ticket 32344.
+    - Instead of keeping a list of configuration options to check for
+      relative paths, check all the options whose type is "FILENAME".
+      Solves part of ticket 32339.
+    - Our default log (which ordinarily sends NOTICE-level messages to
+      standard output) is now handled in a more logical manner.
+      Previously, we replaced the configured log options if they were
+      empty. Now, we interpret an empty set of log options as meaning
+      "use the default log". Closes ticket 31999.
+    - Remove some unused arguments from the options_validate() function,
+      to simplify our code and tests. Closes ticket 32187.
+    - Simplify the options_validate() code so that it looks at the
+      default options directly, rather than taking default options as an
+      argument. This change lets us simplify its interface. Closes
+      ticket 32185.
+    - Use our new configuration architecture to move most authority-
+      related options to the directory authority module. Closes
+      ticket 32806.
+    - When parsing the command line, handle options that determine our
+      "quiet level" and our mode of operation (e.g., --dump-config and
+      so on) all in one table. Closes ticket 32003.
+
+  o Code simplification and refactoring (controller):
+    - Create a new abstraction for formatting control protocol reply
+      lines based on key-value pairs. Refactor some existing control
+      protocol code to take advantage of this. Closes ticket 30984.
+    - Create a helper function that can fetch network status or
+      microdesc consensuses. Closes ticket 31684.
+
+  o Code simplification and refactoring (dirauth modularization):
+    - Remove the last remaining HAVE_MODULE_DIRAUTH inside a function.
+      Closes ticket 32163.
+    - Replace some confusing identifiers in process_descs.c. Closes
+      ticket 29826.
+    - Simplify some relay and dirauth config code. Closes ticket 32213.
+
+  o Code simplification and refactoring (mainloop):
+    - Simplify the ip_address_changed() function by removing redundant
+      checks. Closes ticket 33091.
+
+  o Code simplification and refactoring (misc):
+    - Make all the structs we declare follow the same naming convention
+      of ending with "_t". Closes ticket 32415.
+    - Move and rename some configuration-related code for clarity.
+      Closes ticket 32304.
+    - Our include.am files are now broken up by subdirectory.
+      Previously, src/core/include.am covered all of the subdirectories
+      in "core", "feature", and "app". Closes ticket 32137.
+    - Remove underused NS*() macros from test code: they make our tests
+      more confusing, especially for code-formatting tools. Closes
+      ticket 32887.
+
+  o Code simplification and refactoring (relay modularization):
+    - Disable relay_periodic when the relay module is disabled. Closes
+      ticket 32244.
+    - Disable relay_sys when the relay module is disabled. Closes
+      ticket 32245.
+
+  o Code simplification and refactoring (tool support):
+    - Add numerous missing dependencies to our include files, so that
+      they can be included in different reasonable orders and still
+      compile. Addresses part of ticket 32764.
+    - Fix some parts of our code that were difficult for Coccinelle to
+      parse. Related to ticket 31705.
+    - Fix some small issues in our code that prevented automatic
+      formatting tools from working. Addresses part of ticket 32764.
+
+  o Documentation (manpage):
+    - Alphabetize the Server and Directory server sections of the tor
+      manpage. Also split Statistics options into their own section of
+      the manpage. Closes ticket 33188. Work by Swati Thacker as part of
+      Google Season of Docs.
+    - Document the __OwningControllerProcess torrc option and specify
+      its polling interval. Resolves issue 32971.
+    - Split "Circuit Timeout" options and "Node Selection" options into
+      their own sections of the tor manpage. Closes tickets 32928 and
+      32929. Work by Swati Thacker as part of Google Season of Docs.
+    - Alphabetize the Client Options section of the tor manpage. Closes
+      ticket 32846.
+    - Alphabetize the General Options section of the tor manpage. Closes
+      ticket 32708.
+    - In the tor(1) manpage, reword and improve formatting of the
+      COMMAND-LINE OPTIONS and DESCRIPTION sections. Closes ticket
+      32277. Based on work by Swati Thacker as part of Google Season
+      of Docs.
+    - In the tor(1) manpage, reword and improve formatting of the FILES,
+      SEE ALSO, and BUGS sections. Closes ticket 32176. Based on work by
+      Swati Thacker as part of Google Season of Docs.
+
+  o Testing (Appveyor CI):
+    - In our Appveyor Windows CI, copy required DLLs to test and app
+      directories, before running tor's tests. This ensures that tor.exe
+      and test*.exe use the correct version of each DLL. This fix is not
+      required, but we hope it will avoid DLL search issues in future.
+      Fixes bug 33673; bugfix on 0.3.4.2-alpha.
+    - On Appveyor, skip the crypto/openssl_version test, which is
+      failing because of a mismatched library installation. Fix
+      for 33643.
+
+  o Testing (circuit, EWMA):
+    - Add unit tests for circuitmux and EWMA subsystems. Closes
+      ticket 32196.
+
+  o Testing (Travis CI):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+      allow_failure), to speed up the build. Closes ticket 33195.
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.4.2.7 - 2020-03-18
+  This is the third stable release in the 0.4.2.x series. It backports
+  numerous fixes from later releases, including a fix for TROVE-2020-
+  002, a major denial-of-service vulnerability that affected all
+  released Tor instances since 0.2.1.5-alpha. Using this vulnerability,
+  an attacker could cause Tor instances to consume a huge amount of CPU,
+  disrupting their operations for several seconds or minutes. This
+  attack could be launched by anybody against a relay, or by a directory
+  cache against any client that had connected to it. The attacker could
+  launch this attack as much as they wanted, thereby disrupting service
+  or creating patterns that could aid in traffic analysis. This issue
+  was found by OSS-Fuzz, and is also tracked as CVE-2020-10592.
+
+  We do not have reason to believe that this attack is currently being
+  exploited in the wild, but nonetheless we advise everyone to upgrade
+  as soon as packages are available.
+
+  o Major bugfixes (security, denial-of-service, backport from 0.4.3.3-alpha):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (circuit padding, memory leak, backport from 0.4.3.3-alpha):
+    - Avoid a remotely triggered memory leak in the case that a circuit
+      padding machine is somehow negotiated twice on the same circuit.
+      Fixes bug 33619; bugfix on 0.4.0.1-alpha. Found by Tobias Pulls.
+      This is also tracked as TROVE-2020-004 and CVE-2020-10593.
+
+  o Major bugfixes (directory authority, backport from 0.4.3.3-alpha):
+    - Directory authorities will now send a 503 (not enough bandwidth)
+      code to clients when under bandwidth pressure. Known relays and
+      other authorities will always be answered regardless of the
+      bandwidth situation. Fixes bug 33029; bugfix on 0.1.2.5-alpha.
+
+  o Minor features (continuous integration, backport from 0.4.3.2-alpha):
+    - Stop allowing failures on the Travis CI stem tests job. It looks
+      like all the stem hangs we were seeing before are now fixed.
+      Closes ticket 33075.
+
+  o Minor bugfixes (bridges, backport from 0.4.3.1-alpha):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.3.2-alpha):
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (onion services v3, backport from 0.4.3.3-alpha):
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+
+  o Minor bugfixes (rust, build, backport from 0.4.3.2-alpha):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Testing (Travis CI, backport from 0.4.3.3-alpha):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+      allow_failure), to speed up the build. Closes ticket 33195.
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.4.1.9 - 2020-03-18
+  Tor 0.4.1.9 backports important fixes from later Tor releases,
+  including a fix for TROVE-2020-002, a major denial-of-service
+  vulnerability that affected all released Tor instances since
+  0.2.1.5-alpha. Using this vulnerability, an attacker could cause Tor
+  instances to consume a huge amount of CPU, disrupting their operations
+  for several seconds or minutes. This attack could be launched by
+  anybody against a relay, or by a directory cache against any client
+  that had connected to it. The attacker could launch this attack as
+  much as they wanted, thereby disrupting service or creating patterns
+  that could aid in traffic analysis. This issue was found by OSS-Fuzz,
+  and is also tracked as CVE-2020-10592.
+
+  We do not have reason to believe that this attack is currently being
+  exploited in the wild, but nonetheless we advise everyone to upgrade
+  as soon as packages are available.
+
+  o Major bugfixes (security, denial-of-service, backport from 0.4.3.3-alpha):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (circuit padding, memory leak, backport from 0.4.3.3-alpha):
+    - Avoid a remotely triggered memory leak in the case that a circuit
+      padding machine is somehow negotiated twice on the same circuit.
+      Fixes bug 33619; bugfix on 0.4.0.1-alpha. Found by Tobias Pulls.
+      This is also tracked as TROVE-2020-004 and CVE-2020-10593.
+
+  o Minor bugfixes (bridges, backport from 0.4.3.1-alpha):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.3.2-alpha):
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (onion services v3, backport from 0.4.3.3-alpha):
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+
+  o Minor bugfixes (rust, build, backport from 0.4.3.2-alpha):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Testing (Travis CI, backport from 0.4.3.3-alpha):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+      allow_failure), to speed up the build. Closes ticket 33195.
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.3.5.10 - 2020-03-18
+  Tor 0.3.5.10 backports many fixes from later Tor releases, including a
+  fix for TROVE-2020-002, a major denial-of-service vulnerability that
+  affected all released Tor instances since 0.2.1.5-alpha. Using this
+  vulnerability, an attacker could cause Tor instances to consume a huge
+  amount of CPU, disrupting their operations for several seconds or
+  minutes. This attack could be launched by anybody against a relay, or
+  by a directory cache against any client that had connected to it. The
+  attacker could launch this attack as much as they wanted, thereby
+  disrupting service or creating patterns that could aid in traffic
+  analysis. This issue was found by OSS-Fuzz, and is also tracked
+  as CVE-2020-10592.
+
+  We do not have reason to believe that this attack is currently being
+  exploited in the wild, but nonetheless we advise everyone to upgrade
+  as soon as packages are available.
+
+  o Major bugfixes (security, denial-of-service, backport from 0.4.3.3-alpha):
+    - Fix a denial-of-service bug that could be used by anyone to
+      consume a bunch of CPU on any Tor relay or authority, or by
+      directories to consume a bunch of CPU on clients or hidden
+      services. Because of the potential for CPU consumption to
+      introduce observable timing patterns, we are treating this as a
+      high-severity security issue. Fixes bug 33119; bugfix on
+      0.2.1.5-alpha. Found by OSS-Fuzz. We are also tracking this issue
+      as TROVE-2020-002 and CVE-2020-10592.
+
+  o Major bugfixes (linux seccomp sandbox, backport from 0.4.3.1-alpha):
+    - Correct how we use libseccomp. Particularly, stop assuming that
+      rules are applied in a particular order or that more rules are
+      processed after the first match. Neither is the case! In
+      libseccomp <2.4.0 this lead to some rules having no effect.
+      libseccomp 2.4.0 changed how rules are generated, leading to a
+      different ordering, which in turn led to a fatal crash during
+      startup. Fixes bug 29819; bugfix on 0.2.5.1-alpha. Patch by
+      Peter Gerber.
+
+  o Minor features (continuous integration, backport from 0.4.3.2-alpha):
+    - Stop allowing failures on the Travis CI stem tests job. It looks
+      like all the stem hangs we were seeing before are now fixed.
+      Closes ticket 33075.
+
+  o Minor bugfixes (bridges, backport from 0.4.3.1-alpha):
+    - Lowercase the configured value of BridgeDistribution before adding
+      it to the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
+
+  o Minor bugfixes (crash, backport from 0.4.2.4-rc):
+    - When running Tor with an option like --verify-config or
+      --dump-config that does not start the event loop, avoid crashing
+      if we try to exit early because of an error. Fixes bug 32407;
+      bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.3.2-alpha):
+    - If we encounter a bug when flushing a buffer to a TLS connection,
+      only log the bug once per invocation of the Tor process.
+      Previously we would log with every occurrence, which could cause
+      us to run out of disk space. Fixes bug 33093; bugfix
+      on 0.3.2.2-alpha.
+
+  o Minor bugfixes (onion services v3, backport from 0.4.3.3-alpha):
+    - Fix an assertion failure that could result from a corrupted
+      ADD_ONION control port command. Found by Saibato. Fixes bug 33137;
+      bugfix on 0.3.3.1-alpha. This issue is also tracked
+      as TROVE-2020-003.
+
+  o Minor bugfixes (rust, build, backport from 0.4.3.2-alpha):
+    - Fix a syntax warning given by newer versions of Rust that was
+      creating problems for our continuous integration. Fixes bug 33212;
+      bugfix on 0.3.5.1-alpha.
+
+  o Testing (backport from 0.4.3.1-alpha):
+    - Re-enable the Travis CI macOS Chutney build, but don't let it
+      prevent the Travis job from finishing. (The Travis macOS jobs are
+      slow, so we don't want to have it delay the whole CI process.)
+      Closes ticket 32629.
+    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on
+      Ubuntu Bionic. Turning off the Sandbox is a work-around, until we
+      fix the sandbox errors in 32722. Closes ticket 32240.
+
+  o Testing (continuous integration, backport from 0.4.3.1-alpha):
+    - Use zstd in our Travis Linux builds. Closes ticket 32242.
+
+  o Testing (Travis CI, backport from 0.4.3.3-alpha):
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed: putting the slowest jobs
+      first takes full advantage of Travis job concurrency. Closes
+      ticket 33194.
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires
+    - When a Travis chutney job fails, use chutney's new "diagnostics.sh"
+      tool to produce detailed diagnostic output. Closes ticket 32792.
+
+
+Changes in version 0.4.2.6 - 2020-01-30
+  This is the second stable release in the 0.4.2.x series. It backports
+  several bugfixes from 0.4.3.1-alpha, including some that had affected
+  the Linux seccomp2 sandbox or Windows services. If you're running with
+  one of those configurations, you'll probably want to upgrade;
+  otherwise, you should be fine with 0.4.2.5.
+
+  o Major bugfixes (linux seccomp sandbox, backport from 0.4.3.1-alpha):
+    - Correct how we use libseccomp. Particularly, stop assuming that
+      rules are applied in a particular order or that more rules are
+      processed after the first match. Neither is the case! In
+      libseccomp <2.4.0 this led to some rules having no effect.
+      libseccomp 2.4.0 changed how rules are generated, leading to a
+      different ordering, which in turn led to a fatal crash during
+      startup. Fixes bug 29819; bugfix on 0.2.5.1-alpha. Patch by
+      Peter Gerber.
+    - Fix crash when reloading logging configuration while the
+      experimental sandbox is enabled. Fixes bug 32841; bugfix on
+      0.4.1.7. Patch by Peter Gerber.
+
+  o Minor bugfixes (correctness checks, backport from 0.4.3.1-alpha):
+    - Use GCC/Clang's printf-checking feature to make sure that
+      tor_assertf() arguments are correctly typed. Fixes bug 32765;
+      bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (logging, crash, backport from 0.4.3.1-alpha):
+    - Avoid a possible crash when trying to log a (fatal) assertion
+      failure about mismatched magic numbers in configuration objects.
+      Fixes bug 32771; bugfix on 0.4.2.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.3.1-alpha):
+    - When TOR_DISABLE_PRACTRACKER is set, do not apply it to the
+      test_practracker.sh script. Doing so caused a test failure. Fixes
+      bug 32705; bugfix on 0.4.2.1-alpha.
+    - When TOR_DISABLE_PRACTRACKER is set, log a notice to stderr when
+      skipping practracker checks. Fixes bug 32705; bugfix
+      on 0.4.2.1-alpha.
+
+  o Minor bugfixes (windows service, backport from 0.4.3.1-alpha):
+    - Initialize the publish/subscribe system when running as a windows
+      service. Fixes bug 32778; bugfix on 0.4.1.1-alpha.
+
+  o Testing (backport from 0.4.3.1-alpha):
+    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on
+      Ubuntu Bionic. Turning off the Sandbox is a work-around, until we
+      fix the sandbox errors in 32722. Closes ticket 32240.
+    - Re-enable the Travis CI macOS Chutney build, but don't let it
+      prevent the Travis job from finishing. (The Travis macOS jobs are
+      slow, so we don't want to have it delay the whole CI process.)
+      Closes ticket 32629.
+
+  o Testing (continuous integration, backport from 0.4.3.1-alpha):
+    - Use zstd in our Travis Linux builds. Closes ticket 32242.
+
+
+Changes in version 0.4.1.8 - 2020-01-30
+  This release backports several bugfixes from later release series,
+  including some that had affected the Linux seccomp2 sandbox or Windows
+  services. If you're running with one of those configurations, you'll
+  probably want to upgrade; otherwise, you should be fine with your
+  current version of 0.4.1.x.
+
+  o Major bugfixes (linux seccomp sandbox, backport from 0.4.3.1-alpha):
+    - Correct how we use libseccomp. Particularly, stop assuming that
+      rules are applied in a particular order or that more rules are
+      processed after the first match. Neither is the case! In
+      libseccomp <2.4.0 this led to some rules having no effect.
+      libseccomp 2.4.0 changed how rules are generated, leading to a
+      different ordering, which in turn led to a fatal crash during
+      startup. Fixes bug 29819; bugfix on 0.2.5.1-alpha. Patch by
+      Peter Gerber.
+    - Fix crash when reloading logging configuration while the
+      experimental sandbox is enabled. Fixes bug 32841; bugfix on
+      0.4.1.7. Patch by Peter Gerber.
+
+  o Minor bugfixes (crash, backport form 0.4.2.4-rc):
+    - When running Tor with an option like --verify-config or
+      --dump-config that does not start the event loop, avoid crashing
+      if we try to exit early because of an error. Fixes bug 32407;
+      bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (windows service, backport from 0.4.3.1-alpha):
+    - Initialize the publish/subscribe system when running as a windows
+      service. Fixes bug 32778; bugfix on 0.4.1.1-alpha.
+
+  o Testing (backport from 0.4.3.1-alpha):
+    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on
+      Ubuntu Bionic. Turning off the Sandbox is a work-around, until we
+      fix the sandbox errors in 32722. Closes ticket 32240.
+    - Re-enable the Travis CI macOS Chutney build, but don't let it
+      prevent the Travis job from finishing. (The Travis macOS jobs are
+      slow, so we don't want to have it delay the whole CI process.)
+      Closes ticket 32629.
+
+  o Testing (continuous integration, backport from 0.4.3.1-alpha):
+    - Use zstd in our Travis Linux builds. Closes ticket 32242.
+
+
+Changes in version 0.4.2.5 - 2019-12-09
+  This is the first stable release in the 0.4.2.x series. This series
+  improves reliability and stability, and includes several stability and
+  correctness improvements for onion services. It also fixes many smaller
+  bugs present in previous series.
+
+  Per our support policy, we will support the 0.4.2.x series for nine
+  months, or until three months after the release of a stable 0.4.3.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.1.4-rc. For a complete list of only
+  the changes since 0.4.2.4-rc, see the ChangeLog file.
+
+  o Major features (directory authorities):
+    - Directory authorities now reject relays running all currently
+      deprecated release series. The currently supported release series
+      are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549.
+
+  o Major features (onion service v3, denial of service):
+    - Add onion service introduction denial of service defenses. Intro
+      points can now rate-limit client introduction requests, using
+      parameters that can be sent by the service within the
+      ESTABLISH_INTRO cell. If the cell extension for this is not used,
+      the intro point will honor the consensus parameters. Closes
+      ticket 30924.
+
+  o Major bugfixes (circuit build, guard):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close.
+      Previously we could end up in the situation where a subsystem is
+      notified of a circuit opening, but the circuit is still marked for
+      close, leading to undesirable behavior. Fixes bug 30871; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (crash, Linux, Android):
+    - Tolerate systems (including some Android installations) where
+      madvise and MADV_DONTDUMP are available at build-time, but not at
+      run time. Previously, these systems would notice a failed syscall
+      and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha.
+    - Tolerate systems (including some Linux installations) where
+      madvise and/or MADV_DONTFORK are available at build-time, but not
+      at run time. Previously, these systems would notice a failed
+      syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (embedded Tor):
+    - Avoid a possible crash when restarting Tor in embedded mode and
+      enabling a different set of publish/subscribe messages. Fixes bug
+      31898; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (relay):
+    - Relays now respect their AccountingMax bandwidth again. When
+      relays entered "soft" hibernation (which typically starts when
+      we've hit 90% of our AccountingMax), we had stopped checking
+      whether we should enter hard hibernation. Soft hibernation refuses
+      new connections and new circuits, but the existing circuits can
+      continue, meaning that relays could have exceeded their configured
+      AccountingMax. Fixes bug 32108; bugfix on 0.4.0.1-alpha.
+
+  o Major bugfixes (torrc parsing):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (v3 onion services):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor feature (onion services, control port):
+    - The ADD_ONION command's keyword "BEST" now defaults to ED25519-V3
+      (v3) onion services. Previously it defaulted to RSA1024 (v2).
+      Closes ticket 29669.
+
+  o Minor features (auto-formatting scripts):
+    - When annotating C macros, never generate a line that our check-
+      spaces script would reject. Closes ticket 31759.
+    - When annotating C macros, try to remove cases of double-negation.
+      Closes ticket 31779.
+
+  o Minor features (best practices tracker):
+    - Our best-practices tracker now integrates with our include-checker
+      tool to keep track of how many layering violations we have not yet
+      fixed. We hope to reduce this number over time to improve Tor's
+      modularity. Closes ticket 31176.
+    - Add a TOR_PRACTRACKER_OPTIONS variable for passing arguments to
+      practracker from the environment. We may want this for continuous
+      integration. Closes ticket 31309.
+    - Give a warning rather than an error when a practracker exception
+      is violated by a small amount, add a --list-overbroad option to
+      practracker that lists exceptions that are stricter than they need
+      to be, and provide an environment variable for disabling
+      practracker. Closes ticket 30752.
+    - Our best-practices tracker now looks at headers as well as C
+      files. Closes ticket 31175.
+
+  o Minor features (build system):
+    - Make pkg-config use --prefix when cross-compiling, if
+      PKG_CONFIG_PATH is not set. Closes ticket 32191.
+    - Add --disable-manpage and --disable-html-manual options to
+      configure script. This will enable shortening build times by not
+      building documentation. Resolves issue 19381.
+
+  o Minor features (compilation):
+    - Log a more useful error message when we are compiling and one of
+      the compile-time hardening options we have selected can be linked
+      but not executed. Closes ticket 27530.
+
+  o Minor features (configuration):
+    - The configuration code has been extended to allow splitting
+      configuration data across multiple objects. Previously, all
+      configuration data needed to be kept in a single object, which
+      tended to become bloated. Closes ticket 31240.
+
+  o Minor features (continuous integration):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+    - When running CI builds on Travis, put some random data in
+      ~/.torrc, to make sure no tests are reading the Tor configuration
+      file from its default location. Resolves issue 30102.
+
+  o Minor features (debugging):
+    - Log a nonfatal assertion failure if we encounter a configuration
+      line whose command is "CLEAR" but which has a nonempty value. This
+      should be impossible, according to the rules of our configuration
+      line parsing. Closes ticket 31529.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Minor features (git hooks):
+    - Our pre-commit git hook now checks for a special file before
+      running practracker, so that practracker only runs on branches
+      that are based on master. Since the pre-push hook calls the pre-
+      commit hook, practracker will also only run before pushes of
+      branches based on master. Closes ticket 30979.
+
+  o Minor features (git scripts):
+    - Add a "--" command-line argument, to separate git-push-all.sh
+      script arguments from arguments that are passed through to git
+      push. Closes ticket 31314.
+    - Add a -r <remote-name> argument to git-push-all.sh, so the script
+      can push test branches to a personal remote. Closes ticket 31314.
+    - Add a -t <test-branch-prefix> argument to git-merge-forward.sh and
+      git-push-all.sh, which makes these scripts create, merge forward,
+      and push test branches. Closes ticket 31314.
+    - Add a -u argument to git-merge-forward.sh, so that the script can
+      re-use existing test branches after a merge failure and fix.
+      Closes ticket 31314.
+    - Add a TOR_GIT_PUSH env var, which sets the default git push
+      command and arguments for git-push-all.sh. Closes ticket 31314.
+    - Add a TOR_PUSH_DELAY variable to git-push-all.sh, which makes the
+      script push master and maint branches with a delay between each
+      branch. These delays trigger the CI jobs in a set order, which
+      should show the most likely failures first. Also make pushes
+      atomic by default, and make the script pass any command-line
+      arguments to git push. Closes ticket 29879.
+    - Call the shellcheck script from the pre-commit hook. Closes
+      ticket 30967.
+    - Skip pushing test branches that are the same as a remote
+      maint/release/master branch in git-push-all.sh by default. Add a
+      -s argument, so git-push-all.sh can push all test branches. Closes
+      ticket 31314.
+
+  o Minor features (IPv6, logging):
+    - Log IPv6 addresses as well as IPv4 addresses when describing
+      routerinfos, routerstatuses, and nodes. Closes ticket 21003.
+
+  o Minor features (maintenance scripts):
+    - Add a Coccinelle script to detect bugs caused by incrementing or
+      decrementing a variable inside a call to log_debug(). Since
+      log_debug() is a macro whose arguments are conditionally
+      evaluated, it is usually an error to do this. One such bug was
+      30628, in which SENDME cells were miscounted by a decrement
+      operator inside a log_debug() call. Closes ticket 30743.
+
+  o Minor features (onion service v3):
+    - Do not allow single hop clients to fetch or post an HS descriptor
+      from an HSDir. Closes ticket 24964.
+
+  o Minor features (onion service):
+    - Disallow single-hop clients at the introduction point. We've
+      removed Tor2web support a while back and single-hop rendezvous
+      attempts are blocked at the relays. This change should remove load
+      off the network from spammy clients. Close ticket 24963.
+
+  o Minor features (onion services v3):
+    - Assist users who try to setup v2 client authorization in v3 onion
+      services by pointing them to the right documentation. Closes
+      ticket 28966.
+
+  o Minor features (stem tests):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor features (testing):
+    - When running tests that attempt to look up hostnames, replace the
+      libc name lookup functions with ones that do not actually touch
+      the network. This way, the tests complete more quickly in the
+      presence of a slow or missing DNS resolver. Closes ticket 31841.
+    - Add a script to invoke "tor --dump-config" and "tor
+      --verify-config" with various configuration options, and see
+      whether tor's resulting configuration or error messages are what
+      we expect. Use it for integration testing of our +Option and
+      /Option flags. Closes ticket 31637.
+    - Improve test coverage for our existing configuration parsing and
+      management API. Closes ticket 30893.
+    - Add integration tests to make sure that practracker gives the
+      outputs we expect. Closes ticket 31477.
+    - The practracker self-tests are now run as part of the Tor test
+      suite. Closes ticket 31304.
+
+  o Minor features (testing, continuous integration):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Minor features (token bucket):
+    - Implement a generic token bucket that uses a single counter, for
+      use in anti-DoS onion service work. Closes ticket 30687.
+
+  o Minor bugfixes (Appveyor continuous integration):
+    - Avoid spurious errors when Appveyor CI fails before the install
+      step. Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (best practices tracker):
+    - Fix a few issues in the best-practices script, including tests,
+      tab tolerance, error reporting, and directory-exclusion logic.
+      Fixes bug 29746; bugfix on 0.4.1.1-alpha.
+    - When running check-best-practices, only consider files in the src
+      subdirectory. Previously we had recursively considered all
+      subdirectories, which made us get confused by the temporary
+      directories made by "make distcheck". Fixes bug 31578; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (build system):
+    - Interpret "--disable-module-dirauth=no" correctly. Fixes bug
+      32124; bugfix on 0.3.4.1-alpha.
+    - Interpret "--with-tcmalloc=no" correctly. Fixes bug 32124; bugfix
+      on 0.2.0.20-rc.
+    - Stop failing when jemalloc is requested, but tcmalloc is not
+      found. Fixes bug 32124; bugfix on 0.3.5.1-alpha.
+    - When pkg-config is not installed, or a library that depends on
+      pkg-config is not found, tell the user what to do to fix the
+      problem. Fixes bug 31922; bugfix on 0.3.1.1-alpha.
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (chutney, makefiles, documentation):
+    - "make test-network-all" now shows the warnings from each test-
+      network.sh run on the console, so developers see new warnings
+      early. We've also improved the documentation for this feature, and
+      renamed a Makefile variable so the code is self-documenting. Fixes
+      bug 30455; bugfix on 0.3.0.4-rc.
+
+  o Minor bugfixes (client, onion service v3):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (code quality):
+    - Fix "make check-includes" so it runs correctly on out-of-tree
+      builds. Fixes bug 31335; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Add more stub functions to fix compilation on Android with link-
+      time optimization when --disable-module-dirauth is used.
+      Previously, these compilation settings would make the compiler
+      look for functions that didn't exist. Fixes bug 31552; bugfix
+      on 0.4.1.1-alpha.
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (configuration):
+    - Invalid floating-point values in the configuration file are now
+      treated as errors in the configuration. Previously, they were
+      ignored and treated as zero. Fixes bug 31475; bugfix on 0.0.1.
+
+  o Minor bugfixes (connections):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix the MAPADDRESS controller command to accept one or more
+      arguments. Previously, it required two or more arguments, and
+      ignored the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (coverity):
+    - Add an assertion when parsing a BEGIN cell so that coverity can be
+      sure that we are not about to dereference a NULL address. Fixes
+      bug 31026; bugfix on 0.2.4.7-alpha. This is CID 1447296.
+    - In our siphash implementation, when building for coverity, use
+      memcpy in place of a switch statement, so that coverity can tell
+      we are not accessing out-of-bounds memory. Fixes bug 31025; bugfix
+      on 0.2.8.1-alpha. This is tracked as CID 1447293 and 1447295.
+    - Fix several coverity warnings from our unit tests. Fixes bug
+      31030; bugfix on 0.2.4.1-alpha, 0.3.2.1-alpha, and 0.4.0.1-alpha.
+
+  o Minor bugfixes (crash):
+    - When running Tor with an option like --verify-config or
+      --dump-config that does not start the event loop, avoid crashing
+      if we try to exit early because of an error. Fixes bug 32407;
+      bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (developer tooling):
+    - Only log git script changes in the post-merge script when the
+      merge was to the master branch. Fixes bug 31040; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Return a distinct status when formatting annotations fails. Fixes
+      bug 30780; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (error handling):
+    - Always lock the backtrace buffer before it is used. Fixes bug
+      31734; bugfix on 0.2.5.3-alpha.
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - When tor aborts due to an error, close log file descriptors before
+      aborting. Closing the logs makes some OSes flush log file buffers,
+      rather than deleting buffered log lines. Fixes bug 31594; bugfix
+      on 0.2.5.2-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (git hooks):
+    - Remove a duplicate call to practracker from the pre-push hook. The
+      pre-push hook already calls the pre-commit hook, which calls
+      practracker. Fixes bug 31462; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (git scripts):
+    - Stop hard-coding the bash path in the git scripts. Some OSes don't
+      have bash in /usr/bin, others have an ancient bash at this path.
+      Fixes bug 30840; bugfix on 0.4.0.1-alpha.
+    - Stop hard-coding the tor master branch name and worktree path in
+      the git scripts. Fixes bug 30841; bugfix on 0.4.0.1-alpha.
+    - Allow git-push-all.sh to be run from any directory. Previously,
+      the script only worked if run from an upstream worktree directory.
+      Closes ticket 31678.
+
+  o Minor bugfixes (guards):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (ipv6):
+    - Check for private IPv6 addresses alongside their IPv4 equivalents
+      when authorities check descriptors. Previously, we only checked
+      for private IPv4 addresses. Fixes bug 31088; bugfix on
+      0.2.3.21-rc. Patch by Neel Chauhan.
+    - When parsing microdescriptors, we should check the IPv6 exit
+      policy alongside IPv4. Previously, we checked both exit policies
+      for only router info structures, while microdescriptors were
+      IPv4-only. Fixes bug 27284; bugfix on 0.2.3.1-alpha. Patch by
+      Neel Chauhan.
+
+  o Minor bugfixes (logging):
+    - Add a missing check for HAVE_PTHREAD_H, because the backtrace code
+      uses mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha.
+    - Disable backtrace signal handlers when shutting down tor. Fixes
+      bug 31614; bugfix on 0.2.5.2-alpha.
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+    - When initialising log domain masks, only set known log domains.
+      Fixes bug 31854; bugfix on 0.2.1.1-alpha.
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+    - Fix a code issue that would have broken our parsing of log domains
+      as soon as we had 33 of them. Fortunately, we still only have 29.
+      Fixes bug 31451; bugfix on 0.4.1.4-rc.
+
+  o Minor bugfixes (logging, protocol violations):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory management):
+    - Stop leaking a small amount of memory in nt_service_install(), in
+      unreachable code. Fixes bug 30799; bugfix on 0.2.0.7-alpha. Patch
+      by Xiaoyin Liu.
+
+  o Minor bugfixes (modules):
+    - Explain what the optional Directory Authority module is, and what
+      happens when it is disabled. Fixes bug 31825; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (multithreading):
+    - Avoid some undefined behaviour when freeing mutexes. Fixes bug
+      31736; bugfix on 0.0.7.
+
+  o Minor bugfixes (networking, IP addresses):
+    - When parsing addresses via Tor's internal DNS lookup API, reject
+      IPv4 addresses in square brackets, and accept IPv6 addresses in
+      square brackets. This change completes the work started in 23082,
+      making address parsing consistent between tor's internal DNS
+      lookup and address parsing APIs. Fixes bug 30721; bugfix
+      on 0.2.1.5-alpha.
+    - When parsing addresses via Tor's internal address:port parsing and
+      DNS lookup APIs, require IPv6 addresses with ports to have square
+      brackets. But allow IPv6 addresses without ports, whether or not
+      they have square brackets. Fixes bug 30721; bugfix
+      on 0.2.1.5-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - When purging the client descriptor cache, close any introduction
+      point circuits associated with purged cache entries. This avoids
+      picking those circuits later when connecting to the same
+      introduction points. Fixes bug 30921; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - In the hs_ident_circuit_t data structure, remove the unused field
+      circuit_type and the respective argument in hs_ident_circuit_new().
+      This field was set by clients (for introduction) and services (for
+      introduction and rendezvous) but was never used afterwards. Fixes
+      bug 31490; bugfix on 0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (operator tools):
+    - Make tor-print-ed-signing-cert(1) print certificate expiration
+      date in RFC 1123 and UNIX timestamp formats, to make output
+      machine readable. Fixes bug 31012; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (process management):
+    - Remove overly strict assertions that triggered when a pluggable
+      transport failed to launch. Fixes bug 31091; bugfix
+      on 0.4.0.1-alpha.
+    - Remove an assertion in the Unix process backend. This assertion
+      would trigger when we failed to find the executable for a child
+      process. Fixes bug 31810; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (rust):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+    - Raise the minimum rustc version to 1.31.0, as checked by configure
+      and CI. Fixes bug 31442; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (sendme, code structure):
+    - Rename the trunnel SENDME file definition from sendme.trunnel to
+      sendme_cell.trunnel to avoid having twice sendme.{c|h} in the
+      repository. Fixes bug 30769; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (statistics):
+    - Stop removing the ed25519 signature if the extra info file is too
+      big. If the signature data was removed, but the keyword was kept,
+      this could result in an unparseable extra info file. Fixes bug
+      30958; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (subsystems):
+    - Make the subsystem init order match the subsystem module
+      dependencies. Call windows process security APIs as early as
+      possible. Initialize logging before network and time, so that
+      network and time can use logging. Fixes bug 31615; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Avoid intermittent test failures due to a test that had relied on
+      inconsistent timing sources. Fixes bug 31995; bugfix
+      on 0.3.1.3-alpha.
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+    - Teach the util/socketpair_ersatz test to work correctly when we
+      have no network stack configured. Fixes bug 30804; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (tests, SunOS):
+    - Avoid a map_anon_nofork test failure due to a signed/unsigned
+      integer comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (tls, logging):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Minor bugfixes (v2 single onion services):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (v3 onion services):
+    - When cleaning up intro circuits for a v3 onion service, don't
+      remove circuits that have an established or pending circuit, even
+      if they ran out of retries. This way, we don't remove a circuit on
+      its last retry. Fixes bug 31652; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (v3 single onion services):
+    - Always retry v3 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v3 single onion services used a 3-hop
+      path when rend circuits were retried after a remote or delayed
+      failure, but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring:
+    - Refactor connection_control_process_inbuf() to reduce the size of
+      a practracker exception. Closes ticket 31840.
+    - Refactor the microdescs_parse_from_string() function into smaller
+      pieces, for better comprehensibility. Closes ticket 31675.
+    - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit
+      tests and fuzzers, rather than using hard-coded values. Closes
+      ticket 31334.
+    - Interface for function `decrypt_desc_layer` cleaned up. Closes
+      ticket 31589.
+
+  o Documentation:
+    - Correct the description of "GuardLifetime". Fixes bug 31189;
+      bugfix on 0.3.0.1-alpha.
+    - Make clear in the man page, in both the bandwidth section and the
+      AccountingMax section, that Tor counts in powers of two, not
+      powers of ten: 1 GByte is 1024*1024*1024 bytes, not one billion
+      bytes. Resolves ticket 32106.
+    - Document the signal-safe logging behaviour in the tor man page.
+      Also add some comments to the relevant functions. Closes
+      ticket 31839.
+    - Explain why we can't destroy the backtrace buffer mutex. Explain
+      why we don't need to destroy the log mutex. Closes ticket 31736.
+    - The Tor source code repository now includes a (somewhat dated)
+      description of Tor's modular architecture, in doc/HACKING/design.
+      This is based on the old "tor-guts.git" repository, which we are
+      adopting and superseding. Closes ticket 31849.
+    - Improve documentation in circuit padding subsystem. Patch by
+      Tobias Pulls. Closes ticket 31113.
+    - Include an example usage for IPv6 ORPort in our sample torrc.
+      Closes ticket 31320; patch from Ali Raheem.
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+  o Removed features:
+    - No longer include recommended package digests in votes as detailed
+      in proposal 301. The RecommendedPackages torrc option is
+      deprecated and will no longer have any effect. "package" lines
+      will still be considered when computing consensuses for consensus
+      methods that include them. (This change has no effect on the list
+      of recommended Tor versions, which is still in use.) Closes
+      ticket 29738.
+    - Remove torctl.in from contrib/dist directory. Resolves
+      ticket 30550.
+
+  o Testing:
+    - Require C99 standards-conforming code in Travis CI, but allow GNU
+      gcc extensions. Also activates clang's -Wtypedef-redefinition
+      warnings. Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+    - Run shellcheck for all non-third-party shell scripts that are
+      shipped with Tor. Closes ticket 29533.
+    - When checking shell scripts, ignore any user-created directories.
+      Closes ticket 30967.
+
+  o Code simplification and refactoring (config handling):
+    - Extract our variable manipulation code from confparse.c to a new
+      lower-level typedvar.h module. Closes ticket 30864.
+    - Lower another layer of object management from confparse.c to a
+      more general tool. Now typed structure members are accessible via
+      an abstract type. Implements ticket 30914.
+    - Move our backend logic for working with configuration and state
+      files into a lower-level library, since it no longer depends on
+      any tor-specific functionality. Closes ticket 31626.
+    - Numerous simplifications in configuration-handling logic: remove
+      duplicated macro definitions, replace magical names with flags,
+      and refactor "TestingTorNetwork" to use the same default-option
+      logic as the rest of Tor. Closes ticket 30935.
+    - Replace our ad-hoc set of flags for configuration variables and
+      configuration variable types with fine-grained orthogonal flags
+      corresponding to the actual behavior we want. Closes ticket 31625.
+
+  o Code simplification and refactoring (misc):
+    - Eliminate some uses of lower-level control reply abstractions,
+      primarily in the onion_helper functions. Closes ticket 30889.
+    - Rework bootstrap tracking to use the new publish-subscribe
+      subsystem. Closes ticket 29976.
+    - Rewrite format_node_description() and router_get_verbose_nickname()
+      to use strlcpy() and strlcat(). The previous implementation used
+      memcpy() and pointer arithmetic, which was error-prone. Closes
+      ticket 31545. This is CID 1452819.
+    - Split extrainfo_dump_to_string() into smaller functions. Closes
+      ticket 30956.
+    - Use the ptrdiff_t type consistently for expressing variable
+      offsets and pointer differences. Previously we incorrectly (but
+      harmlessly) used int and sometimes off_t for these cases. Closes
+      ticket 31532.
+    - Use the subsystems mechanism to manage the main event loop code.
+      Closes ticket 30806.
+    - Various simplifications and minor improvements to the circuit
+      padding machines. Patch by Tobias Pulls. Closes tickets 31112
+      and 31098.
+
+  o Documentation (hard-coded directories):
+    - Improve the documentation for the DirAuthority and FallbackDir
+      torrc options. Closes ticket 30955.
+
+  o Documentation (tor.1 man page):
+    - Fix typo in tor.1 man page: the option is "--help", not "-help".
+      Fixes bug 31008; bugfix on 0.2.2.9-alpha.
+
+  o Testing (continuous integration):
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+
+
+Changes in version 0.4.1.7 - 2019-12-09
+  This release backports several bugfixes to improve stability and
+  correctness.  Anyone experiencing build problems or crashes with 0.4.1.6,
+  including all relays relying on AccountingMax, should upgrade.
+
+  o Major features (directory authorities, backport from 0.4.2.2-alpha):
+    - Directory authorities now reject relays running all currently
+      deprecated release series. The currently supported release series
+      are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549.
+
+  o Major bugfixes (embedded Tor, backport from 0.4.2.2-alpha):
+    - Avoid a possible crash when restarting Tor in embedded mode and
+      enabling a different set of publish/subscribe messages. Fixes bug
+      31898; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.4.2.3-alpha):
+    - Relays now respect their AccountingMax bandwidth again. When
+      relays entered "soft" hibernation (which typically starts when
+      we've hit 90% of our AccountingMax), we had stopped checking
+      whether we should enter hard hibernation. Soft hibernation refuses
+      new connections and new circuits, but the existing circuits can
+      continue, meaning that relays could have exceeded their configured
+      AccountingMax. Fixes bug 32108; bugfix on 0.4.0.1-alpha.
+
+  o Major bugfixes (torrc parsing, backport from 0.4.2.2-alpha):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (v3 onion services, backport from 0.4.2.3-alpha):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.4.2.2-alpha):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (geoip, backport from 0.4.2.5):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Minor bugfixes (Appveyor CI, backport from 0.4.2.2-alpha):
+    - Avoid spurious errors when Appveyor CI fails before the install step.
+      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (client, onion service v3, backport from 0.4.2.4-rc):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (connections, backport from 0.4.2.3-rc):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (error handling, backport from 0.4.2.1-alpha):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - When tor aborts due to an error, close log file descriptors before
+      aborting. Closing the logs makes some OSes flush log file buffers,
+      rather than deleting buffered log lines. Fixes bug 31594; bugfix
+      on 0.2.5.2-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.2-alpha):
+    - Add a missing check for HAVE_PTHREAD_H, because the backtrace code
+      uses mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha.
+    - Disable backtrace signal handlers when shutting down tor. Fixes
+      bug 31614; bugfix on 0.2.5.2-alpha.
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations, backport from 0.4.2.2-alpha):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API, backport from 0.4.2.3-alpha):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (multithreading, backport from 0.4.2.2-alpha):
+    - Avoid some undefined behaviour when freeing mutexes. Fixes bug
+      31736; bugfix on 0.0.7.
+
+  o Minor bugfixes (process management, backport from 0.4.2.3-alpha):
+    - Remove overly strict assertions that triggered when a pluggable
+      transport failed to launch. Fixes bug 31091; bugfix
+      on 0.4.0.1-alpha.
+    - Remove an assertion in the Unix process backend. This assertion
+      would trigger when we failed to find the executable for a child
+      process. Fixes bug 31810; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.4.2.2-alpha):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.2.3-alpha):
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (tests, SunOS, backport from 0.4.2.2-alpha):
+    - Avoid a map_anon_nofork test failure due to a signed/unsigned
+      integer comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (tls, logging, backport from 0.4.2.3-alpha):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Documentation (backport from 0.4.2.2-alpha):
+    - Explain why we can't destroy the backtrace buffer mutex. Explain
+      why we don't need to destroy the log mutex. Closes ticket 31736.
+
+  o Testing (continuous integration, backport from 0.4.2.3-alpha):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Testing (continuous integration, backport from 0.4.2.4-rc):
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+
+  o Testing (continuous integration, backport from 0.4.2.5):
+    - Require C99 standards-conforming code in Travis CI, but allow GNU gcc
+      extensions. Also activates clang's -Wtypedef-redefinition warnings.
+      Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+
+
+Changes in version 0.4.0.6 - 2019-12-09
+  This is the second stable release in the 0.4.0.x series. This release
+  backports several bugfixes to improve stability and correctness.  Anyone
+  experiencing build problems or crashes with 0.4.0.5, including all relays
+  relying on AccountingMax, should upgrade.
+
+  Note that, per our support policy, support for the 0.4.0.x series will end
+  on 2 Feb 2020.  Anyone still running 0.4.0.x should plan to upgrade to the
+  latest stable release, or downgrade to 0.3.5.x, which will get long-term
+  support until 1 Feb 2022.
+
+  o Directory authority changes (backport from 0.4.1.5):
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Major bugfixes (bridges, backport from 0.4.1.2-alpha):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit build, guard, backport from 0.4.1.4-rc):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (Onion service reachability, backport from 0.4.1.3-alpha):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Major bugfixes (onion service v3, backport from 0.4.1.1-alpha):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.4.2.3-alpha):
+    - Relays now respect their AccountingMax bandwidth again. When
+      relays entered "soft" hibernation (which typically starts when
+      we've hit 90% of our AccountingMax), we had stopped checking
+      whether we should enter hard hibernation. Soft hibernation refuses
+      new connections and new circuits, but the existing circuits can
+      continue, meaning that relays could have exceeded their configured
+      AccountingMax. Fixes bug 32108; bugfix on 0.4.0.1-alpha.
+
+  o Major bugfixes (torrc parsing, backport from 0.4.2.2-alpha):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (v3 onion services, backport from 0.4.2.3-alpha):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (compile-time modules, backport from version 0.4.1.1-alpha):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration, backport from 0.4.1.1-alpha):
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+
+  o Minor features (continuous integration, backport from 0.4.1.4-rc):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+
+  o Minor features (continuous integration, backport from 0.4.2.2-alpha):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (fallback directory list, backport from 0.4.1.4-rc):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor features (geoip, backport from 0.4.2.5):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (Appveyor CI, backport from 0.4.2.2-alpha):
+    - Avoid spurious errors when Appveyor CI fails before the install step.
+      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (circuit isolation, backport from 0.4.1.3-alpha):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (circuit padding, backport from 0.4.1.4-rc):
+    - On relays, properly check that a padding machine is absent before
+      logging a warning about it being absent. Fixes bug 30649; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (client, onion service v3, backport from 0.4.2.4-rc):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (clock skew detection, backport from 0.4.1.5):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation warning, backport from 0.4.1.5):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.1.5):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations, backport from 0.4.1.1-alpha):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (configuration, proxies, backport from 0.4.1.2-alpha):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (connections, backport from 0.4.2.3-rc):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.4.1.3-alpha):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (crash on exit, backport from 0.4.1.4-rc):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (directory authorities, backport from 0.4.1.3-alpha):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.4.1.2-alpha):
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (error handling, backport from 0.4.2.1-alpha):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6, backport from 0.4.2.1-alpha):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.1.1-alpha):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.2-alpha):
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations, backport from 0.4.2.2-alpha):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API, backport from 0.4.2.3-alpha):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.1-alpha):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.4-rc):
+    - Fix a trivial memory leak when parsing an invalid value
+      from a download schedule in the configuration. Fixes bug
+      30894; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (NetBSD, backport from 0.4.1.2-alpha):
+    - Fix usage of minherit() on NetBSD and other platforms that define
+      MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug
+      30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell.
+
+  o Minor bugfixes (onion services, backport from 0.4.1.1-alpha):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (out-of-memory handler, backport from 0.4.1.2-alpha):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.4.1.2-alpha):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (process management, backport from 0.4.2.3-alpha):
+    - Remove overly strict assertions that triggered when a pluggable
+      transport failed to launch. Fixes bug 31091; bugfix
+      on 0.4.0.1-alpha.
+    - Remove an assertion in the Unix process backend. This assertion
+      would trigger when we failed to find the executable for a child
+      process. Fixes bug 31810; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.4.2.2-alpha):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.2.3-alpha):
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (tls, logging, backport from 0.4.2.3-alpha):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+  o Testing (backport from 0.4.1.2-alpha):
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+  o Testing (continuous integration, backport from 0.4.1.1-alpha):
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Testing (continuous integration, backport from 0.4.1.5):
+    - In Travis, make stem log a controller trace to the console, and
+      tail stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary. Closes
+      ticket 30694.
+
+  o Testing (continuous integration, backport from 0.4.2.3-alpha):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Testing (continuous integration, backport from 0.4.2.4-rc):
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+
+  o Testing (continuous integration, backport from 0.4.2.5):
+    - Require C99 standards-conforming code in Travis CI, but allow GNU gcc
+      extensions. Also activates clang's -Wtypedef-redefinition warnings.
+      Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+
+
+Changes in version 0.3.5.9 - 2019-12-09
+  Tor 0.3.5.9 backports serveral fixes from later releases, including
+  several that affect bridge users, relay stability, onion services,
+  and much more.
+
+  o Directory authority changes (backport from 0.4.1.5):
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Major bugfixes (bridges, backport from 0.4.1.2-alpha):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit build, guard, backport from 0.4.1.4-rc):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (NSS, relay, backport from 0.4.0.4-rc):
+    - When running with NSS, disable TLS 1.2 ciphersuites that use
+      SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for
+      these ciphersuites don't work -- which caused relays to fail to
+      handshake with one another when these ciphersuites were enabled.
+      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (Onion service reachability, backport from 0.4.1.3-alpha):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Major bugfixes (onion service v3, backport from 0.4.1.1-alpha):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (torrc parsing, backport from 0.4.2.2-alpha):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (v3 onion services, backport from 0.4.2.3-alpha):
+    - Onion services now always use the exact number of intro points
+      configured with the HiddenServiceNumIntroductionPoints option (or
+      fewer if nodes are excluded). Before, a service could sometimes
+      pick more intro points than configured. Fixes bug 31548; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (address selection, backport from 0.4.0.3-alpha):
+    - Treat the subnet 100.64.0.0/10 as public for some purposes;
+      private for others. This subnet is the RFC 6598 (Carrier Grade
+      NAT) IP range, and is deployed by many ISPs as an alternative to
+      RFC 1918 that does not break existing internal networks. Tor now
+      blocks SOCKS and control ports on these addresses and warns users
+      if client ports or ExtORPorts are listening on a RFC 6598 address.
+      Closes ticket 28525. Patch by Neel Chauhan.
+
+  o Minor features (bandwidth authority, backport from 0.4.0.4-rc):
+    - Make bandwidth authorities ignore relays that are reported in the
+      bandwidth file with the flag "vote=0". This change allows us to
+      report unmeasured relays for diagnostic reasons without including
+      their bandwidth in the bandwidth authorities' vote. Closes
+      ticket 29806.
+
+  o Minor features (compile-time modules, backport from version 0.4.1.1-alpha):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration, backport from 0.4.0.4-rc):
+    - On Travis Rust builds, cleanup Rust registry and refrain from
+      caching the "target/" directory to speed up builds. Resolves
+      issue 29962.
+
+  o Minor features (continuous integration, backport from 0.4.0.5):
+    - In Travis, tell timelimit to use stem's backtrace signals, and
+      launch python directly from timelimit, so python receives the
+      signals from timelimit, rather than make. Closes ticket 30117.
+
+  o Minor features (continuous integration, backport from 0.4.1.1-alpha):
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+
+  o Minor features (continuous integration, backport from 0.4.1.4-rc):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+
+  o Minor features (continuous integration, backport from 0.4.2.2-alpha):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (fallback directory list, backport from 0.4.1.4-rc):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor features (geoip, backport from 0.4.2.5):
+    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 32685.
+
+  o Minor features (NSS, diagnostic, backport from 0.4.0.4-rc):
+    - Try to log an error from NSS (if there is any) and a more useful
+      description of our situation if we are using NSS and a call to
+      SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (security, backport from 0.4.0.4-rc):
+    - Verify in more places that we are not about to create a buffer
+      with more than INT_MAX bytes, to avoid possible OOB access in the
+      event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and
+      fixed by Tobias Stoeckmann.
+    - Fix a potential double free bug when reading huge bandwidth files.
+      The issue is not exploitable in the current Tor network because
+      the vulnerable code is only reached when directory authorities
+      read bandwidth files, but bandwidth files come from a trusted
+      source (usually the authorities themselves). Furthermore, the
+      issue is only exploitable in rare (non-POSIX) 32-bit architectures,
+      which are not used by any of the current authorities. Fixes bug
+      30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfix (continuous integration, backport from 0.4.0.4-rc):
+    - Reset coverage state on disk after Travis CI has finished. This
+      should prevent future coverage merge errors from causing the test
+      suite for the "process" subsystem to fail. The process subsystem
+      was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix
+      on 0.2.9.15.
+    - Terminate test-stem if it takes more than 9.5 minutes to run.
+      (Travis terminates the job after 10 minutes of no output.)
+      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (Appveyor CI, backport from 0.4.2.2-alpha):
+    - Avoid spurious errors when Appveyor CI fails before the install step.
+      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.4.0.4-rc):
+    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug
+      29824; bugfix on 0.3.1.1-alpha. This is Coverity warning
+      CID 1444119.
+
+  o Minor bugfixes (circuit isolation, backport from 0.4.1.3-alpha):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (client, onion service v3, backport from 0.4.2.4-rc):
+    - Fix a BUG() assertion that occurs within a very small race window
+      between when a client intro circuit opens and when its descriptor
+      gets cleaned up from the cache. The circuit is now closed early,
+      which will trigger a re-fetch of the descriptor and continue the
+      connection. Fixes bug 28970; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (clock skew detection, backport from 0.4.1.5):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation warning, backport from 0.4.1.5):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.0.2-alpha):
+    - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug
+      29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (compilation, backport from 0.4.1.5):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations, backport from 0.4.1.1-alpha):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (configuration, proxies, backport from 0.4.1.2-alpha):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (connections, backport from 0.4.2.3-rc):
+    - Avoid trying to read data from closed connections, which can cause
+      needless loops in Libevent and infinite loops in Shadow. Fixes bug
+      30344; bugfix on 0.1.1.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.4.1.3-alpha):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (crash on exit, backport from 0.4.1.4-rc):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (directory authorities, backport from 0.4.1.3-alpha):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.4.1.2-alpha):
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (error handling, backport from 0.4.2.1-alpha):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6, backport from 0.4.2.1-alpha):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.0.2-alpha):
+    - Avoid logging that we are relaxing a circuit timeout when that
+      timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.0.3-alpha):
+    - Correct a misleading error message when IPv4Only or IPv6Only is
+      used but the resolved address can not be interpreted as an address
+      of the specified IP version. Fixes bug 13221; bugfix on
+      0.2.3.9-alpha. Patch from Kris Katterjohn.
+    - Log the correct port number for listening sockets when "auto" is
+      used to let Tor pick the port number. Previously, port 0 was
+      logged instead of the actual port number. Fixes bug 29144; bugfix
+      on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+    - Stop logging a BUG() warning when Tor is waiting for exit
+      descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.1.1-alpha):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.2-alpha):
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations, backport from 0.4.2.2-alpha):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (mainloop, periodic events, in-process API, backport from 0.4.2.3-alpha):
+    - Reset the periodic events' "enabled" flag when Tor is shut down
+      cleanly. Previously, this flag was left on, which caused periodic
+      events not to be re-enabled when Tor was relaunched in-process
+      with tor_api.h after a shutdown. Fixes bug 32058; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.1-alpha):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak, backport from 0.4.1.4-rc):
+    - Fix a trivial memory leak when parsing an invalid value
+      from a download schedule in the configuration. Fixes bug
+      30894; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (memory management, backport from 0.4.0.3-alpha):
+    - Refactor the shared random state's memory management so that it
+      actually takes ownership of the shared random value pointers.
+      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (memory management, testing, backport from 0.4.0.3-alpha):
+    - Stop leaking parts of the shared random state in the shared-random
+      unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.1.1-alpha):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (out-of-memory handler, backport from 0.4.1.2-alpha):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.4.1.2-alpha):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (relay, backport from 0.4.2.2-alpha):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.0.5):
+    - Abort on panic in all build profiles, instead of potentially
+      unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (single onion services, backport from 0.4.0.3-alpha):
+    - Allow connections to single onion services to remain idle without
+      being disconnected. Previously, relays acting as rendezvous points
+      for single onion services were mistakenly closing idle rendezvous
+      circuits after 60 seconds, thinking that they were unused
+      directory-fetching circuits that had served their purpose. Fixes
+      bug 29665; bugfix on 0.2.1.26.
+
+  o Minor bugfixes (stats, backport from 0.4.0.3-alpha):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.0.3-alpha):
+    - Downgrade some LOG_ERR messages in the address/* tests to
+      warnings. The LOG_ERR messages were occurring when we had no
+      configured network. We were failing the unit tests, because we
+      backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug
+      29530; bugfix on 0.3.5.8.
+    - Fix our gcov wrapper script to look for object files at the
+      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.0.4-rc):
+    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a
+      recent test-network.sh to use new chutney features in CI. Fixes
+      bug 29703; bugfix on 0.2.9.1-alpha.
+    - Fix a test failure on Windows caused by an unexpected "BUG"
+      warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.4.2.3-alpha):
+    - When testing port rebinding, don't busy-wait for tor to log.
+      Instead, actually sleep for a short time before polling again.
+      Also improve the formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (TLS protocol, backport form 0.4.0.4-rc):
+    - When classifying a client's selection of TLS ciphers, if the
+      client ciphers are not yet available, do not cache the result.
+      Previously, we had cached the unavailability of the cipher list
+      and never looked again, which in turn led us to assume that the
+      client only supported the ancient V1 link protocol. This, in turn,
+      was causing Stem integration tests to stall in some cases. Fixes
+      bug 30021; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (tls, logging, backport from 0.4.2.3-alpha):
+    - Log bugs about the TLS read buffer's length only once, rather than
+      filling the logs with similar warnings. Fixes bug 31939; bugfix
+      on 0.3.0.4-rc.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (Windows, CI, backport from 0.4.0.3-alpha):
+    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit
+      Windows Server 2012 R2 job. The remaining 2 jobs still provide
+      coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set
+      fast_finish, so failed jobs terminate the build immediately. Fixes
+      bug 29601; bugfix on 0.3.5.4-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+  o Testing (backport from 0.4.1.2-alpha):
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+  o Testing (continuous integration, backport from 0.4.1.1-alpha):
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Testing (continuous integration, backport from 0.4.1.5):
+    - In Travis, make stem log a controller trace to the console, and
+      tail stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary. Closes
+      ticket 30694.
+
+  o Testing (continuous integration, backport from 0.4.2.3-alpha):
+    - Disable all but one Travis CI macOS build, to mitigate slow
+      scheduling of Travis macOS jobs. Closes ticket 32177.
+    - Run the chutney IPv6 networks as part of Travis CI. Closes
+      ticket 30860.
+    - Simplify the Travis CI build matrix, and optimise for build time.
+      Closes ticket 31859.
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
+
+  o Testing (continuous integration, backport from 0.4.2.4-rc):
+    - Use Ubuntu Bionic images for our Travis CI builds, so we can get a
+      recent version of coccinelle. But leave chutney on Ubuntu Trusty,
+      until we can fix some Bionic permissions issues (see ticket
+      32240). Related to ticket 31919.
+    - Install the mingw OpenSSL package in Appveyor. This makes sure
+      that the OpenSSL headers and libraries match in Tor's Appveyor
+      builds. (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
+    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
+
+  o Testing (continuous integration, backport from 0.4.2.5):
+    - Require C99 standards-conforming code in Travis CI, but allow GNU gcc
+      extensions. Also activates clang's -Wtypedef-redefinition warnings.
+      Build some jobs with -std=gnu99, and some jobs without.
+      Closes ticket 32500.
+
+
+Changes in version 0.4.1.6 - 2019-09-19
+  This release backports several bugfixes to improve stability and
+  correctness.  Anyone experiencing build problems or crashes with 0.4.1.5,
+  or experiencing reliability issues with single onion services, should
+  upgrade.
+
+  o Major bugfixes (crash, Linux, Android, backport from 0.4.2.1-alpha):
+    - Tolerate systems (including some Android installations) where
+      madvise and MADV_DONTDUMP are available at build-time, but not at
+      run time. Previously, these systems would notice a failed syscall
+      and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha.
+    - Tolerate systems (including some Linux installations) where
+      madvise and/or MADV_DONTFORK are available at build-time, but not
+      at run time. Previously, these systems would notice a failed
+      syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Add more stub functions to fix compilation on Android with link-
+      time optimization when --disable-module-dirauth is used.
+      Previously, these compilation settings would make the compiler
+      look for functions that didn't exist. Fixes bug 31552; bugfix
+      on 0.4.1.1-alpha.
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix the MAPADDRESS controller command to accept one or more
+      arguments. Previously, it required two or more arguments, and ignored
+      the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6, backport from 0.4.2.1-alpha):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (v3 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v3 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v3 single onion services used a 3-hop
+      path when rend circuits were retried after a remote or delayed
+      failure, but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+
+Changes in version 0.4.1.5 - 2019-08-20
+  This is the first stable release in the 0.4.1.x series. This series
+  adds experimental circuit-level padding, authenticated SENDME cells to
+  defend against certain attacks, and several performance improvements
+  to save on CPU consumption. It fixes bugs in bootstrapping and v3
+  onion services. It also includes numerous smaller features and
+  bugfixes on earlier versions.
+
+  Per our support policy, we will support the 0.4.1.x series for nine
+  months, or until three months after the release of a stable 0.4.2.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.0.5. For a list of only the changes
+  since 0.4.1.4-rc, see the ChangeLog file.
+
+  o Directory authority changes:
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Major features (circuit padding):
+    - Onion service clients now add padding cells at the start of their
+      INTRODUCE and RENDEZVOUS circuits, to make those circuits' traffic
+      look more like general purpose Exit traffic. The overhead for this
+      is 2 extra cells in each direction for RENDEZVOUS circuits, and 1
+      extra upstream cell and 10 downstream cells for INTRODUCE
+      circuits. This feature is only enabled when also supported by the
+      circuit's middle node. (Clients may specify fixed middle nodes
+      with the MiddleNodes option, and may force-disable this feature
+      with the CircuitPadding option.) Closes ticket 28634.
+
+  o Major features (code organization):
+    - Tor now includes a generic publish-subscribe message-passing
+      subsystem that we can use to organize intermodule dependencies. We
+      hope to use this to reduce dependencies between modules that don't
+      need to be related, and to generally simplify our codebase. Closes
+      ticket 28226.
+
+  o Major features (controller protocol):
+    - Controller commands are now parsed using a generalized parsing
+      subsystem. Previously, each controller command was responsible for
+      parsing its own input, which led to strange inconsistencies.
+      Closes ticket 30091.
+
+  o Major features (flow control):
+    - Implement authenticated SENDMEs as detailed in proposal 289. A
+      SENDME cell now includes the digest of the traffic that it
+      acknowledges, so that once an end point receives the SENDME, it
+      can confirm the other side's knowledge of the previous cells that
+      were sent, and prevent certain types of denial-of-service attacks.
+      This behavior is controlled by two new consensus parameters: see
+      the proposal for more details. Fixes ticket 26288.
+
+  o Major features (performance):
+    - Our node selection algorithm now excludes nodes in linear time.
+      Previously, the algorithm was quadratic, which could slow down
+      heavily used onion services. Closes ticket 30307.
+
+  o Major features (performance, RNG):
+    - Tor now constructs a fast secure pseudorandom number generator for
+      each thread, to use when performance is critical. This PRNG is
+      based on AES-CTR, using a buffering construction similar to
+      libottery and the (newer) OpenBSD arc4random() code. It
+      outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for
+      small outputs. Although we believe it to be cryptographically
+      strong, we are only using it when necessary for performance.
+      Implements tickets 29023 and 29536.
+
+  o Major bugfixes (bridges):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit build, guard):
+    - On relays, properly check that a padding machine is absent before
+      logging a warning about it being absent. Fixes bug 30649; bugfix
+      on 0.4.0.1-alpha.
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (onion service reachability):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Major bugfixes (onion service v3):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (authenticated SENDME):
+    - Ensure that there is enough randomness on every circuit to prevent
+      an attacker from successfully predicting the hashes they will need
+      to include in authenticated SENDME cells. At a random interval, if
+      we have not sent randomness already, we now leave some extra space
+      at the end of a cell that we can fill with random bytes. Closes
+      ticket 26846.
+
+  o Minor features (circuit padding logging):
+    - Demote noisy client-side warn logs about circuit padding to protocol
+      warnings. Add additional log messages and circuit ID fields to help
+      with bug 30992 and any other future issues.
+
+  o Minor features (circuit padding):
+    - We now use a fast PRNG when scheduling circuit padding. Part of
+      ticket 28636.
+    - Allow the padding machine designer to pick the edges of their
+      histogram instead of trying to compute them automatically using an
+      exponential formula. Resolves some undefined behavior in the case
+      of small histograms and allows greater flexibility on machine
+      design. Closes ticket 29298; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to hold a circuit open until they
+      are done padding it. Closes ticket 28780.
+
+  o Minor features (compile-time modules):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+    - When running coverage builds on Travis, we now set
+      TOR_TEST_RNG_SEED, to avoid RNG-based coverage differences. Part
+      of ticket 28878.
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Minor features (controller):
+    - Add onion service version 3 support to the HSFETCH command.
+      Previously, only version 2 onion services were supported. Closes
+      ticket 25417. Patch by Neel Chauhan.
+
+  o Minor features (debugging):
+    - Introduce tor_assertf() and tor_assertf_nonfatal() to enable
+      logging of additional information during assert failure. Now we
+      can use format strings to include information for trouble
+      shooting. Resolves ticket 29662.
+
+  o Minor features (defense in depth):
+    - In smartlist_remove_keeporder(), set unused pointers to NULL, in
+      case a bug causes them to be used later. Closes ticket 30176.
+      Patch from Tobias Stoeckmann.
+    - Tor now uses a cryptographically strong PRNG even for decisions
+      that we do not believe are security-sensitive. Previously, for
+      performance reasons, we had used a trivially predictable linear
+      congruential generator algorithm for certain load-balancing and
+      statistical sampling decisions. Now we use our fast RNG in those
+      cases. Closes ticket 29542.
+
+  o Minor features (developer tools):
+    - Tor's "practracker" test script now checks for files and functions
+      that seem too long and complicated. Existing overlong functions
+      and files are accepted for now, but should eventually be
+      refactored. Closes ticket 29221.
+    - Add some scripts used for git maintenance to scripts/git. Closes
+      ticket 29391.
+    - Call practracker from pre-push and pre-commit git hooks to let
+      developers know if they made any code style violations. Closes
+      ticket 30051.
+    - Add a script to check that each header has a well-formed and
+      unique guard macro. Closes ticket 29756.
+
+  o Minor features (fallback directory list):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30852.
+    - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30522.
+
+  o Minor features (HTTP tunnel):
+    - Return an informative web page when the HTTPTunnelPort is used as
+      an HTTP proxy. Closes ticket 27821, patch by "eighthave".
+
+  o Minor features (IPv6, v3 onion services):
+    - Make v3 onion services put IPv6 addresses in service descriptors.
+      Before this change, service descriptors only contained IPv4
+      addresses. Implements 26992.
+
+  o Minor features (logging):
+    - Give a more useful assertion failure message if we think we have
+      minherit() but we fail to make a region non-inheritable. Give a
+      compile-time warning if our support for minherit() is incomplete.
+      Closes ticket 30686.
+
+  o Minor features (maintenance):
+    - Add a new "make autostyle" target that developers can use to apply
+      all automatic Tor style and consistency conversions to the
+      codebase. Closes ticket 30539.
+
+  o Minor features (modularity):
+    - The "--disable-module-dirauth" compile-time option now disables
+      even more dirauth-only code. Closes ticket 30345.
+
+  o Minor features (performance):
+    - Use OpenSSL's implementations of SHA3 when available (in OpenSSL
+      1.1.1 and later), since they tend to be faster than tiny-keccak.
+      Closes ticket 28837.
+
+  o Minor features (testing):
+    - The circuitpadding tests now use a reproducible RNG implementation,
+      so that if a test fails, we can learn why. Part of ticket 28878.
+    - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED,
+      to set the RNG seed for tests that use a reproducible RNG. Part of
+      ticket 28878.
+    - When running tests in coverage mode, take additional care to make
+      our coverage deterministic, so that we can accurately track
+      changes in code coverage. Closes ticket 30519.
+    - Tor's unit test code now contains helper functions to replace the
+      PRNG with a deterministic or reproducible version for testing.
+      Previously, various tests implemented this in various ways.
+      Implements ticket 29732.
+    - We now have a script, cov-test-determinism.sh, to identify places
+      where our unit test coverage has become nondeterministic. Closes
+      ticket 29436.
+    - Check that representative subsets of values of `int` and `unsigned
+      int` can be represented by `void *`. Resolves issue 29537.
+
+  o Minor bugfixes (bridge authority):
+    - Bridge authorities now set bridges as running or non-running when
+      about to dump their status to a file. Previously, they set bridges
+      as running in response to a GETINFO command, but those shouldn't
+      modify data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (channel padding statistics):
+    - Channel padding write totals and padding-enabled totals are now
+      counted properly in relay extrainfo descriptors. Fixes bug 29231;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (circuit isolation):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (circuit padding):
+    - Add a "CircuitPadding" torrc option to disable circuit padding.
+      Fixes bug 28693; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to specify that they do not
+      contribute much overhead, and provide consensus flags and torrc
+      options to force clients to only use these low overhead machines.
+      Fixes bug 29203; bugfix on 0.4.0.1-alpha.
+    - Provide a consensus parameter to fully disable circuit padding, to
+      be used in emergency network overload situations. Fixes bug 30173;
+      bugfix on 0.4.0.1-alpha.
+    - The circuit padding subsystem will no longer schedule padding if
+      dormant mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha.
+    - Inspect a circuit-level cell queue before sending padding, to
+      avoid sending padding while too much data is already queued. Fixes
+      bug 29204; bugfix on 0.4.0.1-alpha.
+    - Avoid calling monotime_absolute_usec() in circuit padding machines
+      that do not use token removal or circuit RTT estimation. Fixes bug
+      29085; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (clock skew detection):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compatibility, standards compliance):
+    - Fix a bug that would invoke undefined behavior on certain
+      operating systems when trying to asprintf() a string exactly
+      INT_MAX bytes long. We don't believe this is exploitable, but it's
+      better to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha.
+      Found and fixed by Tobias Stoeckmann.
+
+  o Minor bugfixes (compilation warning):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix on
+      0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds.  Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (configuration, proxies):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (continuous integration):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Teach the controller parser to distinguish an object preceded by
+      an argument list from one without. Previously, it couldn't
+      distinguish an argument list from the first line of a multiline
+      object. Fixes bug 29984; bugfix on 0.2.3.8-alpha.
+
+  o Minor bugfixes (crash on exit):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (developer tooling):
+    - Fix pre-push hook to allow fixup and squash commits when pushing
+      to non-upstream git remote. Fixes bug 30286; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+    - Directory authorities with IPv6 support now always mark themselves
+      as reachable via IPv6. Fixes bug 24338; bugfix on 0.2.4.1-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (documentation):
+    - Improve the documentation for using MapAddress with ".exit". Fixes
+      bug 30109; bugfix on 0.1.0.1-rc.
+    - Improve the monotonic time module and function documentation to
+      explain what "monotonic" actually means, and document some results
+      that have surprised people. Fixes bug 29640; bugfix
+      on 0.2.9.1-alpha.
+    - Use proper formatting when providing an example on quoting options
+      that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc.
+
+  o Minor bugfixes (logging):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+    - Warn operators when the MyFamily option is set but ContactInfo is
+      missing, as the latter should be set too. Fixes bug 25110; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+    - Fix a trivial memory leak when parsing an invalid value from a
+      download schedule in the configuration. Fixes bug 30894; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (NetBSD):
+    - Fix usage of minherit() on NetBSD and other platforms that define
+      MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug
+      30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell.
+
+  o Minor bugfixes (onion services):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+    - When refusing to launch a controller's HSFETCH request because of
+      rate-limiting, respond to the controller with a new response,
+      "QUERY_RATE_LIMITED". Previously, we would log QUERY_NO_HSDIR for
+      this case. Fixes bug 28269; bugfix on 0.3.1.1-alpha. Patch by
+      Neel Chauhan.
+    - When relaunching a circuit to a rendezvous service, mark the
+      circuit as needing high-uptime routers as appropriate. Fixes bug
+      17357; bugfix on 0.1.0.1-rc. Patch by Neel Chauhan.
+    - Stop ignoring IPv6 link specifiers sent to v3 onion services.
+      (IPv6 support for v3 onion services is still incomplete: see
+      ticket 23493 for details.) Fixes bug 23588; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (onion services, performance):
+    - When building circuits to onion services, call tor_addr_parse()
+      less often. Previously, we called tor_addr_parse() in
+      circuit_is_acceptable() even if its output wasn't used. This
+      change should improve performance when building circuits. Fixes
+      bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan.
+
+  o Minor bugfixes (out-of-memory handler):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (performance):
+    - When checking whether a node is a bridge, use a fast check to make
+      sure that its identity is set. Previously, we used a constant-time
+      check, which is not necessary in this case. Fixes bug 30308;
+      bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as
+      well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (probability distributions):
+    - Refactor and improve parts of the probability distribution code
+      that made Coverity complain. Fixes bug 29805; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (python):
+    - Stop assuming that /usr/bin/python3 exists. For scripts that work
+      with python2, use /usr/bin/python. Otherwise, use /usr/bin/env
+      python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha.
+
+  o Minor bugfixes (relay):
+    - When running as a relay, if IPv6Exit is set to 1 while ExitRelay
+      is auto, act as if ExitRelay is 1. Previously, we would ignore
+      IPv6Exit if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on
+      0.3.5.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (static analysis):
+    - Fix several spurious Coverity warnings about the unit tests, to
+      lower our chances of missing real warnings in the future. Fixes
+      bug 30150; bugfix on 0.3.5.1-alpha and various other Tor versions.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including bandwidth usage
+      statistics, GeoIPFile hashes, ServerTransportPlugin lines, and
+      bridge statistics by country in extra-info documents. Fixes bug
+      29018; bugfix on 0.2.4.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Call setrlimit() to disable core dumps in test_bt_cl.c. Previously
+      we used `ulimit -c` in test_bt.sh, which violates POSIX shell
+      compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha.
+    - Fix some incorrect code in the v3 onion service unit tests. Fixes
+      bug 29243; bugfix on 0.3.2.1-alpha.
+    - In the "routerkeys/*" tests, check the return values of mkdir()
+      for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha.
+      Found by Coverity as CID 1444254.
+    - Split test_utils_general() into several smaller test functions.
+      This makes it easier to perform resource deallocation on assert
+      failure, and fixes Coverity warnings CID 1444117 and CID 1444118.
+      Fixes bug 29823; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - Fix a memory leak in tor-resolve that could happen if Tor gave it
+      a malformed SOCKS response. (Memory leaks in tor-resolve don't
+      actually matter, but it's good to fix them anyway.) Fixes bug
+      30151; bugfix on 0.4.0.1-alpha.
+
+  o Code simplification and refactoring:
+    - Abstract out the low-level formatting of replies on the control
+      port. Implements ticket 30007.
+    - Add several assertions in an attempt to fix some Coverity
+      warnings. Closes ticket 30149.
+    - Introduce a connection_dir_buf_add() helper function that checks
+      for compress_state of dir_connection_t and automatically writes a
+      string to directory connection with or without compression.
+      Resolves issue 28816.
+    - Make the base32_decode() API return the number of bytes written,
+      for consistency with base64_decode(). Closes ticket 28913.
+    - Move most relay-only periodic events out of mainloop.c into the
+      relay subsystem. Closes ticket 30414.
+    - Refactor and encapsulate parts of the codebase that manipulate
+      crypt_path_t objects. Resolves issue 30236.
+    - Refactor several places in our code that Coverity incorrectly
+      believed might have memory leaks. Closes ticket 30147.
+    - Remove redundant return values in crypto_format, and the
+      associated return value checks elsewhere in the code. Make the
+      implementations in crypto_format consistent, and remove redundant
+      code. Resolves ticket 29660.
+    - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that
+      it is not a constant-time function. Closes ticket 30309.
+    - Replace hs_desc_link_specifier_t with link_specifier_t, and remove
+      all hs_desc_link_specifier_t-specific code. Fixes bug 22781;
+      bugfix on 0.3.2.1-alpha.
+    - Simplify v3 onion service link specifier handling code. Fixes bug
+      23576; bugfix on 0.3.2.1-alpha.
+    - Split crypto_digest.c into NSS code, OpenSSL code, and shared
+      code. Resolves ticket 29108.
+    - Split control.c into several submodules, in preparation for
+      distributing its current responsibilities throughout the codebase.
+      Closes ticket 29894.
+    - Start to move responsibility for knowing about periodic events to
+      the appropriate subsystems, so that the mainloop doesn't need to
+      know all the periodic events in the rest of the codebase.
+      Implements tickets 30293 and 30294.
+
+  o Documentation:
+    - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md.
+      Closes ticket 30630.
+    - Document how to find git commits and tags for bug fixes in
+      CodingStandards.md. Update some file documentation. Closes
+      ticket 30261.
+
+  o Removed features:
+    - Remove the linux-tor-prio.sh script from contrib/operator-tools
+      directory. Resolves issue 29434.
+    - Remove the obsolete OpenSUSE initscript. Resolves issue 30076.
+    - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves
+      issue 30075.
+
+  o Testing:
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+  o Code simplification and refactoring (shell scripts):
+    - Clean up many of our shell scripts to fix shellcheck warnings.
+      These include autogen.sh (ticket 26069), test_keygen.sh (ticket
+      29062), test_switch_id.sh (ticket 29065), test_rebind.sh (ticket
+      29063), src/test/fuzz/minimize.sh (ticket 30079), test_rust.sh
+      (ticket 29064), torify (ticket 29070), asciidoc-helper.sh (29926),
+      fuzz_multi.sh (30077), fuzz_static_testcases.sh (ticket 29059),
+      nagios-check-tor-authority-cert (ticket 29071),
+      src/test/fuzz/fixup_filenames.sh (ticket 30078), test-network.sh
+      (ticket 29060), test_key_expiration.sh (ticket 30002),
+      zero_length_keys.sh (ticket 29068), and test_workqueue_*.sh
+      (ticket 29067).
+
+  o Testing (chutney):
+    - In "make test-network-all", test IPv6-only v3 single onion
+      services, using the chutney network single-onion-v23-ipv6-md.
+      Closes ticket 27251.
+
+  o Testing (continuous integration):
+    - In Travis, make stem log a controller trace to the console, and tail
+      stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary.
+      Closes ticket 30694.
+
+
+Changes in version 0.4.0.5 - 2019-05-02
+  This is the first stable release in the 0.4.0.x series. It contains
+  improvements for power management and bootstrap reporting, as well as
+  preliminary backend support for circuit padding to prevent some kinds
+  of traffic analysis. It also continues our work in refactoring Tor for
+  long-term maintainability.
+
+  Per our support policy, we will support the 0.4.0.x series for nine
+  months, or until three months after the release of a stable 0.4.1.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.3.5.7. For a complete list of changes
+  since 0.4.0.4-rc, see the ChangeLog file.
+
+  o Major features (battery management, client, dormant mode):
+    - When Tor is running as a client, and it is unused for a long time,
+      it can now enter a "dormant" state. When Tor is dormant, it avoids
+      network and CPU activity until it is reawoken either by a user
+      request or by a controller command. For more information, see the
+      configuration options starting with "Dormant". Implements tickets
+      2149 and 28335.
+    - The client's memory of whether it is "dormant", and how long it
+      has spent idle, persists across invocations. Implements
+      ticket 28624.
+    - There is a DormantOnFirstStartup option that integrators can use
+      if they expect that in many cases, Tor will be installed but
+      not used.
+
+  o Major features (bootstrap reporting):
+    - When reporting bootstrap progress, report the first connection
+      uniformly, regardless of whether it's a connection for building
+      application circuits. This allows finer-grained reporting of early
+      progress than previously possible, with the improvements of ticket
+      27169. Closes tickets 27167 and 27103. Addresses ticket 27308.
+    - When reporting bootstrap progress, treat connecting to a proxy or
+      pluggable transport as separate from having successfully used that
+      proxy or pluggable transport to connect to a relay. Closes tickets
+      27100 and 28884.
+
+  o Major features (circuit padding):
+    - Implement preliminary support for the circuit padding portion of
+      Proposal 254. The implementation supports Adaptive Padding (aka
+      WTF-PAD) state machines for use between experimental clients and
+      relays. Support is also provided for APE-style state machines that
+      use probability distributions instead of histograms to specify
+      inter-packet delay. At the moment, Tor does not provide any
+      padding state machines that are used in normal operation: for now,
+      this feature exists solely for experimentation. Closes
+      ticket 28142.
+
+  o Major features (refactoring):
+    - Tor now uses an explicit list of its own subsystems when
+      initializing and shutting down. Previously, these systems were
+      managed implicitly in various places throughout the codebase.
+      (There may still be some subsystems using the old system.) Closes
+      ticket 28330.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth message and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (NSS, relay):
+    - When running with NSS, disable TLS 1.2 ciphersuites that use
+      SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for
+      these ciphersuites don't work -- which caused relays to fail to
+      handshake with one another when these ciphersuites were enabled.
+      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (windows, startup):
+    - When reading a consensus file from disk, detect whether it was
+      written in text mode, and re-read it in text mode if so. Always
+      write consensus files in binary mode so that we can map them into
+      memory later. Previously, we had written in text mode, which
+      confused us when we tried to map the file on windows. Fixes bug
+      28614; bugfix on 0.4.0.1-alpha.
+
+  o Minor features (address selection):
+    - Treat the subnet 100.64.0.0/10 as public for some purposes;
+      private for others. This subnet is the RFC 6598 (Carrier Grade
+      NAT) IP range, and is deployed by many ISPs as an alternative to
+      RFC 1918 that does not break existing internal networks. Tor now
+      blocks SOCKS and control ports on these addresses and warns users
+      if client ports or ExtORPorts are listening on a RFC 6598 address.
+      Closes ticket 28525. Patch by Neel Chauhan.
+
+  o Minor features (bandwidth authority):
+    - Make bandwidth authorities ignore relays that are reported in the
+      bandwidth file with the flag "vote=0". This change allows us to
+      report unmeasured relays for diagnostic reasons without including
+      their bandwidth in the bandwidth authorities' vote. Closes
+      ticket 29806.
+    - When a directory authority is using a bandwidth file to obtain the
+      bandwidth values that will be included in the next vote, serve
+      this bandwidth file at /tor/status-vote/next/bandwidth. Closes
+      ticket 21377.
+
+  o Minor features (bootstrap reporting):
+    - When reporting bootstrap progress, stop distinguishing between
+      situations where only internal paths are available and situations
+      where external paths are available. Previously, Tor would often
+      erroneously report that it had only internal paths. Closes
+      ticket 27402.
+
+  o Minor features (compilation):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (continuous integration):
+    - On Travis Rust builds, cleanup Rust registry and refrain from
+      caching the "target/" directory to speed up builds. Resolves
+      issue 29962.
+    - Log Python version during each Travis CI job. Resolves
+      issue 28551.
+    - In Travis, tell timelimit to use stem's backtrace signals, and
+      launch python directly from timelimit, so python receives the
+      signals from timelimit, rather than make. Closes ticket 30117.
+
+  o Minor features (controller):
+    - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP.
+      Implements ticket 28843.
+
+  o Minor features (developer tooling):
+    - Check that bugfix versions in changes files look like Tor versions
+      from the versions spec. Warn when bugfixes claim to be on a future
+      release. Closes ticket 27761.
+    - Provide a git pre-commit hook that disallows committing if we have
+      any failures in our code and changelog formatting checks. It is
+      now available in scripts/maint/pre-commit.git-hook. Implements
+      feature 28976.
+    - Provide a git hook script to prevent "fixup!" and "squash!"
+      commits from ending up in the master branch, as scripts/main/pre-
+      push.git-hook. Closes ticket 27993.
+
+  o Minor features (diagnostic):
+    - Add more diagnostic log messages in an attempt to solve the issue
+      of NUL bytes appearing in a microdescriptor cache. Related to
+      ticket 28223.
+
+  o Minor features (directory authority):
+    - When a directory authority is using a bandwidth file to obtain
+      bandwidth values, include the digest of that file in the vote.
+      Closes ticket 26698.
+    - Directory authorities support a new consensus algorithm, under
+      which the family lines in microdescriptors are encoded in a
+      canonical form. This change makes family lines more compressible
+      in transit, and on the client. Closes ticket 28266; implements
+      proposal 298.
+
+  o Minor features (directory authority, relay):
+    - Authorities now vote on a "StaleDesc" flag to indicate that a
+      relay's descriptor is so old that the relay should upload again
+      soon. Relays treat this flag as a signal to upload a new
+      descriptor. This flag will eventually let us remove the
+      'published' date from routerstatus entries, and make our consensus
+      diffs much smaller. Closes ticket 26770; implements proposal 293.
+
+  o Minor features (dormant mode):
+    - Add a DormantCanceledByStartup option to tell Tor that it should
+      treat a startup event as cancelling any previous dormant state.
+      Integrators should use this option with caution: it should only be
+      used if Tor is being started because of something that the user
+      did, and not if Tor is being automatically started in the
+      background. Closes ticket 29357.
+
+  o Minor features (fallback directory mirrors):
+    - Update the fallback whitelist based on operator opt-ins and opt-
+      outs. Closes ticket 24805, patch by Phoul.
+
+  o Minor features (FreeBSD):
+    - On FreeBSD-based systems, warn relay operators if the
+      "net.inet.ip.random_id" sysctl (IP ID randomization) is disabled.
+      Closes ticket 28518.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29992.
+
+  o Minor features (HTTP standards compliance):
+    - Stop sending the header "Content-type: application/octet-stream"
+      along with transparently compressed documents: this confused
+      browsers. Closes ticket 28100.
+
+  o Minor features (IPv6):
+    - We add an option ClientAutoIPv6ORPort, to make clients randomly
+      prefer a node's IPv4 or IPv6 ORPort. The random preference is set
+      every time a node is loaded from a new consensus or bridge config.
+      We expect that this option will enable clients to bootstrap more
+      quickly without having to determine whether they support IPv4,
+      IPv6, or both. Closes ticket 27490. Patch by Neel Chauhan.
+    - When using addrs_in_same_network_family(), avoid choosing circuit
+      paths that pass through the same IPv6 subnet more than once.
+      Previously, we only checked IPv4 subnets. Closes ticket 24393.
+      Patch by Neel Chauhan.
+
+  o Minor features (log messages):
+    - Improve log message in v3 onion services that could print out
+      negative revision counters. Closes ticket 27707. Patch
+      by "ffmancera".
+
+  o Minor features (memory usage):
+    - Save memory by storing microdescriptor family lists with a more
+      compact representation. Closes ticket 27359.
+    - Tor clients now use mmap() to read consensus files from disk, so
+      that they no longer need keep the full text of a consensus in
+      memory when parsing it or applying a diff. Closes ticket 27244.
+
+  o Minor features (NSS, diagnostic):
+    - Try to log an error from NSS (if there is any) and a more useful
+      description of our situation if we are using NSS and a call to
+      SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241.
+
+  o Minor features (parsing):
+    - Directory authorities now validate that router descriptors and
+      ExtraInfo documents are in a valid subset of UTF-8, and reject
+      them if they are not. Closes ticket 27367.
+
+  o Minor features (performance):
+    - Cache the results of summarize_protocol_flags(), so that we don't
+      have to parse the same protocol-versions string over and over.
+      This should save us a huge number of malloc calls on startup, and
+      may reduce memory fragmentation with some allocators. Closes
+      ticket 27225.
+    - Remove a needless memset() call from get_token_arguments, thereby
+      speeding up the tokenization of directory objects by about 20%.
+      Closes ticket 28852.
+    - Replace parse_short_policy() with a faster implementation, to
+      improve microdescriptor parsing time. Closes ticket 28853.
+    - Speed up directory parsing a little by avoiding use of the non-
+      inlined strcmp_len() function. Closes ticket 28856.
+    - Speed up microdescriptor parsing by about 30%, to help improve
+      startup time. Closes ticket 28839.
+
+  o Minor features (pluggable transports):
+    - Add support for emitting STATUS updates to Tor's control port from
+      a pluggable transport process. Closes ticket 28846.
+    - Add support for logging to Tor's logging subsystem from a
+      pluggable transport process. Closes ticket 28180.
+
+  o Minor features (process management):
+    - Add a new process API for handling child processes. This new API
+      allows Tor to have bi-directional communication with child
+      processes on both Unix and Windows. Closes ticket 28179.
+    - Use the subsystem manager to initialize and shut down the process
+      module. Closes ticket 28847.
+
+  o Minor features (relay):
+    - When listing relay families, list them in canonical form including
+      the relay's own identity, and try to give a more useful set of
+      warnings. Part of ticket 28266 and proposal 298.
+
+  o Minor features (required protocols):
+    - Before exiting because of a missing required protocol, Tor will
+      now check the publication time of the consensus, and not exit
+      unless the consensus is newer than the Tor program's own release
+      date. Previously, Tor would not check the consensus publication
+      time, and so might exit because of a missing protocol that might
+      no longer be required in a current consensus. Implements proposal
+      297; closes ticket 27735.
+
+  o Minor features (testing):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+    - Allow a HeartbeatPeriod of less than 30 minutes in testing Tor
+      networks. Closes ticket 28840. Patch by Rob Jansen.
+    - Use the approx_time() function when setting the "Expires" header
+      in directory replies, to make them more testable. Needed for
+      ticket 30001.
+
+  o Minor bugfixes (security):
+    - Fix a potential double free bug when reading huge bandwidth files.
+      The issue is not exploitable in the current Tor network because
+      the vulnerable code is only reached when directory authorities
+      read bandwidth files, but bandwidth files come from a trusted
+      source (usually the authorities themselves). Furthermore, the
+      issue is only exploitable in rare (non-POSIX) 32-bit architectures,
+      which are not used by any of the current authorities. Fixes bug
+      30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by
+      Tobias Stoeckmann.
+    - Verify in more places that we are not about to create a buffer
+      with more than INT_MAX bytes, to avoid possible OOB access in the
+      event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and
+      fixed by Tobias Stoeckmann.
+
+  o Minor bugfix (continuous integration):
+    - Reset coverage state on disk after Travis CI has finished. This
+      should prevent future coverage merge errors from causing the test
+      suite for the "process" subsystem to fail. The process subsystem
+      was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix
+      on 0.2.9.15.
+    - Terminate test-stem if it takes more than 9.5 minutes to run.
+      (Travis terminates the job after 10 minutes of no output.)
+      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (build, compatibility, rust):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug
+      29824; bugfix on 0.3.1.1-alpha. This is Coverity warning
+      CID 1444119.
+
+  o Minor bugfixes (client, clock skew):
+    - Bootstrap successfully even when Tor's clock is behind the clocks
+      on the authorities. Fixes bug 28591; bugfix on 0.2.0.9-alpha.
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix compilation warnings in test_circuitpadding.c. Fixes bug
+      29169; bugfix on 0.4.0.1-alpha.
+    - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug
+      29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn.
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (directory clients):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Even when a directory mirror's clock is behind the clocks on the
+      authorities, we now allow the mirror to serve "future"
+      consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (DNS):
+    - Gracefully handle an empty or absent resolve.conf file by falling
+      back to using "localhost" as a DNS server (and hoping it works).
+      Previously, we would just stop running as an exit. Fixes bug
+      21900; bugfix on 0.2.1.10-alpha.
+
+  o Minor bugfixes (documentation):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (guards):
+    - In count_acceptable_nodes(), the minimum number is now one bridge
+      or guard node, and two non-guard nodes for a circuit. Previously,
+      we had added up the sum of all nodes with a descriptor, but that
+      could cause us to build failing circuits when we had either too
+      many bridges or not enough guard nodes. Fixes bug 25885; bugfix on
+      0.2.3.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (IPv6):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (linux seccomp sandbox):
+    - Fix startup crash when experimental sandbox support is enabled.
+      Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber.
+
+  o Minor bugfixes (logging):
+    - Correct a misleading error message when IPv4Only or IPv6Only is
+      used but the resolved address can not be interpreted as an address
+      of the specified IP version. Fixes bug 13221; bugfix on
+      0.2.3.9-alpha. Patch from Kris Katterjohn.
+    - Log the correct port number for listening sockets when "auto" is
+      used to let Tor pick the port number. Previously, port 0 was
+      logged instead of the actual port number. Fixes bug 29144; bugfix
+      on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+    - Stop logging a BUG() warning when Tor is waiting for exit
+      descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha.
+    - Avoid logging that we are relaxing a circuit timeout when that
+      timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha.
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (memory management):
+    - Refactor the shared random state's memory management so that it
+      actually takes ownership of the shared random value pointers.
+      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
+    - Stop leaking parts of the shared random state in the shared-random
+      unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (misc):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (networking):
+    - Introduce additional checks into tor_addr_parse() to reject
+      certain incorrect inputs that previously were not detected. Fixes
+      bug 23082; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (onion service v3, client):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (onion services):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (periodic events):
+    - Refrain from calling routerlist_remove_old_routers() from
+      check_descriptor_callback(). Instead, create a new hourly periodic
+      event. Fixes bug 27929; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Make sure that data is continously read from standard output and
+      standard error pipes of a pluggable transport child-process, to
+      avoid deadlocking when a pipe's buffer is full. Fixes bug 26360;
+      bugfix on 0.2.3.6-alpha.
+
+  o Minor bugfixes (rust):
+    - Abort on panic in all build profiles, instead of potentially
+      unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (scheduler):
+    - When re-adding channels to the pending list, check the correct
+      channel's sched_heap_idx. This issue has had no effect in mainline
+      Tor, but could have led to bugs down the road in improved versions
+      of our circuit scheduling code. Fixes bug 29508; bugfix
+      on 0.3.2.10.
+
+  o Minor bugfixes (shellcheck):
+    - Look for scripts in their correct locations during "make
+      shellcheck". Previously we had looked in the wrong place during
+      out-of-tree builds. Fixes bug 30263; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (single onion services):
+    - Allow connections to single onion services to remain idle without
+      being disconnected. Previously, relays acting as rendezvous points
+      for single onion services were mistakenly closing idle rendezvous
+      circuits after 60 seconds, thinking that they were unused
+      directory-fetching circuits that had served their purpose. Fixes
+      bug 29665; bugfix on 0.2.1.26.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a
+      recent test-network.sh to use new chutney features in CI. Fixes
+      bug 29703; bugfix on 0.2.9.1-alpha.
+    - Fix a test failure on Windows caused by an unexpected "BUG"
+      warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix
+      on 0.2.9.3-alpha.
+    - Downgrade some LOG_ERR messages in the address/* tests to
+      warnings. The LOG_ERR messages were occurring when we had no
+      configured network. We were failing the unit tests, because we
+      backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug
+      29530; bugfix on 0.3.5.8.
+    - Fix our gcov wrapper script to look for object files at the
+      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
+    - Decrease the false positive rate of stochastic probability
+      distribution tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha.
+    - Fix intermittent failures on an adaptive padding test. Fixes one
+      case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Disable an unstable circuit-padding test that was failing
+      intermittently because of an ill-defined small histogram. Such
+      histograms will be allowed again after 29298 is implemented. Fixes
+      a second case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+    - Instead of relying on hs_free_all() to clean up all onion service
+      objects in test_build_descriptors(), we now deallocate them one by
+      one. This lets Coverity know that we are not leaking memory there
+      and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha.
+    - Check the time in the "Expires" header using approx_time(). Fixes
+      bug 30001; bugfix on 0.4.0.4-rc.
+
+  o Minor bugfixes (TLS protocol):
+    - When classifying a client's selection of TLS ciphers, if the
+      client ciphers are not yet available, do not cache the result.
+      Previously, we had cached the unavailability of the cipher list
+      and never looked again, which in turn led us to assume that the
+      client only supported the ancient V1 link protocol. This, in turn,
+      was causing Stem integration tests to stall in some cases. Fixes
+      bug 30021; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (UI):
+    - Lower log level of unlink() errors during bootstrap. Fixes bug
+      29930; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (usability):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+  o Minor bugfixes (Windows, CI):
+    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit
+      Windows Server 2012 R2 job. The remaining 2 jobs still provide
+      coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set
+      fast_finish, so failed jobs terminate the build immediately. Fixes
+      bug 29601; bugfix on 0.3.5.4-alpha.
+
+  o Code simplification and refactoring:
+    - Introduce a connection_dir_buf_add() helper function that detects
+      whether compression is in use, and adds a string accordingly.
+      Resolves issue 28816.
+    - Refactor handle_get_next_bandwidth() to use
+      connection_dir_buf_add(). Implements ticket 29897.
+    - Reimplement NETINFO cell parsing and generation to rely on
+      trunnel-generated wire format handling code. Closes ticket 27325.
+    - Remove unnecessary unsafe code from the Rust macro "cstr!". Closes
+      ticket 28077.
+    - Rework SOCKS wire format handling to rely on trunnel-generated
+      parsing/generation code. Resolves ticket 27620.
+    - Split out bootstrap progress reporting from control.c into a
+      separate file. Part of ticket 27402.
+    - The .may_include files that we use to describe our directory-by-
+      directory dependency structure now describe a noncircular
+      dependency graph over the directories that they cover. Our
+      checkIncludes.py tool now enforces this noncircularity. Closes
+      ticket 28362.
+
+  o Documentation:
+    - Clarify that Tor performs stream isolation among *Port listeners
+      by default. Resolves issue 29121.
+    - In the manpage entry describing MapAddress torrc setting, use
+      example IP addresses from ranges specified for use in documentation
+      by RFC 5737. Resolves issue 28623.
+    - Mention that you cannot add a new onion service if Tor is already
+      running with Sandbox enabled. Closes ticket 28560.
+    - Improve ControlPort documentation. Mention that it accepts
+      address:port pairs, and can be used multiple times. Closes
+      ticket 28805.
+    - Document the exact output of "tor --version". Closes ticket 28889.
+
+  o Removed features:
+    - Remove the old check-tor script. Resolves issue 29072.
+    - Stop responding to the 'GETINFO status/version/num-concurring' and
+      'GETINFO status/version/num-versioning' control port commands, as
+      those were deprecated back in 0.2.0.30. Also stop listing them in
+      output of 'GETINFO info/names'. Resolves ticket 28757.
+    - The scripts used to generate and maintain the list of fallback
+      directories have been extracted into a new "fallback-scripts"
+      repository. Closes ticket 27914.
+
+  o Testing:
+    - Run shellcheck for scripts in the in scripts/ directory. Closes
+      ticket 28058.
+    - Add unit tests for tokenize_string() and get_next_token()
+      functions. Resolves ticket 27625.
+
+  o Code simplification and refactoring (onion service v3):
+    - Consolidate the authorized client descriptor cookie computation
+      code from client and service into one function. Closes
+      ticket 27549.
+
+  o Code simplification and refactoring (shell scripts):
+    - Cleanup scan-build.sh to silence shellcheck warnings. Closes
+      ticket 28007.
+    - Fix issues that shellcheck found in chutney-git-bisect.sh.
+      Resolves ticket 28006.
+    - Fix issues that shellcheck found in updateRustDependencies.sh.
+      Resolves ticket 28012.
+    - Fix shellcheck warnings in cov-diff script. Resolves issue 28009.
+    - Fix shellcheck warnings in run_calltool.sh. Resolves ticket 28011.
+    - Fix shellcheck warnings in run_trunnel.sh. Resolves issue 28010.
+    - Fix shellcheck warnings in scripts/test/coverage. Resolves
+      issue 28008.
+
+
+Changes in version 0.3.5.8 - 2019-02-21
+  Tor 0.3.5.8 backports several fixes from later releases, including fixes
+  for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x
+  releases.
+
+  It also includes a fix for a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking, backport from 0.4.0.2-alpha):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth messsage and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Minor features (compilation, backport from 0.4.0.2-alpha):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor features (testing, backport from 0.4.0.2-alpha):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+
+  o Minor bugfixes (onion service v3, client, backport from 0.4.0.1-alpha):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (IPv6, backport from 0.4.0.1-alpha):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (client, clock skew, backport from 0.4.0.1-alpha):
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.0.1-alpha):
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (documentation, backport from 0.4.0.2-alpha):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (logging, backport from 0.4.0.1-alpha):
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (logging, backport from 0.4.0.2-alpha):
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (misc, backport from 0.4.0.2-alpha):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (tests, directory clients, backport from 0.4.0.1-alpha):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (tests, backport from 0.4.0.2-alpha):
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (usability, backport from 0.4.0.1-alpha):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.11 - 2019-02-21
+  Tor 0.3.4.11 is the third stable release in its series.  It includes
+  a fix for a medium-severity security bug affecting Tor 0.3.2.1-alpha and
+  later. All Tor instances running an affected release should upgrade to
+  0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.3.3.12 - 2019-02-21
+  Tor 0.3.3.12 fixes a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  This release marks the end of support for the Tor 0.3.3.x series. We
+  recommend that users switch to either the Tor 0.3.4 series (supported
+  until at least 10 June 2019), or the Tor 0.3.5 series, which will
+  receive long-term support until at least 1 Feb 2022.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.3.3.11 - 2019-01-07
+  Tor 0.3.3.11 backports numerous fixes from later versions of Tor.
+  numerous fixes, including an important fix for anyone using OpenSSL
+  1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade
+  to this version, or to a later series.
+
+  As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019.
+  We anticipate that this will be the last release of Tor 0.3.3, unless
+  some major bug is before then. Some time between now and then, users
+  should switch to either the Tor 0.3.4 series (supported until at least
+  10 June 2019), or the Tor 0.3.5 series, which will receive long-term
+  support until at least 1 Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.3.5.1-alpha):
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, backport from 0.3.5.3-alpha):
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha):
+    - Avoid undefined behavior in an end-of-string check when parsing
+      the BEGIN line in a directory object. Fixes bug 28202; bugfix
+      on 0.2.0.3-alpha.
+
+  o Minor bugfixes (code safety, backport from 0.3.5.3-alpha):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha):
+    - Log additional info when we get a relay that shares an ed25519 ID
+      with a different relay, instead making a BUG() warning. Fixes bug
+      27800; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha):
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to
+      an HTTP request we couldn't handle. Fixes bug 26470; bugfix on
+      0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (protover, backport from 0.3.5.3-alpha):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.1-alpha):
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.2-alpha):
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (rust, backport from 0.3.5.4-alpha):
+    - Fix a potential null dereference in protover_all_supported(). Add
+      a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
+    - Return a string that can be safely freed by C code, not one
+      created by the rust allocator, in protover_all_supported(). Fixes
+      bug 27740; bugfix on 0.3.3.1-alpha.
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
+
+  o Minor bugfixes (testing, backport from 0.3.5.1-alpha):
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.4-alpha):
+    - Treat backtrace test failures as expected on BSD-derived systems
+      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
+      (FreeBSD failures have been treated as expected since 18204 in
+      0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.10 - 2019-01-07
+  Tor 0.3.4.9 is the second stable release in its series; it backports
+  numerous fixes, including an important fix for relays, and for anyone
+  using OpenSSL 1.1.1. Anyone running an  earlier version of Tor 0.3.4
+  should upgrade.
+
+  As a reminder, the Tor 0.3.4 series will be supported until 10 June
+  2019. Some time between now and then, users should switch to the Tor
+  0.3.5 series, which will receive long-term support until at least 1
+  Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (relay, directory, backport from 0.3.5.7):
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+
+  o Minor features (controller, backport from 0.3.5.1-alpha):
+    - For purposes of CIRC_BW-based dropped cell detection, track half-
+      closed stream ids, and allow their ENDs, SENDMEs, DATA and path
+      bias check cells to arrive without counting it as dropped until
+      either the END arrives, or the windows are empty. Closes
+      ticket 25573.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28395.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.5.7 - 2019-01-07
+  Tor 0.3.5.7 is the first stable release in its series; it includes
+  compilation and portability fixes, and a fix for a severe problem
+  affecting directory caches.
+
+  The Tor 0.3.5 series includes several new features and performance
+  improvements, including client authorization for v3 onion services,
+  cleanups to bootstrap reporting, support for improved bandwidth-
+  measurement tools, experimental support for NSS in place of OpenSSL,
+  and much more. It also begins a full reorganization of Tor's code
+  layout, for improved modularity and maintainability in the future.
+  Finally, there is the usual set of performance improvements and
+  bugfixes that we try to do in every release series.
+
+  There are a couple of changes in the 0.3.5 that may affect
+  compatibility. First, the default version for newly created onion
+  services is now v3. Use the HiddenServiceVersion option if you want to
+  override this. Second, some log messages related to bootstrapping have
+  changed; if you use stem, you may need to update to the latest version
+  so it will recognize them.
+
+  We have designated 0.3.5 as a "long-term support" (LTS) series: we
+  will continue to patch major bugs in typical configurations of 0.3.5
+  until at least 1 Feb 2022. (We do not plan to provide long-term
+  support for embedding, Rust support, NSS support, running a directory
+  authority, or unsupported platforms. For these, you will need to stick
+  with the latest stable release.)
+
+  Below are the changes since 0.3.4.9. For a complete list of changes
+  since 0.3.5.6-rc, see the ChangeLog file.
+
+  o Major features (bootstrap):
+    - Don't report directory progress until after a connection to a
+      relay or bridge has succeeded. Previously, we'd report 80%
+      progress based on cached directory information when we couldn't
+      even connect to the network. Closes ticket 27169.
+
+  o Major features (new code layout):
+    - Nearly all of Tor's source code has been moved around into more
+      logical places. The "common" directory is now divided into a set
+      of libraries in "lib", and files in the "or" directory have been
+      split into "core" (logic absolutely needed for onion routing),
+      "feature" (independent modules in Tor), and "app" (to configure
+      and invoke the rest of Tor). See doc/HACKING/CodeStructure.md for
+      more information. Closes ticket 26481.
+
+      This refactoring is not complete: although the libraries have been
+      refactored to be acyclic, the main body of Tor is still too
+      interconnected. We will attempt to improve this in the future.
+
+  o Major features (onion services v3):
+    - Implement onion service client authorization at the descriptor
+      level: only authorized clients can decrypt a service's descriptor
+      to find out how to contact it. A new torrc option was added to
+      control this client side: ClientOnionAuthDir <path>. On the
+      service side, if the "authorized_clients/" directory exists in the
+      onion service directory path, client configurations are read from
+      the files within. See the manpage for more details. Closes ticket
+      27547. Patch done by Suphanat Chunhapanya (haxxpop).
+    - Improve revision counter generation in next-gen onion services.
+      Onion services can now scale by hosting multiple instances on
+      different hosts without synchronization between them, which was
+      previously impossible because descriptors would get rejected by
+      HSDirs. Addresses ticket 25552.
+    - Version 3 onion services can now use the per-service
+      HiddenServiceExportCircuitID option to differentiate client
+      circuits. It communicates with the service by using the HAProxy
+      protocol to assign virtual IP addresses to inbound client
+      circuits. Closes ticket 4700. Patch by Mahrud Sayrafi.
+
+  o Major features (onion services, UI change):
+    - For a newly created onion service, the default version is now 3.
+      Tor still supports existing version 2 services, but the operator
+      now needs to set "HiddenServiceVersion 2" in order to create a new
+      version 2 service. For existing services, Tor now learns the
+      version by reading the key file. Closes ticket 27215.
+
+  o Major features (portability, cryptography, experimental, TLS):
+    - Tor now has the option to compile with the NSS library instead of
+      OpenSSL. This feature is experimental, and we expect that bugs may
+      remain. It is mainly intended for environments where Tor's
+      performance is not CPU-bound, and where NSS is already known to be
+      installed. To try it out, configure Tor with the --enable-nss
+      flag. Closes tickets 26631, 26815, and 26816.
+
+      If you are experimenting with this option and using an old cached
+      consensus, Tor may fail to start. To solve this, delete your
+      "cached-consensus" and "cached-microdesc-consensus" files,
+      (if present), and restart Tor.
+
+  o Major features (relay, UI change):
+    - Relays no longer run as exits by default. If the "ExitRelay"
+      option is auto (or unset), and no exit policy is specified with
+      ExitPolicy or ReducedExitPolicy, we now treat ExitRelay as 0.
+      Previously in this case, we allowed exit traffic and logged a
+      warning message. Closes ticket 21530. Patch by Neel Chauhan.
+    - Tor now validates that the ContactInfo config option is valid UTF-
+      8 when parsing torrc. Closes ticket 27428.
+
+  o Major bugfixes (compilation):
+    - Fix compilation on ARM (and other less-used CPUs) when compiling
+      with OpenSSL before 1.1. Fixes bug 27781; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (compilation, rust):
+    - Rust tests can now build and run successfully with the
+      --enable-fragile-hardening option enabled. Doing this currently
+      requires the rust beta channel; it will be possible with stable
+      rust once Rust version 1.31 is released. Patch from Alex Crichton.
+      Fixes bugs 27272, 27273, and 27274. Bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (directory authority):
+    - Actually check that the address we get from DirAuthority
+      configuration line is valid IPv4. Explicitly disallow DirAuthority
+      address to be a DNS hostname. Fixes bug 26488; bugfix
+      on 0.1.2.10-rc.
+
+  o Major bugfixes (embedding, main loop):
+    - When DisableNetwork becomes set, actually disable periodic events
+      that are already enabled. (Previously, we would refrain from
+      enabling new ones, but we would leave the old ones turned on.)
+      Fixes bug 28348; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (main loop, bootstrap):
+    - Make sure Tor bootstraps and works properly if only the
+      ControlPort is set. Prior to this fix, Tor would only bootstrap
+      when a client port was set (Socks, Trans, NATD, DNS or HTTPTunnel
+      port). Fixes bug 27849; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (onion service v3):
+    - On an intro point for a version 3 onion service, stop closing
+      introduction circuits on a NACK. This lets the client decide
+      whether to reuse the circuit or discard it. Previously, we closed
+      intro circuits when sending NACKs. Fixes bug 27841; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chaunan.
+
+  o Major bugfixes (OpenSSL, portability):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (relay bandwidth statistics):
+    - When we close relayed circuits, report the data in the circuit
+      queues as being written in our relay bandwidth stats. This
+      mitigates guard discovery and other attacks that close circuits
+      for the explicit purpose of noticing this discrepancy in
+      statistics. Fixes bug 23512; bugfix on 0.0.8pre3.
+
+  o Major bugfixes (relay):
+    - When our write bandwidth limit is exhausted, stop writing on the
+      connection. Previously, we had a typo in the code that would make
+      us stop reading instead, leading to relay connections being stuck
+      indefinitely and consuming kernel RAM. Fixes bug 28089; bugfix
+      on 0.3.4.1-alpha.
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Major bugfixes (restart-in-process):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (admin tools):
+    - Add a new --key-expiration option to print the expiration date of
+      the signing cert in an ed25519_signing_cert file. Resolves
+      issue 19506.
+
+  o Minor features (build):
+    - If you pass the "--enable-pic" option to configure, Tor will try
+      to tell the compiler to build position-independent code suitable
+      to link into a dynamic library. (The default remains -fPIE, for
+      code suitable for a relocatable executable.) Closes ticket 23846.
+
+  o Minor features (code correctness, testing):
+    - Tor's build process now includes a "check-includes" make target to
+      verify that no module of Tor relies on any headers from a higher-
+      level module. We hope to use this feature over time to help
+      refactor our codebase. Closes ticket 26447.
+
+  o Minor features (code layout):
+    - We have a new "lowest-level" error-handling API for use by code
+      invoked from within the logging module. With this interface, the
+      logging code is no longer at risk of calling into itself if a
+      failure occurs while it is trying to log something. Closes
+      ticket 26427.
+
+  o Minor features (compilation):
+    - When possible, place our warning flags in a separate file, to
+      avoid flooding verbose build logs. Closes ticket 28924.
+    - Tor's configure script now supports a --with-malloc= option to
+      select your malloc implementation. Supported options are
+      "tcmalloc", "jemalloc", "openbsd" (deprecated), and "system" (the
+      default). Addresses part of ticket 20424. Based on a patch from
+      Alex Xu.
+
+  o Minor features (config):
+    - The "auto" keyword in torrc is now case-insensitive. Closes
+      ticket 26663.
+
+  o Minor features (continuous integration):
+    - Add a Travis CI build for --enable-nss on Linux gcc. Closes
+      ticket 27751.
+    - Add new CI job to Travis configuration to run stem-based
+      integration tests. Closes ticket 27913.
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+    - Report what program produced the mysterious core file that we
+      occasionally see on Travis CI during make distcheck. Closes
+      ticket 28024.
+    - Don't do a distcheck with --disable-module-dirauth in Travis.
+      Implements ticket 27252.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, Windows):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+    - Build tor on Windows Server 2012 R2 and Windows Server 2016 using
+      Appveyor's CI. Closes ticket 28318.
+
+  o Minor features (controller):
+    - Emit CIRC_BW events as soon as we detect that we processed an
+      invalid or otherwise dropped cell on a circuit. This allows
+      vanguards and other controllers to react more quickly to dropped
+      cells. Closes ticket 27678.
+    - For purposes of CIRC_BW-based dropped cell detection, track half-
+      closed stream ids, and allow their ENDs, SENDMEs, DATA and path
+      bias check cells to arrive without counting it as dropped until
+      either the END arrives, or the windows are empty. Closes
+      ticket 25573.
+    - Implement a 'GETINFO md/all' controller command to enable getting
+      all known microdescriptors. Closes ticket 8323.
+    - The GETINFO command now support an "uptime" argument, to return
+      Tor's uptime in seconds. Closes ticket 25132.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the DNS cache so that it doesn't
+      fill up the memory. This check is important for our DoS mitigation
+      subsystem. Closes ticket 18642. Patch by Neel Chauhan.
+
+  o Minor features (development):
+    - Tor's makefile now supports running the "clippy" Rust style tool
+      on our Rust code. Closes ticket 22156.
+
+  o Minor features (directory authority):
+    - There is no longer an artificial upper limit on the length of
+      bandwidth lines. Closes ticket 26223.
+    - When a bandwidth file is used to obtain the bandwidth measurements,
+      include this bandwidth file headers in the votes. Closes
+      ticket 3723.
+    - Improved support for networks with only a single authority or a
+      single fallback directory. Patch from Gabriel Somlo. Closes
+      ticket 25928.
+
+  o Minor features (embedding API):
+    - The Tor controller API now supports a function to launch Tor with
+      a preconstructed owning controller FD, so that embedding
+      applications don't need to manage controller ports and
+      authentication. Closes ticket 24204.
+    - The Tor controller API now has a function that returns the name
+      and version of the backend implementing the API. Closes
+      ticket 26947.
+
+  o Minor features (fallback directory list):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (memory management):
+    - Get Libevent to use the same memory allocator as Tor, by calling
+      event_set_mem_functions() during initialization. Resolves
+      ticket 8415.
+
+  o Minor features (memory usage):
+    - When not using them, store legacy TAP public onion keys in DER-
+      encoded format, rather than as expanded public keys. This should
+      save several megabytes on typical clients. Closes ticket 27246.
+
+  o Minor features (OpenSSL bug workaround):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor features (OpenSSL):
+    - When possible, use RFC5869 HKDF implementation from OpenSSL rather
+      than our own. Resolves ticket 19979.
+
+  o Minor features (performance):
+    - Remove about 96% of the work from the function that we run at
+      startup to test our curve25519_basepoint implementation. Since
+      this function has yet to find an actual failure, we now only run
+      it for 8 iterations instead of 200. Based on our profile
+      information, this change should save around 8% of our startup time
+      on typical desktops, and may have a similar effect on other
+      platforms. Closes ticket 28838.
+    - Stop re-validating our hardcoded Diffie-Hellman parameters on
+      every startup. Doing this wasted time and cycles, especially on
+      low-powered devices. Closes ticket 28851.
+
+  o Minor features (Rust, code quality):
+    - Improve rust code quality in the rust protover implementation by
+      making it more idiomatic. Includes changing an internal API to
+      take &str instead of &String. Closes ticket 26492.
+
+  o Minor features (testing):
+    - Add scripts/test/chutney-git-bisect.sh, for bisecting using
+      chutney. Implements ticket 27211.
+
+  o Minor features (tor-resolve):
+    - The tor-resolve utility can now be used with IPv6 SOCKS proxies.
+      Side-effect of the refactoring for ticket 26526.
+
+  o Minor features (UI):
+    - Log each included configuration file or directory as we read it,
+      to provide more visibility about where Tor is reading from. Patch
+      from Unto Sten; closes ticket 27186.
+    - Lower log level of "Scheduler type KIST has been enabled" to INFO.
+      Closes ticket 26703.
+
+  o Minor bugfixes (32-bit OSX and iOS, timing):
+    - Fix an integer overflow bug in our optimized 32-bit millisecond-
+      difference algorithm for 32-bit Apple platforms. Previously, it
+      would overflow when calculating the difference between two times
+      more than 47 days apart. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+    - Improve the precision of our 32-bit millisecond difference
+      algorithm for 32-bit Apple platforms. Fixes part of bug 27139;
+      bugfix on 0.3.4.1-alpha.
+    - Relax the tolerance on the mainloop/update_time_jumps test when
+      running on 32-bit Apple platforms. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (bootstrap):
+    - Try harder to get descriptors in non-exit test networks, by using
+      the mid weight for the third hop when there are no exits. Fixes
+      bug 27237; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Avoid casting smartlist index to int implicitly, as it may trigger
+      a warning (-Wshorten-64-to-32). Fixes bug 26282; bugfix on
+      0.2.3.13-alpha, 0.2.7.1-alpha and 0.2.1.1-alpha.
+    - Use time_t for all values in
+      predicted_ports_prediction_time_remaining(). Rework the code that
+      computes difference between durations/timestamps. Fixes bug 27165;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (client, memory usage):
+    - When not running as a directory cache, there is no need to store
+      the text of the current consensus networkstatus in RAM.
+      Previously, however, clients would store it anyway, at a cost of
+      over 5 MB. Now, they do not. Fixes bug 27247; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (client, ReachableAddresses):
+    - Instead of adding a "reject *:*" line to ReachableAddresses when
+      loading the configuration, add one to the policy after parsing it
+      in parse_reachable_addresses(). This prevents extra "reject *.*"
+      lines from accumulating on reloads. Fixes bug 20874; bugfix on
+      0.1.1.5-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (code quality):
+    - Rename sandbox_getaddrinfo() and other functions to no longer
+      misleadingly suggest that they are sandbox-only. Fixes bug 26525;
+      bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (code safety):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+      on 0.0.6.
+
+  o Minor bugfixes (compilation):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (configuration):
+    - Refuse to start with relative file paths and RunAsDaemon set
+      (regression from the fix for bug 22731). Fixes bug 28298; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (configuration, Onion Services):
+    - In rend_service_parse_port_config(), disallow any input to remain
+      after address-port pair was parsed. This will catch address and
+      port being whitespace-separated by mistake of the user. Fixes bug
+      27044; bugfix on 0.2.9.10.
+
+  o Minor bugfixes (connection, relay):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows):
+    - Stop reinstalling identical packages in our Windows CI. Fixes bug
+      27464; bugfix on 0.3.4.1-alpha.
+    - Install only the necessary mingw packages during our appveyor
+      builds. This change makes the build a little faster, and prevents
+      a conflict with a preinstalled mingw openssl that appveyor now
+      ships. Fixes bugs 27765 and 27943; bugfix on 0.3.4.2-alpha.
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (controller):
+    - Consider all routerinfo errors other than "not a server" to be
+      transient for the purpose of "GETINFO exit-policy/*" controller
+      request. Print stacktrace in the unlikely case of failing to
+      recompute routerinfo digest. Fixes bug 27034; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (correctness):
+    - Fix an unreached code path where we checked the value of
+      "hostname" inside send_resolved_hostname_cell(). Previously, we
+      used it before checking it; now we check it first. Fixes bug
+      28879; bugfix on 0.1.2.7-alpha.
+
+  o Minor bugfixes (directory connection shutdown):
+    - Avoid a double-close when shutting down a stalled directory
+      connection. Fixes bug 26896; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (directory permissions):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to an
+      HTTP request we couldn't handle. Fixes bug 26470; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (ipv6):
+    - In addrs_in_same_network_family(), we choose the subnet size based
+      on the IP version (IPv4 or IPv6). Previously, we chose a fixed
+      subnet size of /16 for both IPv4 and IPv6 addresses. Fixes bug
+      15518; bugfix on 0.2.3.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Permit the "shutdown()" system call, which is apparently used by
+      OpenSSL under some circumstances. Fixes bug 28183; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Stop talking about the Named flag in log messages. Clients have
+      ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix
+      on 0.3.2.1-alpha.
+    - As a precaution, do an early return from log_addr_has_changed() if
+      Tor is running as client. Also, log a stack trace for debugging as
+      this function should only be called when Tor runs as server. Fixes
+      bug 26892; bugfix on 0.1.1.9-alpha.
+    - Refrain from mentioning bug 21018 in the logs, as it is already
+      fixed. Fixes bug 25477; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (logging, documentation):
+    - When SafeLogging is enabled, scrub IP address in
+      channel_tls_process_netinfo_cell(). Also, add a note to manpage
+      that scrubbing is not guaranteed on loglevels below Notice. Fixes
+      bug 26882; bugfix on 0.2.4.10-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+    - Fix a small memory leak when calling Tor with --dump-config. Fixes
+      bug 27893; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (netflow padding):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion service v2):
+    - Log at level "info", not "warning", in the case that we do not
+      have a consensus when a .onion request comes in. This can happen
+      normally while bootstrapping. Fixes bug 27040; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+    - Build the service descriptor's signing key certificate before
+      uploading, so we always have a fresh one: leaving no chances for
+      it to expire service side. Fixes bug 27838; bugfix
+      on 0.3.2.1-alpha.
+    - Stop dumping a stack trace when trying to connect to an intro
+      point without having a descriptor for it. Fixes bug 27774; bugfix
+      on 0.3.2.1-alpha.
+    - When selecting a v3 rendezvous point, don't only look at the
+      protover, but also check whether the curve25519 onion key is
+      present. This way we avoid picking a relay that supports the v3
+      rendezvous but for which we don't have the microdescriptor. Fixes
+      bug 27797; bugfix on 0.3.2.1-alpha.
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+    - When replacing a descriptor in the client cache, make sure to
+      close all client introduction circuits for the old descriptor, so
+      we don't end up with unusable leftover circuits. Fixes bug 27471;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (OS compatibility):
+    - Properly handle configuration changes that move a listener to/from
+      wildcard IP address. If the first attempt to bind a socket fails,
+      close the old listener and try binding the socket again. Fixes bug
+      17873; bugfix on 0.0.8pre-1.
+
+  o Minor bugfixes (performance)::
+    - Rework node_is_a_configured_bridge() to no longer call
+      node_get_all_orports(), which was performing too many memory
+      allocations. Fixes bug 27224; bugfix on 0.2.3.9.
+
+  o Minor bugfixes (protover):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (protover, rust):
+    - Reject extra commas in version strings. Fixes bug 27197; bugfix
+      on 0.3.3.3-alpha.
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (relay shutdown, systemd):
+    - Notify systemd of ShutdownWaitLength so it can be set to longer
+      than systemd's TimeoutStopSec. In Tor's systemd service file, set
+      TimeoutSec to 60 seconds to allow Tor some time to shut down.
+      Fixes bug 28113; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (relay statistics):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (relay):
+    - Consider the fact that we'll be making direct connections to our
+      entry and guard nodes when computing the fraction of nodes that
+      have their descriptors. Also, if we are using bridges and there is
+      at least one bridge with a full descriptor, treat the fraction of
+      guards available as 100%. Fixes bug 25886; bugfix on 0.2.4.10-alpha.
+      Patch by Neel Chauhan.
+    - Update the message logged on relays when DirCache is disabled.
+      Since 0.3.3.5-rc, authorities require DirCache (V2Dir) for the
+      Guard flag. Fixes bug 24312; bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (testing):
+    - Stop running stem's unit tests as part of "make test-stem", but
+      continue to run stem's unit and online tests during "make test-
+      stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha.
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+    - Make the hs_service tests use the same time source when creating
+      the introduction point and when testing it. Now tests work better
+      on very slow systems like ARM or Travis. Fixes bug 27810; bugfix
+      on 0.3.2.1-alpha.
+    - Revise the "conditionvar_timeout" test so that it succeeds even on
+      heavily loaded systems where the test threads are not scheduled
+      within 200 msec. Fixes bug 27073; bugfix on 0.2.6.3-alpha.
+    - Fix two unit tests to work when HOME environment variable is not
+      set. Fixes bug 27096; bugfix on 0.2.8.1-alpha.
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+    - When logging a version mismatch in our openssl_version tests,
+      report the actual offending version strings. Fixes bug 26152;
+      bugfix on 0.2.9.1-alpha.
+    - Fix forking tests on Windows when there is a space somewhere in
+      the path. Fixes bug 26437; bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (Windows):
+    - Correctly identify Windows 8.1, Windows 10, and Windows Server
+      2008 and later from their NT versions. Fixes bug 28096; bugfix on
+      0.2.2.34; reported by Keifer Bly.
+    - On recent Windows versions, the GetVersionEx() function may report
+      an earlier Windows version than the running OS. To avoid user
+      confusion, add "[or later]" to Tor's version string on affected
+      versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported
+      by Keifer Bly.
+    - Remove Windows versions that were never supported by the
+      GetVersionEx() function. Stop duplicating the latest Windows
+      version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34;
+      reported by Keifer Bly.
+
+  o Code simplification and refactoring:
+    - When parsing a port configuration, make it more obvious to static
+      analyzer tools that we always initialize the address. Closes
+      ticket 28881.
+    - Divide more large Tor source files -- especially ones that span
+      multiple areas of functionality -- into smaller parts, including
+      onion.c and main.c. Closes ticket 26747.
+    - Divide the "routerparse.c" module into separate modules for each
+      group of parsed objects. Closes ticket 27924.
+    - Move protover_rust.c to the same place protover.c was moved to.
+      Closes ticket 27814.
+    - Split directory.c into separate pieces for client, server, and
+      common functionality. Closes ticket 26744.
+    - Split the non-statistics-related parts from the rephist.c and
+      geoip.c modules. Closes ticket 27892.
+    - Split the router.c file into relay-only and shared components, to
+      help with future modularization. Closes ticket 27864.
+    - Divide the routerlist.c and dirserv.c modules into smaller parts.
+      Closes ticket 27799.
+    - 'updateFallbackDirs.py' now ignores the blacklist file, as it's not
+      longer needed. Closes ticket 26502.
+    - Include paths to header files within Tor are now qualified by
+      directory within the top-level src directory.
+    - Many structures have been removed from the centralized "or.h"
+      header, and moved into their own headers. This will allow us to
+      reduce the number of places in the code that rely on each
+      structure's contents and layout. Closes ticket 26383.
+    - Remove ATTR_NONNULL macro from codebase. Resolves ticket 26527.
+    - Remove GetAdaptersAddresses_fn_t. The code that used it was
+      removed as part of the 26481 refactor. Closes ticket 27467.
+    - Rework Tor SOCKS server code to use Trunnel and benefit from
+      autogenerated functions for parsing and generating SOCKS wire
+      format. New implementation is cleaner, more maintainable and
+      should be less prone to heartbleed-style vulnerabilities.
+      Implements a significant fraction of ticket 3569.
+    - Split sampled_guards_update_from_consensus() and
+      select_entry_guard_for_circuit() into subfunctions. In
+      entry_guards_update_primary() unite three smartlist enumerations
+      into one and move smartlist comparison code out of the function.
+      Closes ticket 21349.
+    - Tor now assumes that you have standards-conformant stdint.h and
+      inttypes.h headers when compiling. Closes ticket 26626.
+    - Unify our bloom filter logic. Previously we had two copies of this
+      code: one for routerlist filtering, and one for address set
+      calculations. Closes ticket 26510.
+    - Use the simpler strcmpstart() helper in
+      rend_parse_v2_service_descriptor instead of strncmp(). Closes
+      ticket 27630.
+    - Utility functions that can perform a DNS lookup are now wholly
+      separated from those that can't, in separate headers and C
+      modules. Closes ticket 26526.
+
+  o Documentation:
+    - In the tor-resolve(1) manpage, fix the reference to socks-
+      extensions.txt by adding a web URL. Resolves ticket 27853.
+    - Mention that we require Python to be 2.7 or newer for some
+      integration tests that we ship with Tor. Resolves ticket 27677.
+    - Copy paragraph and URL to Tor's code of conduct document from
+      CONTRIBUTING to new CODE_OF_CONDUCT file. Resolves ticket 26638.
+    - Remove old instructions from INSTALL document. Closes ticket 26588.
+    - Warn users that they should not include MyFamily line(s) in their
+      torrc when running Tor bridge. Closes ticket 26908.
+
+  o Removed features:
+    - Tor no longer supports building with the dmalloc library. For
+      debugging memory issues, we suggest using gperftools or msan
+      instead. Closes ticket 26426.
+    - Tor no longer attempts to run on Windows environments without the
+      GetAdaptersAddresses() function. This function has existed since
+      Windows XP, which is itself already older than we support.
+    - Remove Tor2web functionality for version 2 onion services. The
+      Tor2webMode and Tor2webRendezvousPoints options are now obsolete.
+      (This feature was never shipped in vanilla Tor and it was only
+      possible to use this feature by building the support at compile
+      time. Tor2webMode is not implemented for version 3 onion services.)
+      Closes ticket 26367.
+
+  o Testing:
+    - Increase logging and tag all log entries with timestamps in
+      test_rebind.py. Provides diagnostics for issue 28229.
+
+  o Code simplification and refactoring (shared random, dirauth):
+    - Change many tor_assert() to use BUG() instead. The idea is to not
+      crash a dirauth but rather scream loudly with a stacktrace and let
+      it continue run. The shared random subsystem is very resilient and
+      if anything wrong happens with it, at worst a non coherent value
+      will be put in the vote and discarded by the other authorities.
+      Closes ticket 19566.
+
+  o Documentation (onion services):
+    - Improve HSv3 client authorization by making some options more
+      explicit and detailed. Closes ticket 28026. Patch by Mike Tigas.
+    - Document in the man page that changing ClientOnionAuthDir value or
+      adding a new file in the directory will not work at runtime upon
+      sending a HUP if Sandbox 1. Closes ticket 28128.
+    - Note in the man page that the only real way to fully revoke an
+      onion service v3 client authorization is by restarting the tor
+      process. Closes ticket 28275.
+
+
 Changes in version 0.3.4.9 - 2018-11-02
   Tor 0.3.4.9 is the second stable release in its series; it backports
   numerous fixes, including a fix for a bandwidth management bug that
@@ -1034,7 +6069,7 @@ Changes in version 0.3.4.8 - 2018-09-10
     - Remove duplicate code in parse_{c,s}method_line and bootstrap
       their functionalities into a single function. Fixes bug 6236;
       bugfix on 0.2.3.6-alpha.
-    - We remove the PortForwsrding and PortForwardingHelper options,
+    - We remove the PortForwarding and PortForwardingHelper options,
       related functions, and the port_forwarding tests. These options
       were used by the now-deprecated Vidalia to help ordinary users
       become Tor relays or bridges. Closes ticket 25409. Patch by
diff --git a/autogen.sh b/autogen.sh
index 276dd4047c..63ef6d49ef 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,9 +1,9 @@
 #!/bin/sh
 
-if [ -x "`which autoreconf 2>/dev/null`" ] ; then
+if command -v autoreconf; then
   opt="-i -f -W all,error"
 
-  for i in $@; do
+  for i in "$@"; do
     case "$i" in
       -v)
         opt="${opt} -v"
@@ -11,6 +11,7 @@ if [ -x "`which autoreconf 2>/dev/null`" ] ; then
     esac
   done
 
+  # shellcheck disable=SC2086
   exec autoreconf $opt
 fi
 
diff --git a/changes/29241_diagnostic b/changes/29241_diagnostic
deleted file mode 100644
index 1e38654957..0000000000
--- a/changes/29241_diagnostic
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (NSS, diagnostic):
-    - Try to log an error from NSS (if there is any) and a more useful
-      description of our situation if we are using NSS and a call to
-      SSL_ExportKeyingMaterial() fails.  Diagnostic for ticket 29241.
diff --git a/changes/40241_v2 b/changes/40241_v2
deleted file mode 100644
index 85038297f7..0000000000
--- a/changes/40241_v2
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix another warning about unreachable fallthrough annotations
-      when building with "--enable-all-bugs-are-fatal" on some compilers.
-      Fixes bug 40241; bugfix on 0.4.5.3-rc.
diff --git a/changes/bug12399 b/changes/bug12399
deleted file mode 100644
index 922c08c5e3..0000000000
--- a/changes/bug12399
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (logging):
-    - Change log level of message "Hash of session info was not as expected"
-      to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix on 0.1.1.10-alpha.
diff --git a/changes/bug13221 b/changes/bug13221
deleted file mode 100644
index 13935a1921..0000000000
--- a/changes/bug13221
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Correct a misleading error message when IPv4Only or IPv6Only
-      is used but the resolved address can not be interpreted as an
-      address of the specified IP version.  Fixes bug 13221; bugfix
-      on 0.2.3.9-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug22619 b/changes/bug22619
deleted file mode 100644
index 9c71996f5b..0000000000
--- a/changes/bug22619
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (circuit isolation):
-    - Fix a logic error that prevented the SessionGroup sub-option from
-      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
diff --git a/changes/bug23507 b/changes/bug23507
deleted file mode 100644
index de18273fdb..0000000000
--- a/changes/bug23507
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (v3 single onion services):
-    - Make v3 single onion services fall back to a 3-hop intro, when there
-      all intro points are unreachable via a 1-hop path. Previously, v3
-      single onion services failed when all intro nodes were unreachable
-      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug23818_v2 b/changes/bug23818_v2
deleted file mode 100644
index 0219a20f49..0000000000
--- a/changes/bug23818_v2
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (v2 single onion services):
-    - Always retry v2 single onion service intro and rend circuits with a
-      3-hop path. Previously, v2 single onion services used a 3-hop path
-      when rend circuits were retried after a remote or delayed failure,
-      but a 1-hop path for immediate retries. Fixes bug 23818;
-      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug23818_v3 b/changes/bug23818_v3
deleted file mode 100644
index c430144d81..0000000000
--- a/changes/bug23818_v3
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (v3 single onion services):
-    - Always retry v3 single onion service intro and rend circuits with a
-      3-hop path. Previously, v3 single onion services used a 3-hop path
-      when rend circuits were retried after a remote or delayed failure,
-      but a 1-hop path for immediate retries. Fixes bug 23818;
-      bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug24661 b/changes/bug24661
deleted file mode 100644
index a915a93e0e..0000000000
--- a/changes/bug24661
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (client, guard selection):
-    - When Tor's consensus has expired, but is still reasonably live, use it
-      to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug27197 b/changes/bug27197
deleted file mode 100644
index e389f85065..0000000000
--- a/changes/bug27197
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (protover, rust):
-    - Reject extra commas in version string. Fixes bug 27197; bugfix on
-      0.3.3.3-alpha.
diff --git a/changes/bug27199 b/changes/bug27199
deleted file mode 100644
index f9d2a422f9..0000000000
--- a/changes/bug27199
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (rust):
-    - Abort on panic in all build profiles, instead of potentially unwinding
-      into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27315 b/changes/bug27315
new file mode 100644
index 0000000000..8af3ac8559
--- /dev/null
+++ b/changes/bug27315
@@ -0,0 +1,6 @@
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Fix a regression on sandboxing rules for the openat() syscall.
+      The fix for bug 25440 fixed the problem on systems with glibc >=
+      2.27 but broke tor on previous versions of glibc. We now apply 
+      the correct seccomp rule according to the running glibc version.
+      Patch from Daniel Pinto. Fixes bug 27315; bugfix on 0.3.5.11.
diff --git a/changes/bug27740 b/changes/bug27740
deleted file mode 100644
index 76a17b7dda..0000000000
--- a/changes/bug27740
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (rust):
-    - Return a string that can be safely freed by C code, not one created by
-      the rust allocator, in protover_all_supported(). Fixes bug 27740; bugfix
-      on 0.3.3.1-alpha.
diff --git a/changes/bug27741 b/changes/bug27741
deleted file mode 100644
index 531e264b63..0000000000
--- a/changes/bug27741
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (rust, directory authority):
-    - Fix an API mismatch in the rust implementation of
-      protover_compute_vote(). This bug could have caused crashes on any
-      directory authorities running Tor with Rust (which we do not yet
-      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
diff --git a/changes/bug27750 b/changes/bug27750
deleted file mode 100644
index c234788b1c..0000000000
--- a/changes/bug27750
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (connection, relay):
-    - Avoid a wrong BUG() stacktrace in case a closing connection is being held
-      open because the write side is rate limited but not the read side. Now,
-      the connection read side is simply shutdown instead of kept open until tor
-      is able to flush the connection and then fully close it. Fixes bug 27750;
-      bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27800 b/changes/bug27800
deleted file mode 100644
index 63d5dbc681..0000000000
--- a/changes/bug27800
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (directory authority):
-    - Log additional info when we get a relay that shares an ed25519
-      ID with a different relay, instead making a BUG() warning.
-      Fixes bug 27800; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug27804 b/changes/bug27804
deleted file mode 100644
index fa7fec0bc5..0000000000
--- a/changes/bug27804
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (rust):
-    - Fix a potential null dereference in protover_all_supported().
-      Add a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27841 b/changes/bug27841
deleted file mode 100644
index 9cd1da7275..0000000000
--- a/changes/bug27841
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (onion services):
-    - On an intro point for a version 3 onion service, we do not close
-      an introduction circuit on an NACK. This lets the client decide
-      whether to reuse the circuit or discard it. Previously, we closed
-      intro circuits on NACKs. Fixes bug 27841; bugfix on 0.3.2.1-alpha.
-      Patch by Neel Chaunan
-
diff --git a/changes/bug27948 b/changes/bug27948
deleted file mode 100644
index fea16f3d0f..0000000000
--- a/changes/bug27948
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (tests):
-    - Treat backtrace test failures as expected on BSD-derived systems
-      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
-      (FreeBSD failures have been treated as expected since 18204 in 0.2.8.)
-      Fixes bug 27948; bugfix on 0.2.5.2-alpha.
-
diff --git a/changes/bug27963_timeradd b/changes/bug27963_timeradd
deleted file mode 100644
index 34b361cf8d..0000000000
--- a/changes/bug27963_timeradd
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation, opensolaris):
-    - Add a missing include to compat_pthreads.c, to fix compilation
-      on OpenSolaris and its descendants. Fixes bug 27963; bugfix
-      on 0.3.5.1-alpha.
diff --git a/changes/bug27968 b/changes/bug27968
deleted file mode 100644
index 78c8eee33a..0000000000
--- a/changes/bug27968
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (testing):
-    - Avoid hangs and race conditions in test_rebind.py.
-      Fixes bug 27968; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28096 b/changes/bug28096
deleted file mode 100644
index 6847df9798..0000000000
--- a/changes/bug28096
+++ /dev/null
@@ -1,13 +0,0 @@
-  o Minor bugfixes (Windows):
-    - Correctly identify Windows 8.1, Windows 10, and Windows Server 2008
-      and later from their NT versions.
-      Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly.
-    - On recent Windows versions, the GetVersionEx() function may report
-      an earlier Windows version than the running OS. To avoid user
-      confusion, add "[or later]" to Tor's version string on affected
-      versions of Windows.
-      Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly.
-    - Remove Windows versions that were never supported by the
-      GetVersionEx() function. Stop duplicating the latest Windows
-      version in get_uname().
-      Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly.
diff --git a/changes/bug28115 b/changes/bug28115
deleted file mode 100644
index e3e29968eb..0000000000
--- a/changes/bug28115
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (portability):
-    - Make the OPE code (which is used for v3 onion services) run correctly
-      on big-endian platforms. Fixes bug 28115; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28127 b/changes/bug28127
deleted file mode 100644
index 541128c88e..0000000000
--- a/changes/bug28127
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (onion services):
-    - Unless we have explicitly set HiddenServiceVersion, detect the onion
-      service version and then look for invalid options. Previously, we
-      did the reverse, but that broke existing configs which were pointed
-      to a v2 hidden service and had options like HiddenServiceAuthorizeClient
-      set Fixes bug 28127; bugfix on 0.3.5.1-alpha. Patch by Neel Chauhan.
-
diff --git a/changes/bug28183 b/changes/bug28183
deleted file mode 100644
index 8d35dcdc01..0000000000
--- a/changes/bug28183
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (Linux seccomp2 sandbox):
-    - Permit the "shutdown()" system call, which is apparently
-      used by OpenSSL under some circumstances. Fixes bug 28183;
-      bugfix on 0.2.5.1-alpha.
diff --git a/changes/bug28202 b/changes/bug28202
deleted file mode 100644
index 182daac4f1..0000000000
--- a/changes/bug28202
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (C correctness):
-    - Avoid undefined behavior in an end-of-string check when parsing the
-      BEGIN line in a directory object.  Fixes bug 28202; bugfix on
-      0.2.0.3-alpha.
diff --git a/changes/bug28245 b/changes/bug28245
deleted file mode 100644
index d7e6deb810..0000000000
--- a/changes/bug28245
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (OpenSSL, portability):
-    - Fix our usage of named groups when running as a TLS 1.3 client in
-      OpenSSL 1.1.1. Previously, we only initialized EC groups when running
-      as a server, which caused clients to fail to negotiate TLS 1.3 with
-      relays. Fixes bug 28245; bugfix on 0.2.9.15 when TLS 1.3 support was
-      added.
diff --git a/changes/bug28298 b/changes/bug28298
deleted file mode 100644
index 8db340f3df..0000000000
--- a/changes/bug28298
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (configuration):
-    - Resume refusing to start with relative file paths and RunAsDaemon
-      set (regression from the fix for bug 22731). Fixes bug 28298;
-      bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug28303 b/changes/bug28303
deleted file mode 100644
index 80f1302e5e..0000000000
--- a/changes/bug28303
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix a pair of missing headers on OpenBSD. Fixes bug 28303;
-      bugfix on 0.3.5.1-alpha. Patch from  Kris Katterjohn.
diff --git a/changes/bug28348_034 b/changes/bug28348_034
deleted file mode 100644
index 3913c03a4c..0000000000
--- a/changes/bug28348_034
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (embedding, main loop):
-    - When DisableNetwork becomes set, actually disable periodic events that
-      are already enabled. (Previously, we would refrain from enabling new
-      ones, but we would leave the old ones turned on.)
-      Fixes bug 28348; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28399 b/changes/bug28399
deleted file mode 100644
index 9096db70b0..0000000000
--- a/changes/bug28399
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (continuous integration, Windows):
-    - Stop using an external OpenSSL install, and stop installing MSYS2
-      packages, when building using mingw on Appveyor Windows CI.
-      Fixes bug 28399; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28413 b/changes/bug28413
deleted file mode 100644
index 4c88bea7e7..0000000000
--- a/changes/bug28413
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Initialize a variable in aes_new_cipher(), since some compilers
-      cannot tell that we always initialize it before use. Fixes bug 28413;
-      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug28419 b/changes/bug28419
deleted file mode 100644
index 52ceb0a2a7..0000000000
--- a/changes/bug28419
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (memory leaks):
-    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
-      bugfix on 0.3.3.1-alpha.  Patch from Martin Kepplinger.
\ No newline at end of file
diff --git a/changes/bug28435 b/changes/bug28435
deleted file mode 100644
index 2a886cb8b7..0000000000
--- a/changes/bug28435
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (documentation):
-    - Make Doxygen work again after the 0.3.5 source tree moves.
-      Fixes bug 28435; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28441 b/changes/bug28441
deleted file mode 100644
index d259b9f742..0000000000
--- a/changes/bug28441
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (logging):
-    - Stop talking about the Named flag in log messages. Clients have
-      ignored the Named flag since 0.3.2. Fixes bug 28441;
-      bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug28454 b/changes/bug28454
deleted file mode 100644
index ca46ae2777..0000000000
--- a/changes/bug28454
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (continuous integration, Windows):
-    - Manually configure the zstd compiler options, when building using
-      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not
-      come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28485 b/changes/bug28485
deleted file mode 100644
index a8309ae21f..0000000000
--- a/changes/bug28485
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Add missing dependency on libgdi32.dll for tor-print-ed-signing-cert.exe
-      on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28524 b/changes/bug28524
deleted file mode 100644
index 1cad700422..0000000000
--- a/changes/bug28524
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (restart-in-process, boostrap):
-    - Add missing resets of bootstrap tracking state when shutting
-      down (regression caused by ticket 27169).  Fixes bug 28524;
-      bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28525 b/changes/bug28525
deleted file mode 100644
index 988ffb2192..0000000000
--- a/changes/bug28525
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor features (address selection):
-    - Make Tor aware of the RFC 6598 (Carrier Grade NAT) IP range, which is the
-      subnet 100.64.0.0/10. This is deployed by many ISPs as an alternative to
-      RFC 1918 that does not break existing internal networks. This patch fixes
-      security issues caused by RFC 6518 by blocking control ports on these
-      addresses and warns users if client ports or ExtORPorts are listening on
-      a RFC 6598 address. Closes ticket 28525. Patch by Neel Chauhan.
diff --git a/changes/bug28554 b/changes/bug28554
deleted file mode 100644
index 9a0b281406..0000000000
--- a/changes/bug28554
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (unit tests, guard selection):
-    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
-      bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug28562 b/changes/bug28562
deleted file mode 100644
index e14362164d..0000000000
--- a/changes/bug28562
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (testing):
-    - Use a separate DataDirectory for the test_rebind script.
-      Previously, this script would run using the default DataDirectory,
-      and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha.
-      Patch from Taylor R Campbell.
diff --git a/changes/bug28568 b/changes/bug28568
deleted file mode 100644
index 919ec08903..0000000000
--- a/changes/bug28568
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Stop running stem's unit tests as part of "make test-stem". But continue
-      to run stem's unit and online tests during "make test-stem-full".
-      Fixes bug 28568; bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug28569 b/changes/bug28569
deleted file mode 100644
index 45a57a80ae..0000000000
--- a/changes/bug28569
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (unit tests, directory clients):
-    - Mark outdated dirservers when Tor only has a reasonably live consensus.
-      Fixes bug 28569; bugfix on 0.3.2.5-alpha.
diff --git a/changes/bug28612 b/changes/bug28612
deleted file mode 100644
index 559f254234..0000000000
--- a/changes/bug28612
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (windows services):
-    - Make Tor start correctly as an NT service again: previously it
-      was broken by refactoring.  Fixes bug 28612; bugfix on 0.3.5.3-alpha.
-      
diff --git a/changes/bug28619 b/changes/bug28619
deleted file mode 100644
index 86be8cb2fb..0000000000
--- a/changes/bug28619
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (hidden service v3):
-    - When deleting an ephemeral onion service (DEL_ONION), do not close any
-      rendezvous circuits in order to let the existing client connections
-      finish by themselves or closed by the application. The HS v2 is doing
-      that already so now we have the same behavior for all versions. Fixes
-      bug 28619; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug28656 b/changes/bug28656
deleted file mode 100644
index d3a13d196c..0000000000
--- a/changes/bug28656
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (logging):
-    - Stop logging a BUG() warning when tor is waiting for exit descriptors.
-      Fixes bug 28656; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28698 b/changes/bug28698
deleted file mode 100644
index 716aa0c552..0000000000
--- a/changes/bug28698
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfix (logging):
-    - Avoid logging about relaxing circuits when their time is fixed.
-      Fixes bug 28698; bugfix on 0.2.4.7-alpha
diff --git a/changes/bug28895 b/changes/bug28895
deleted file mode 100644
index 25fb167b2e..0000000000
--- a/changes/bug28895
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (usability):
-    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate()
-      as that confusingly suggests that mentioned guard node is under control
-      and responsibility of end user, which it is not. Fixes bug 28895;
-      bugfix on Tor 0.3.0.1-alpha.
diff --git a/changes/bug28920 b/changes/bug28920
deleted file mode 100644
index e698686a6d..0000000000
--- a/changes/bug28920
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (logging):
-    - Rework rep_hist_log_link_protocol_counts() to iterate through all link
-      protocol versions when logging incoming/outgoing connection counts. Tor
-      no longer skips version 5 and we don't have to remember to update this
-      function when new link protocol version is developed. Fixes bug 28920;
-      bugfix on 0.2.6.10.
diff --git a/changes/bug28938 b/changes/bug28938
deleted file mode 100644
index de6c5f7b79..0000000000
--- a/changes/bug28938
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix missing headers required for proper detection of
-      OpenBSD.  Fixes bug 28938; bugfix on 0.3.5.1-alpha.
-      Patch from Kris Katterjohn.
diff --git a/changes/bug28974 b/changes/bug28974
deleted file mode 100644
index 2d74f5674f..0000000000
--- a/changes/bug28974
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix compilation for Android by adding a missing header to
-      freespace.c.  Fixes bug 28974; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28979 b/changes/bug28979
deleted file mode 100644
index 0625fd5d25..0000000000
--- a/changes/bug28979
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (documentation):
-    - Describe the contents of the v3 onion service client authorization
-      files correctly: They hold public keys, not private keys. Fixes bug
-      28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
diff --git a/changes/bug28981 b/changes/bug28981
deleted file mode 100644
index c0ea92ab35..0000000000
--- a/changes/bug28981
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (misc):
-    - The amount of total available physical memory is now determined
-      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
-      when it is defined and a 64-bit variant is not available.  Fixes
-      bug 28981; bugfix on 0.2.5.4-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug28995 b/changes/bug28995
deleted file mode 100644
index f76b6a085a..0000000000
--- a/changes/bug28995
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfix (IPv6):
-    Fix tor_ersatz_socketpair on IPv6-only systems.  Previously,
-    the IPv6 socket was bound using an address family of AF_INET
-    instead of AF_INET6.  Fixes bug 28995; bugfix on 0.3.5.1-alpha.
-    Patch from Kris Katterjohn.
diff --git a/changes/bug29017 b/changes/bug29017
deleted file mode 100644
index 5c4a53c43f..0000000000
--- a/changes/bug29017
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (stats):
-    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
-      relay and bridge extra-info documents. Fixes bug 29017;
-      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug29029 b/changes/bug29029
deleted file mode 100644
index e100a8c2ed..0000000000
--- a/changes/bug29029
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging, onion services):
-    - Stop logging "Tried to establish rendezvous on non-OR circuit..." as
-      a warning. Instead, log it as a protocol warning, because there is
-      nothing that relay operators can do to fix it. Fixes bug 29029;
-      bugfix on 0.2.5.7-rc.
diff --git a/changes/bug29034 b/changes/bug29034
deleted file mode 100644
index e7aa9af00b..0000000000
--- a/changes/bug29034
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (Onion service reachability):
-    - Properly clean up the introduction point map when circuits change purpose
-      from onion service circuits to pathbias, measurement, or other circuit types.
-      This should fix some service-side instances of introduction point failure.
-      Fixes bug 29034; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug29036 b/changes/bug29036
deleted file mode 100644
index 8b96c5c8fa..0000000000
--- a/changes/bug29036
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfix (continuous integration):
-    - Reset coverage state on disk after Travis CI has finished. This is being
-      done to prevent future gcda file merge errors which causes the test suite
-      for the process subsystem to fail. The process subsystem was introduced
-      in 0.4.0.1-alpha. Fixes bug 29036; bugfix on 0.2.9.15.
diff --git a/changes/bug29040 b/changes/bug29040
deleted file mode 100644
index 0662aaa8a5..0000000000
--- a/changes/bug29040
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (onion services):
-    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains
-      more than one private key for a hidden service. Fixes bug 29040;
-      bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29042 b/changes/bug29042
deleted file mode 100644
index 8d76939cea..0000000000
--- a/changes/bug29042
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Log more information at "warning" level when unable to read a private
-      key; log more information ad "info" level when unable to read a public
-      key. We had warnings here before, but they were lost during our
-      NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29135 b/changes/bug29135
deleted file mode 100644
index fd7b1ae80e..0000000000
--- a/changes/bug29135
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (onion services, logging):
-    - In hs_cache_store_as_client() log an HSDesc we failed to parse at Debug
-      loglevel. Tor used to log it at Warning loglevel, which caused
-      very long log lines to appear for some users. Fixes bug 29135; bugfix on
-      0.3.2.1-alpha.
diff --git a/changes/bug29144 b/changes/bug29144
deleted file mode 100644
index 5801224f14..0000000000
--- a/changes/bug29144
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Log the correct port number for listening sockets when "auto" is
-      used to let Tor pick the port number.  Previously, port 0 was
-      logged instead of the actual port number.  Fixes bug 29144;
-      bugfix on 0.3.5.1-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug29145 b/changes/bug29145
deleted file mode 100644
index 40d3da4b91..0000000000
--- a/changes/bug29145
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation, testing):
-    - Silence a compiler warning in test-memwipe.c on OpenBSD.  Fixes
-      bug 29145; bugfix on 0.2.9.3-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug29161 b/changes/bug29161
deleted file mode 100644
index 39a638acf6..0000000000
--- a/changes/bug29161
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (tests):
-    - Detect and suppress "bug" warnings from the util/time test on Windows.
-      Fixes bug 29161; bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug29175_035 b/changes/bug29175_035
deleted file mode 100644
index 134c1d9529..0000000000
--- a/changes/bug29175_035
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Major bugfixes (networking):
-    - Gracefully handle empty username/password fields in SOCKS5
-      username/password auth messsage and allow SOCKS5 handshake to
-      continue. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29241 b/changes/bug29241
deleted file mode 100644
index 7f25e154d1..0000000000
--- a/changes/bug29241
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (NSS, relay):
-    - When running with NSS, disable TLS 1.2 ciphersuites that use SHA384
-      for their PRF. Due to an NSS bug, the TLS key exporters for these
-      ciphersuites don't work -- which caused relays to fail to handshake
-      with one another when these ciphersuites were enabled.
-      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29244 b/changes/bug29244
deleted file mode 100644
index 6206a95463..0000000000
--- a/changes/bug29244
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (build, compatibility):
-    - Update Cargo.lock file to match the version made by the latest
-      version of Rust, so that "make distcheck" will pass again.
-      Fixes bug 29244; bugfix on 0.3.3.4-alpha.
diff --git a/changes/bug29530_035 b/changes/bug29530_035
deleted file mode 100644
index 6dfcd51e7b..0000000000
--- a/changes/bug29530_035
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (testing):
-    - Downgrade some LOG_ERR messages in the address/* tests to warnings.
-      The LOG_ERR messages were occurring when we had no configured network.
-      We were failing the unit tests, because we backported 28668 to 0.3.5.8,
-      but did not backport 29530. Fixes bug 29530; bugfix on 0.3.5.8.
diff --git a/changes/bug29599 b/changes/bug29599
deleted file mode 100644
index 14e2f5d077..0000000000
--- a/changes/bug29599
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (memory management, testing):
-    - Stop leaking parts of the shared random state in the shared-random unit
-      tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug29601 b/changes/bug29601
deleted file mode 100644
index c4ba5fbc8b..0000000000
--- a/changes/bug29601
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (Windows, CI):
-    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit Windows
-      Server 2012 R2 job. The remaining 2 jobs still provide coverage of
-      64/32-bit, and Windows Server 2016/2012 R2. Also set fast_finish, so
-      failed jobs terminate the build immediately.
-      Fixes bug 29601; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug29665 b/changes/bug29665
deleted file mode 100644
index d89046faf5..0000000000
--- a/changes/bug29665
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (single onion services):
-    - Allow connections to single onion services to remain idle without
-      being disconnected. Relays acting as rendezvous points for
-      single onion services were mistakenly closing idle established
-      rendezvous circuits after 60 seconds, thinking that they are unused
-      directory-fetching circuits that had served their purpose. Fixes
-      bug 29665; bugfix on 0.2.1.26.
diff --git a/changes/bug29670 b/changes/bug29670
deleted file mode 100644
index 00b0c33327..0000000000
--- a/changes/bug29670
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (configuration, proxies):
-    - Fix a bug that prevented us from supporting SOCKS5 proxies that want
-      authentication along with configued (but unused!)
-      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
diff --git a/changes/bug29703 b/changes/bug29703
deleted file mode 100644
index 0e17ee45e6..0000000000
--- a/changes/bug29703
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9.
-      We need a recent test-network.sh to use new chutney features in CI.
-      Fixes bug 29703; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug29706_minimal b/changes/bug29706_minimal
deleted file mode 100644
index 9d4a43326c..0000000000
--- a/changes/bug29706_minimal
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory management, testing):
-    - Stop leaking parts of the shared random state in the shared-random unit
-      tests. The previous fix in 29599 was incomplete.
-      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug29819 b/changes/bug29819
deleted file mode 100644
index d37ac83d66..0000000000
--- a/changes/bug29819
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Minor bugfixes (linux seccomp sandbox):
-    - Correct how we use libseccomp. Particularly, stop assuming that
-      rules are applied in a particular order or that more rules are
-      processed after the first match. Neither is the case! In libseccomp
-      <2.4.0 this lead to some rules having no effect. Libseccomp 2.4.0
-      changed how rules are generated leading to a different ordering
-      which in turn lead to a fatal crash during startup. Fixes bug
-      29819; bugfix on 0.2.5.1-alpha. Patch by Peter Gerber.
diff --git a/changes/bug29875 b/changes/bug29875
deleted file mode 100644
index 58a1c871cd..0000000000
--- a/changes/bug29875
+++ /dev/null
@@ -1,11 +0,0 @@
-  o Major bugfixes (bridges):
-    - Do not count previously configured working bridges towards our total of
-      working bridges. Previously, when Tor's list of bridges changed, it
-      would think that the old bridges were still usable, and delay fetching
-      router descriptors for the new ones.  Fixes part of bug 29875; bugfix
-      on 0.3.0.1-alpha.
-    - Consider our directory information to have changed when our list of
-      bridges changes. Previously, Tor would not re-compute the status of its
-      directory information when bridges changed, and therefore would not
-      realize that it was no longer able to build circuits. Fixes part of bug
-      29875.
diff --git a/changes/bug29922 b/changes/bug29922
deleted file mode 100644
index dacb951097..0000000000
--- a/changes/bug29922
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing, windows):
-    - Fix a test failure caused by an unexpected bug warning in
-      our test for tor_gmtime_r(-1). Fixes bug 29922;
-      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug30011 b/changes/bug30011
deleted file mode 100644
index 4c9069e291..0000000000
--- a/changes/bug30011
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (CI):
-    - Terminate test-stem if it takes more than 9.5 minutes to run.
-      (Travis terminates the job after 10 minutes of no output.)
-      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug30021 b/changes/bug30021
deleted file mode 100644
index 2a887f3cf2..0000000000
--- a/changes/bug30021
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Minor bugfixes (TLS protocol, integration tests):
-    - When classifying a client's selection of TLS ciphers, if the client
-      ciphers are not yet available, do not cache the result. Previously,
-      we had cached the unavailability of the cipher list and never looked
-      again, which in turn led us to assume that the client only supported
-      the ancient V1 link protocol.  This, in turn, was causing Stem
-      integration tests to stall in some cases.
-      Fixes bug 30021; bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug30040 b/changes/bug30040
deleted file mode 100644
index 7d80528a10..0000000000
--- a/changes/bug30040
+++ /dev/null
@@ -1,9 +0,0 @@
-  o Minor bugfixes (security):
-    - Fix a potential double free bug when reading huge bandwidth files. The
-      issue is not exploitable in the current Tor network because the
-      vulnerable code is only reached when directory authorities read bandwidth
-      files, but bandwidth files come from a trusted source (usually the
-      authorities themselves). Furthermore, the issue is only exploitable in
-      rare (non-POSIX) 32-bit architectures which are not used by any of the
-      current authorities. Fixes bug 30040; bugfix on 0.3.5.1-alpha. Bug found
-      and fixed by Tobias Stoeckmann.
diff --git a/changes/bug30041 b/changes/bug30041
deleted file mode 100644
index 801c8f67ac..0000000000
--- a/changes/bug30041
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (hardening):
-    - Verify in more places that we are not about to create a buffer
-      with more than INT_MAX bytes, to avoid possible OOB access in the event
-      of bugs.  Fixes bug 30041; bugfix on 0.2.0.16.  Found and fixed by
-      Tobias Stoeckmann.
diff --git a/changes/bug30148 b/changes/bug30148
deleted file mode 100644
index 7d0257e3fe..0000000000
--- a/changes/bug30148
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory leak):
-    - Avoid a minor memory leak that could occur on relays when
-      creating a keys directory failed. Fixes bug 30148; bugfix on
-      0.3.3.1-alpha.
diff --git a/changes/bug30189 b/changes/bug30189
deleted file mode 100644
index f8c932a5f9..0000000000
--- a/changes/bug30189
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation, unusual configuration):
-    - Avoid failures when building with ALL_BUGS_ARE_FAILED due to
-      missing declarations of abort(), and prevent other such failures
-      in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug30190 b/changes/bug30190
deleted file mode 100644
index e2352c3b9c..0000000000
--- a/changes/bug30190
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (lib):
-    do not log a warning for OpenSSL versions that should be compatible
-    Fixes bug 30190; bugfix on 0.2.4.2-alpha
diff --git a/changes/bug30316 b/changes/bug30316
deleted file mode 100644
index 3e396318ad..0000000000
--- a/changes/bug30316
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (directory authority):
-    - Move the "bandwidth-file-headers" line in directory authority votes
-      so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix on
-      0.3.5.1-alpha.
diff --git a/changes/bug30344 b/changes/bug30344
deleted file mode 100644
index 37561bf944..0000000000
--- a/changes/bug30344
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (connection):
-    - Avoid reading data from closed connections, which can cause needless
-      loops in libevent and infinite loops in Shadow. Fixes bug 30344; bugfix
-      on 0.1.1.1-alpha.
diff --git a/changes/bug30452 b/changes/bug30452
deleted file mode 100644
index 2bb401d87d..0000000000
--- a/changes/bug30452
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (compile-time modules):
-    - Add a --list-modules command to print a list of which compile-time
-      modules are enabled. Closes ticket 30452.
diff --git a/changes/bug30475 b/changes/bug30475
deleted file mode 100644
index 839597b885..0000000000
--- a/changes/bug30475
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes ():
-    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
-      implemenation) when failing to load a hidden service client authorization
-      file.  Fixes bug 30475; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug30561 b/changes/bug30561
deleted file mode 100644
index afb3f02c62..0000000000
--- a/changes/bug30561
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (portability):
-    - Avoid crashing in our tor_vasprintf() implementation on systems that
-      define neither vasprintf() nor _vscprintf(). (This bug has been here
-      long enough that we question whether people are running Tor on such
-      systems, but we're applying the fix out of caution.) Fixes bug 30561;
-      bugfix on 0.2.8.2-alpha. Found and fixed by Tobias Stoeckmann.
diff --git a/changes/bug30713 b/changes/bug30713
deleted file mode 100644
index e00b98da65..0000000000
--- a/changes/bug30713
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (testing):
-    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environmental variable is
-      set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
-    - Skip test_rebind on macOS in Travis, because it is unreliable on
-      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug30744 b/changes/bug30744
deleted file mode 100644
index 9f07d4855f..0000000000
--- a/changes/bug30744
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (continuous integration):
-    - Allow the test-stem job to fail in Travis, because it sometimes hangs.
-      Fixes bug 30744; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug30781 b/changes/bug30781
deleted file mode 100644
index 7c7adf470e..0000000000
--- a/changes/bug30781
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (directory authorities):
-    - Stop crashing after parsing an unknown descriptor purpose annotation.
-      We think this bug can only be triggered by modifying a local file.
-      Fixes bug 30781; bugfix on 0.2.0.8-alpha.
diff --git a/changes/bug30894 b/changes/bug30894
deleted file mode 100644
index 64c14c4e6d..0000000000
--- a/changes/bug30894
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory leaks):
-    - Fix a trivial memory leak when parsing an invalid value
-      from a download schedule in the configuration. Fixes bug
-      30894; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug30916 b/changes/bug30916
deleted file mode 100644
index b006bfc75d..0000000000
--- a/changes/bug30916
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (relay):
-    - Avoid crashing when starting with a corrupt keys directory where
-      the old ntor key and the new ntor key are identical. Fixes bug 30916;
-      bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug31003 b/changes/bug31003
deleted file mode 100644
index 6c75163380..0000000000
--- a/changes/bug31003
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (crash on exit):
-    - Avoid a set of possible code paths that could use try to use freed memory
-      in routerlist_free() while Tor was exiting.  Fixes bug 31003; bugfix on
-      0.1.2.2-alpha.
diff --git a/changes/bug31107 b/changes/bug31107
deleted file mode 100644
index 9652927c30..0000000000
--- a/changes/bug31107
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (logging, protocol violations):
-    - Do not log a nonfatal assertion failure when receiving a VERSIONS
-      cell on a connection using the obsolete v1 link protocol. Log a
-      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
diff --git a/changes/bug31343 b/changes/bug31343
deleted file mode 100644
index 17a8057ead..0000000000
--- a/changes/bug31343
+++ /dev/null
@@ -1,9 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Avoid using labs() on time_t, which can cause compilation warnings
-      on 64-bit Windows builds.  Fixes bug 31343; bugfix on 0.2.4.4-alpha.
-
-  o Minor bugfixes (clock skew detection):
-    - Don't believe clock skew results from NETINFO cells that appear to
-      arrive before the VERSIONS cells they are responding to were sent.
-      Previously, we would accept them up to 3 minutes "in the past".
-      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
diff --git a/changes/bug31408 b/changes/bug31408
deleted file mode 100644
index 3e4ffa927d..0000000000
--- a/changes/bug31408
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (torrc):
-    - Stop ignoring torrc options after an %include directive, when the
-      included directory ends with a file that does not contain any config
-      options. (But does contain comments or whitespace.)
-      Fixes bug 31408; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug31463 b/changes/bug31463
deleted file mode 100644
index d85c0887c3..0000000000
--- a/changes/bug31463
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (rust):
-    - Correctly exclude a redundant rust build job in Travis. Fixes bug 31463;
-      bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug31571 b/changes/bug31571
deleted file mode 100644
index 86de3537ba..0000000000
--- a/changes/bug31571
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (error handling):
-    - Report the tor version whenever an assertion fails. Previously, we only
-      reported the Tor version on some crashes, and some non-fatal assertions.
-      Fixes bug 31571; bugfix on 0.3.5.1-alpha.
-    - On abort, try harder to flush the output buffers of log messages. On
-      some platforms (macOS), log messages can be discarded when the process
-      terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug31657 b/changes/bug31657
deleted file mode 100644
index 08e9d95fdf..0000000000
--- a/changes/bug31657
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (guards):
-    - When tor is missing descriptors for some primary entry guards, make the
-      log message less alarming. It's normal for descriptors to expire, as long
-      as tor fetches new ones soon after. Fixes bug 31657;
-      bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug31837 b/changes/bug31837
deleted file mode 100644
index 0f976edfe0..0000000000
--- a/changes/bug31837
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (testing):
-    - When testing port rebinding, don't busy-wait for tor to log. Instead,
-      actually sleep for a short time before polling again. Also improve the
-      formatting of control commands and log messages.
-      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug31884 b/changes/bug31884
deleted file mode 100644
index ddb6c50d74..0000000000
--- a/changes/bug31884
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (Appveyor CI):
-    - Avoid spurious errors when Appveyor CI fails before the install step.
-      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
diff --git a/changes/bug31939 b/changes/bug31939
deleted file mode 100644
index a36ea495d6..0000000000
--- a/changes/bug31939
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (tls, logging):
-    - Log TLS read buffer length bugs once, rather than filling the logs
-      with similar warnings. Fixes bug 31939; bugfix on 0.3.0.4-rc.
diff --git a/changes/bug32449 b/changes/bug32449
deleted file mode 100644
index 213d8a1014..0000000000
--- a/changes/bug32449
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (CI, appveyor):
-    - Install the mingw OpenSSL package in Appveyor. This makes sure that
-      the OpenSSL headers and libraries match in Tor's Appveyor builds.
-      (This bug was triggered by an Appveyor image update.)
-      Fixes bug 32449; bugfix on 0.3.5.6-rc.
diff --git a/changes/bug32709 b/changes/bug32709
new file mode 100644
index 0000000000..d00b112be6
--- /dev/null
+++ b/changes/bug32709
@@ -0,0 +1,4 @@
+  o Major features (v3 onion services):
+    - Allow v3 onion services to act as OnionBalance backend instances using
+      the HiddenServiceOnionBalanceInstance torrc option. Closes ticket 32709.
+
diff --git a/changes/bug32753 b/changes/bug32753
deleted file mode 100644
index 6f59c7729d..0000000000
--- a/changes/bug32753
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (bridges):
-    - Lowercase the value of BridgeDistribution from torrc before adding it to
-      the descriptor. Fixes bug 32753; bugfix on 0.3.2.3-alpha.
diff --git a/changes/bug32884 b/changes/bug32884
deleted file mode 100644
index 9ab1d24464..0000000000
--- a/changes/bug32884
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (embedded Tor):
-    - When starting Tor any time after the first time in a process, register
-      the thread in which it is running as the main thread.  Previously, we
-      only did this on Windows, which could lead to bugs like 23081 on
-      non-Windows platforms.  Fixes bug 32884; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug33032 b/changes/bug33032
deleted file mode 100644
index 0c665f25df..0000000000
--- a/changes/bug33032
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (key portability):
-    - When reading PEM-encoded key data, tolerate CRLF line-endings even if
-      we are not running on Windows. Previously, non-Windows hosts
-      would reject these line-endings in certain positions, making
-      certain key files hard to move from one host to another.
-      Fixes bug 33032; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug33093_logging b/changes/bug33093_logging
deleted file mode 100644
index e26e4a64af..0000000000
--- a/changes/bug33093_logging
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - If we encounter a bug when flushing a buffer to a TLS connection,
-      only log the bug once per invocation of the Tor process.  Previously we
-      would log with every occurrence, which could cause us to run out of
-      disk space.  Fixes bug 33093; bugfix on 0.3.2.2-alpha.
diff --git a/changes/bug33131 b/changes/bug33131
new file mode 100644
index 0000000000..bc5ef7bc2d
--- /dev/null
+++ b/changes/bug33131
@@ -0,0 +1,3 @@
+  o Minor bugfixes (mainloop):
+    - Better guard against growing a buffer past its maximum 2GB in size.
+      Fixes bug 33131; bugfix on 0.3.0.4-rc.
diff --git a/changes/bug33284 b/changes/bug33284
new file mode 100644
index 0000000000..e6aed4d2d4
--- /dev/null
+++ b/changes/bug33284
@@ -0,0 +1,4 @@
+  o Minor bugfixes (git scripts):
+    - Stop executing the checked-out pre-commit hook from the pre-push hook.
+      Instead, execute the copy in the user's git dir. Fixes bug 33284; bugfix
+      on 0.4.1.1-alpha.
diff --git a/changes/bug33285 b/changes/bug33285
new file mode 100644
index 0000000000..a4d06a7eb8
--- /dev/null
+++ b/changes/bug33285
@@ -0,0 +1,3 @@
+  o Minor bugfixes (protocol versions):
+    - Sort tor's supported protocol version lists, as recommended by the
+      tor directory specification. Fixes bug 33285; bugfix on 0.4.0.1-alpha.
diff --git a/changes/bug33531 b/changes/bug33531
new file mode 100644
index 0000000000..c4284c55c9
--- /dev/null
+++ b/changes/bug33531
@@ -0,0 +1,3 @@
+  o Minor bugfixes (logs):
+    - Remove surprising empty line in info-level log about circuit build
+      timeout. Fixes bug 33531; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug33673 b/changes/bug33673
deleted file mode 100644
index 37c00f2e6e..0000000000
--- a/changes/bug33673
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Testing:
-    - In our Appveyor Windows CI, copy required DLLs to test and app, before
-      running tor's tests. This ensures that tor.exe and test*.exe use the
-      correct version of each DLL. This fix is not required, but we hope it
-      will avoid DLL search issues in future.
-      Fixes bug 33673; bugfix on 0.3.4.2-alpha.
diff --git a/changes/bug33899 b/changes/bug33899
new file mode 100644
index 0000000000..b9b7d7cf13
--- /dev/null
+++ b/changes/bug33899
@@ -0,0 +1,9 @@
+  o Minor bugfixes (IPv6, relay):
+    - Consider IPv6 addresses when checking if a connection is canonical.
+      In 17604, relays assumed that a remote relay could consider an IPv6
+      connection canonical, but did not set the canonical flag on their side
+      of the connection. Fixes bug 33899; bugfix on 0.3.1.1-alpha.
+    - Log IPv6 addresses on connections where this relay is the responder.
+      Previously, responding relays would replace the remote IPv6 address with
+      the IPv4 address from the consensus.
+      Fixes bug 33899; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug33900 b/changes/bug33900
new file mode 100644
index 0000000000..c1649d2284
--- /dev/null
+++ b/changes/bug33900
@@ -0,0 +1,3 @@
+  o Minor bugfixes (IPv4, relay):
+    - Check for invalid zero IPv4 addresses and ports, when sending and
+      receiving extend cells. Fixes bug 33900; bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug33917 b/changes/bug33917
new file mode 100644
index 0000000000..6a8daa9e26
--- /dev/null
+++ b/changes/bug33917
@@ -0,0 +1,5 @@
+  o Minor bugfixes (logging, testing):
+    - Make all of tor's assertion macros support the ALL_BUGS_ARE_FATAL and
+      DISABLE_ASSERTS_IN_UNIT_TESTS debugging modes. Implements these modes
+      for IF_BUG_ONCE(). (It used to log a non-fatal warning, regardless of
+      the debugging mode.) Fixes bug 33917; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug33977 b/changes/bug33977
new file mode 100644
index 0000000000..b424a811a2
--- /dev/null
+++ b/changes/bug33977
@@ -0,0 +1,4 @@
+  o Minor bugfix (refactoring):
+    - Lift circuit_build_times_disabled out of circuit_expire_building loop to
+      save CPU time with many circuits open. Fixes bug 33977; bugfix on
+      0.3.5.9.
diff --git a/changes/bug34078 b/changes/bug34078
deleted file mode 100644
index 1015d24547..0000000000
--- a/changes/bug34078
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (compiler compatibility):
-    - Avoid compiler warnings from Clang 10 related to the use of
-      GCC-style "/* falls through */" comments.  Both Clang and GCC allow
-      __attribute__((fallthrough)) instead, so that's what we're using now.
-      Fixes bug 34078; bugfix on 0.3.1.3-alpha.
diff --git a/changes/bug34084 b/changes/bug34084
new file mode 100644
index 0000000000..524c4cf68e
--- /dev/null
+++ b/changes/bug34084
@@ -0,0 +1,3 @@
+  o Minor bugfixes (onion services v3):
+    - Avoid a non-fatal assert log in an edge-case of opening an intro circuit
+      as a client. Fixes bug 34084; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug34086 b/changes/bug34086
new file mode 100644
index 0000000000..245992f8f4
--- /dev/null
+++ b/changes/bug34086
@@ -0,0 +1,3 @@
+  o Minor bugfixes (onion service v3):
+    - Remove a BUG() warning that could trigger in certain unlikely edge-cases.
+      Fixes bug 34086; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug34130 b/changes/bug34130
new file mode 100644
index 0000000000..b1e5715fdf
--- /dev/null
+++ b/changes/bug34130
@@ -0,0 +1,4 @@
+  o Minor bugfixes (linux seccomp sandbox nss):
+    - Fix startup crash when tor is compiled with --enable-nss and 
+      sandbox support is enabled. Fixes bug 34130; bugfix on
+      0.3.5.1-alpha. Patch by Daniel Pinto.
diff --git a/changes/bug34233 b/changes/bug34233
new file mode 100644
index 0000000000..24c7869783
--- /dev/null
+++ b/changes/bug34233
@@ -0,0 +1,4 @@
+  o Minor bugfixes (portability):
+    - Fix a portability error in the configure script, where we
+      were using "==" instead of "=". Fixes bug 34233; bugfix on
+      0.4.3.5.
diff --git a/changes/bug34299 b/changes/bug34299
new file mode 100644
index 0000000000..464cf0d18a
--- /dev/null
+++ b/changes/bug34299
@@ -0,0 +1,3 @@
+  o Minor bugfixes (man page):
+    - Update the man page to reflect that MinUptimeHidServDirectoryV2
+      defaults to 96 hours. Fixes bug 34299; bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug34400 b/changes/bug34400
new file mode 100644
index 0000000000..e2b56688b9
--- /dev/null
+++ b/changes/bug34400
@@ -0,0 +1,5 @@
+  o Minor bugfixes (v2 onion services):
+    - For HSFETCH commands on v2 onion services addresses, check the length of
+      bytes decoded, not the base32 length. This takes the behavior introduced
+      in commit a517daa56f5848d25ba79617a1a7b82ed2b0a7c0 into consideration.
+      Fixes bug 34400; bugfix on 0.4.1.1-alpha. Patch by Neel Chauhan.
diff --git a/changes/bug40001 b/changes/bug40001
new file mode 100644
index 0000000000..0e3f454619
--- /dev/null
+++ b/changes/bug40001
@@ -0,0 +1,3 @@
+  o Minor features (entry guards):
+    - Reinstate support for GUARD NEW/UP/DOWN control port events.
+      Closes ticket 40001.
\ No newline at end of file
diff --git a/changes/bug40020 b/changes/bug40020
new file mode 100644
index 0000000000..ca6ee2b85b
--- /dev/null
+++ b/changes/bug40020
@@ -0,0 +1,9 @@
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Makes the seccomp sandbox allow the correct syscall for opendir
+      according to the running glibc version. The opendir function
+      either uses open or openat but the current code does not
+      differenciate between opendir and open calls. This adds a new
+      seccomp sandbox rule for opendir. This fixes crashes when 
+      reloading torrc with sandbox enabled when running on glibc
+      2.15 to 2.21 and 2.26. Patch from Daniel Pinto. Fixes bug 40020; 
+      bugfix on 0.3.5.11.
diff --git a/changes/bug40072 b/changes/bug40072
new file mode 100644
index 0000000000..2b82f3f18b
--- /dev/null
+++ b/changes/bug40072
@@ -0,0 +1,4 @@
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Fix startup crash with seccomp sandbox enabled when tor tries to
+      open the data directory. Patch from Daniel Pinto. Fixes bug 40072;
+      bugfix on 0.4.4.3-alpha-dev.
diff --git a/changes/bug40083 b/changes/bug40083
new file mode 100644
index 0000000000..db26017664
--- /dev/null
+++ b/changes/bug40083
@@ -0,0 +1,5 @@
+  o Minor bugfixes (relay, self-testing):
+    - When starting up as a relay, if we haven't been able to verify that
+      we're reachable, only launch reachability tests at most once a minute.
+      Previously, we had been launching tests up to once a second, which
+      was needlessly noisy. Fixes bug 40083; bugfix on 0.2.8.1-alpha.
diff --git a/changes/bug40095 b/changes/bug40095
new file mode 100644
index 0000000000..5c4b3a2b7e
--- /dev/null
+++ b/changes/bug40095
@@ -0,0 +1,4 @@
+  o Minor bugfixes (testing):
+    - When running the subsystem order check, use the python binary
+      configured with the PYTHON environment variable. Fixes bug 40095;
+      bugfix on 0.4.4.1-alpha.
diff --git a/changes/bug40105 b/changes/bug40105
new file mode 100644
index 0000000000..330b6a9744
--- /dev/null
+++ b/changes/bug40105
@@ -0,0 +1,3 @@
+  o Minor bugfixes (guard selection algorithm):
+    - Avoid needless guard-related warning when upgrading from 0.4.3 to 0.4.4.
+      Fixes bug 40105; bugfix on 0.4.4.1-alpha.
\ No newline at end of file
diff --git a/changes/bug40129 b/changes/bug40129
new file mode 100644
index 0000000000..80de5ef355
--- /dev/null
+++ b/changes/bug40129
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - Fix compiler warnings that would occur when building with
+      "--enable-all-bugs-are-fatal" and "--disable-module-relay"
+      at the same time. Fixes bug 40129; bugfix on 0.4.4.1-alpha.
diff --git a/changes/bug40179_part2 b/changes/bug40179_part2
new file mode 100644
index 0000000000..15dc861321
--- /dev/null
+++ b/changes/bug40179_part2
@@ -0,0 +1,4 @@
+  o Minor bugfixes (testing):
+    - Fix the config/parse_tcp_proxy_line test so that it works correctly on
+      systems where the DNS provider hijacks invalid queries.
+      Fixes part of bug 40179; bugfix on 0.4.3.1-alpha.
diff --git a/changes/bug4631 b/changes/bug4631
new file mode 100644
index 0000000000..be3dd2b43e
--- /dev/null
+++ b/changes/bug4631
@@ -0,0 +1,6 @@
+  o Minor bugfixes (directory authorities):
+    - Directory authorities reject votes that arrive too late. In particular,
+      once an authority has started fetching missing votes, it no longer
+      accepts new votes posted by other authorities. This change helps prevent
+      a consensus split, where only some authorities have the late vote.
+      Fixes bug 4631; bugfix on 0.2.0.5-alpha.
diff --git a/changes/chutney_ci b/changes/chutney_ci
deleted file mode 100644
index b17d587329..0000000000
--- a/changes/chutney_ci
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (continuous integration):
-    - Our Travis configuration now uses Chutney to run some network
-      integration tests automatically. Closes ticket 29280.
diff --git a/changes/cid1444119 b/changes/cid1444119
deleted file mode 100644
index bb6854e66f..0000000000
--- a/changes/cid1444119
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (C correctness):
-    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug 29824;
-      bugfix on 0.3.1.1-alpha. This is Coverity warning CID 1444119.
diff --git a/changes/doc31089 b/changes/doc31089
deleted file mode 100644
index 2fc0ba4f7d..0000000000
--- a/changes/doc31089
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation:
-    - Use RFC 2397 data URL scheme to embed image into tor-exit-notice.html
-      so that operators would no longer have to host it themselves.
-      Closes ticket 31089.
diff --git a/changes/doc33417 b/changes/doc33417
new file mode 100644
index 0000000000..0fc868fc65
--- /dev/null
+++ b/changes/doc33417
@@ -0,0 +1,3 @@
+  o Documentation (manpage):
+    - Refrain from mentioning nicknames in manpage section for MyFamily torrc
+      option. Resolves issue 33417.
diff --git a/changes/doc34133 b/changes/doc34133
new file mode 100644
index 0000000000..abe9db6148
--- /dev/null
+++ b/changes/doc34133
@@ -0,0 +1,6 @@
+  o Documentation:
+    - Correctly document that we search for a system torrc file before
+      Document the limitations of using %include on config files with 
+      seccomp sandbox enabled. No new files can be added to the 
+      %included directories. Fixes documentation bug 34133; bugfix 
+      on 0.3.1.1-alpha. Patch by Daniel Pinto.
diff --git a/changes/geoip-2018-11-06 b/changes/geoip-2018-11-06
deleted file mode 100644
index 5c18ea4244..0000000000
--- a/changes/geoip-2018-11-06
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
-      Country database. Closes ticket 28395.
-
diff --git a/changes/geoip-2018-12-05 b/changes/geoip-2018-12-05
deleted file mode 100644
index 20ccf2d8a5..0000000000
--- a/changes/geoip-2018-12-05
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2
-      Country database. Closes ticket 28744.
-
diff --git a/changes/geoip-2019-01-03 b/changes/geoip-2019-01-03
deleted file mode 100644
index 27ffb7f460..0000000000
--- a/changes/geoip-2019-01-03
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29012.
-
diff --git a/changes/geoip-2019-02-05 b/changes/geoip-2019-02-05
deleted file mode 100644
index 78ee6d4242..0000000000
--- a/changes/geoip-2019-02-05
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29478.
-
diff --git a/changes/geoip-2019-03-04 b/changes/geoip-2019-03-04
deleted file mode 100644
index c8ce5dad5d..0000000000
--- a/changes/geoip-2019-03-04
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the March 4 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29666.
-
diff --git a/changes/geoip-2019-04-02 b/changes/geoip-2019-04-02
deleted file mode 100644
index 7302d939f6..0000000000
--- a/changes/geoip-2019-04-02
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29992.
-
diff --git a/changes/geoip-2019-05-13 b/changes/geoip-2019-05-13
deleted file mode 100644
index 0a2fa18971..0000000000
--- a/changes/geoip-2019-05-13
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2
-      Country database. Closes ticket 30522.
-
diff --git a/changes/geoip-2019-06-10 b/changes/geoip-2019-06-10
deleted file mode 100644
index 2d1e065649..0000000000
--- a/changes/geoip-2019-06-10
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2
-      Country database. Closes ticket 30852.
-
diff --git a/changes/geoip-2019-10-01 b/changes/geoip-2019-10-01
deleted file mode 100644
index c7ed17b5c4..0000000000
--- a/changes/geoip-2019-10-01
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2
-      Country database. Closes ticket 31931.
-
diff --git a/changes/geoip-2019-11-06 b/changes/geoip-2019-11-06
deleted file mode 100644
index a470981012..0000000000
--- a/changes/geoip-2019-11-06
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the November 6 2019 Maxmind GeoLite2
-      Country database. Closes ticket 32440.
-
diff --git a/changes/geoip-2019-12-03 b/changes/geoip-2019-12-03
deleted file mode 100644
index ea62b6ee89..0000000000
--- a/changes/geoip-2019-12-03
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the December 3 2019 Maxmind GeoLite2
-      Country database. Closes ticket 32685.
-
diff --git a/changes/log_32868 b/changes/log_32868
deleted file mode 100644
index 34476078b2..0000000000
--- a/changes/log_32868
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (debugging, directory system):
-    - Don't crash when we find a non-guard with a guard-fraction value set.
-      Instead, log a bug warning, in an attempt to figure out how this
-      happened. Diagnostic for ticket 32868.
diff --git a/changes/rust_asan b/changes/rust_asan
deleted file mode 100644
index 1ca7ae6888..0000000000
--- a/changes/rust_asan
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Major bugfixes (compilation, rust):
-    - Rust tests can now build and run successfully with the
-      --enable-fragile-hardening option enabled.
-      Doing this currently requires the rust beta channel; it will
-      be possible with stable rust as of rust version 1.31 is out.
-      Patch from Alex Crichton.
-      Fixes bugs 27272, 27273, and 27274.
-      Bugfix on 0.3.1.1-alpha.
diff --git a/changes/ticket19566 b/changes/ticket19566
deleted file mode 100644
index bf7071e660..0000000000
--- a/changes/ticket19566
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Code simplification and refactoring (shared random, dirauth):
-    - Change many tor_assert() to use BUG() instead. The idea is to not crash
-      a dirauth but rather scream loudly with a stacktrace and let it continue
-      run. The shared random subsystem is very resilient and if anything wrong
-      happens with it, at worst a non coherent value will be put in the vote
-      and discarded by the other authorities. Closes ticket 19566.
diff --git a/changes/ticket24308 b/changes/ticket24308
new file mode 100644
index 0000000000..e614785265
--- /dev/null
+++ b/changes/ticket24308
@@ -0,0 +1,6 @@
+  o Minor features (denial-of-service memory limiter):
+    - Allow the user to configure even lower values for the MaxMemInQueues
+      parameter.  Relays now enforce a minimum of 64 MB, when previously
+      the minimum was 256 MB.  On clients, there is no minimum. Relays and
+      clients will both warn if the value is set so low that Tor is likely
+      to stop working. Closes ticket 24308.
diff --git a/changes/ticket24803 b/changes/ticket24803
deleted file mode 100644
index e76a9eeab9..0000000000
--- a/changes/ticket24803
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features (fallback directory list):
-    - Replace the 150 fallbacks originally introduced in Tor 0.3.3.1-alpha in
-      January 2018 (of which ~115 were still functional), with a list of
-      157 fallbacks (92 new, 65 existing, 85 removed) generated in
-      December 2018.  Closes ticket 24803.
diff --git a/changes/ticket24844 b/changes/ticket24844
new file mode 100644
index 0000000000..da55b4cf67
--- /dev/null
+++ b/changes/ticket24844
@@ -0,0 +1,4 @@
+  o Minor features (v3 onion servies):
+    - Add v3 onion service status to the dumpstats() call which is
+      triggered by a SIGUSR1 signal. Previously, we only did v2
+      onion services. Closes ticket 24844. Patch by Neel Chauhan.
diff --git a/changes/ticket27471 b/changes/ticket27471
deleted file mode 100644
index ffe77d268e..0000000000
--- a/changes/ticket27471
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (hidden service v3, client):
-    - When replacing a descriptor in the client cache with a newer descriptor,
-      make sure to close all client introduction circuits of the old
-      descriptor so we don't end up with unusable leftover circuits. Fixes bug
-      27471; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27751 b/changes/ticket27751
deleted file mode 100644
index 593c473b61..0000000000
--- a/changes/ticket27751
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Minor features (continuous integration):
-    - Add a Travis CI build for --enable-nss on Linux gcc. Closes ticket 27751.
diff --git a/changes/ticket27838 b/changes/ticket27838
deleted file mode 100644
index 1699730d7a..0000000000
--- a/changes/ticket27838
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (hidden service v3):
-    - Build the service descriptor signing key certificate before uploading so
-      we always have a fresh one leaving no chances for it to expire service
-      side. Fixes bug 27838; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27913 b/changes/ticket27913
deleted file mode 100644
index 81ce725932..0000000000
--- a/changes/ticket27913
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing:
-    - Add new CI job to Travis configuration that runs stem-based
-      integration tests. Closes ticket 27913.
diff --git a/changes/ticket27995 b/changes/ticket27995
deleted file mode 100644
index 8c75425749..0000000000
--- a/changes/ticket27995
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (hidden service v3, client authorization):
-    - Fix an assert() when adding a client authorization for the first time
-      and then sending a HUP signal to the service. Before that, tor would
-      stop abruptly. Fixes bug 27995; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket28026 b/changes/ticket28026
deleted file mode 100644
index a6911c2cab..0000000000
--- a/changes/ticket28026
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Documentation (hidden service manpage):
-    - Improve HSv3 client authorization by making some options more explicit
-      and detailed. Closes ticket 28026. Patch by "mtigas".
diff --git a/changes/ticket28113 b/changes/ticket28113
deleted file mode 100644
index 30dd825a9b..0000000000
--- a/changes/ticket28113
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (relay shutdown, systemd):
-    - Notify systemd of ShutdownWaitLength so it can be set to longer than
-      systemd's TimeoutStopSec. In tor's systemd service file, set
-      TimeoutSec to 60 seconds, to allow tor some time to shut down.
-      Fixes bug 28113; bugfix on 0.2.6.2-alpha.
diff --git a/changes/ticket28128 b/changes/ticket28128
deleted file mode 100644
index 6d08c74242..0000000000
--- a/changes/ticket28128
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation (hidden service manpage, sandbox):
-    - Document in the man page that changing ClientOnionAuthDir value or
-      adding a new file in the directory will not work at runtime upon sending
-      a HUP if Sandbox 1. Closes ticket 28128.
diff --git a/changes/ticket28208 b/changes/ticket28208
new file mode 100644
index 0000000000..8818aad1d5
--- /dev/null
+++ b/changes/ticket28208
@@ -0,0 +1,4 @@
+  o Testing:
+    - Use the "bridges+hs-v23" chutney network flavor in "make test-network".
+      This test requires a recent version of chutney (mid-February 2020).
+      Closes ticket 28208.
diff --git a/changes/ticket28229_diag b/changes/ticket28229_diag
deleted file mode 100644
index cd02b81faa..0000000000
--- a/changes/ticket28229_diag
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing:
-    - Increase logging and tag all log entries with timestamps
-      in test_rebind.py. Provides diagnostics for issue 28229.
diff --git a/changes/ticket28275 b/changes/ticket28275
deleted file mode 100644
index eadca86b7b..0000000000
--- a/changes/ticket28275
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation (hidden service v3, man page):
-    - Note in the man page that the only real way to fully revoke an onion
-      service v3 client authorization is by restarting the tor process. Closes
-      ticket 28275.
diff --git a/changes/ticket28318 b/changes/ticket28318
deleted file mode 100644
index 24dc1e9580..0000000000
--- a/changes/ticket28318
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (Windows, continuous integration):
-    - Build tor on Windows Server 2012 R2 and Windows Server 2016 using
-      Appveyor's CI. Closes ticket 28318.
diff --git a/changes/ticket28459 b/changes/ticket28459
deleted file mode 100644
index 6b5839b52b..0000000000
--- a/changes/ticket28459
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (continuous integration, Windows):
-    - Always show the configure and test logs, and upload them as build
-      artifacts, when building for Windows using Appveyor CI.
-      Implements 28459.
diff --git a/changes/ticket28574 b/changes/ticket28574
deleted file mode 100644
index 562810f511..0000000000
--- a/changes/ticket28574
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (continuous integration, Windows):
-    - Explicitly specify the path to the OpenSSL library and do not download
-      OpenSSL from Pacman, but instead use the library that is already provided
-      by AppVeyor. Fixes bug 28574; bugfix on master.
diff --git a/changes/ticket28668 b/changes/ticket28668
deleted file mode 100644
index 6386e0051f..0000000000
--- a/changes/ticket28668
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (testing):
-    - Treat all unexpected ERR and BUG messages as test failures.
-      Closes ticket 28668.
diff --git a/changes/ticket28669 b/changes/ticket28669
deleted file mode 100644
index 32c6114ffc..0000000000
--- a/changes/ticket28669
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfix (hidden service v3, client):
-    - Avoid a BUG() stacktrace in case a SOCKS connection is found waiting for
-      the descriptor while we do have it in the cache. There is a rare case
-      when this can happen. Now, tor will recover and retry the descriptor.
-      Fixes bug 28669; bugfix on 0.3.2.4-alpha.
-    
diff --git a/changes/ticket28795 b/changes/ticket28795
deleted file mode 100644
index 6ae72562bf..0000000000
--- a/changes/ticket28795
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features (fallback directory list):
-    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
-      in December 2018 (of which ~122 were still functional), with a
-      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
-      in June 2019. Closes ticket 28795.
diff --git a/changes/ticket28838 b/changes/ticket28838
deleted file mode 100644
index 6c290bf82b..0000000000
--- a/changes/ticket28838
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Minor features (performance):
-    - Remove about 96% of the work from the function that we run at
-      startup to test our curve25519_basepoint implementation.  Since
-      this function has yet to find an actual failure, we'll only
-      run it for 8 iterations instead of 200. Based on our profile
-      information, this change should save around 8% of our startup
-      time on typical desktops, and may have a similar effect on
-      other platforms.  Closes ticket 28838.
diff --git a/changes/ticket28851 b/changes/ticket28851
deleted file mode 100644
index bab0318662..0000000000
--- a/changes/ticket28851
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (performance):
-    - Stop re-validating our hardcoded Diffie-Hellman parameters on every
-      startup. Doing this wasted time and cycles, especially on low-powered
-      devices. Closes ticket 28851.
diff --git a/changes/ticket28879 b/changes/ticket28879
deleted file mode 100644
index 126420f6ca..0000000000
--- a/changes/ticket28879
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (correctness):
-    - Fix an unreached code-path where we checked the value of "hostname"
-      inside send_resolved_hostnam_cell(). Previously, we used it before
-      checking it; now we check it first. Fixes bug 28879; bugfix on
-      0.1.2.7-alpha.
diff --git a/changes/ticket28881 b/changes/ticket28881
deleted file mode 100644
index 1b015a6c37..0000000000
--- a/changes/ticket28881
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Code simplification and refactoring:
-    -  When parsing a port configuration, make it more
-       obvious to static analyzer tools that we will always initialize the
-       address. Closes ticket 28881.
diff --git a/changes/ticket28883 b/changes/ticket28883
deleted file mode 100644
index 1d8b6cb416..0000000000
--- a/changes/ticket28883
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Make sure that test_rebind.py actually obeys its timeout, even
-      when it receives a large number of log messages. Fixes bug 28883;
-      bugfix on 0.3.5.4-alpha.
diff --git a/changes/ticket28912 b/changes/ticket28912
deleted file mode 100644
index 4119b778bc..0000000000
--- a/changes/ticket28912
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (relay, directory):
-    - A connection serving directory information wouldn't get reactivated after
-      the first chunk of data was sent (usually 32KB). Tor now always activate
-      the main loop event that goes through these connections as long as at
-      least one connection is still active. Fixes bug 28912; bugfix on
-      0.3.4.1-alpha. Patch by "cypherpunks3".
diff --git a/changes/ticket28924 b/changes/ticket28924
deleted file mode 100644
index 055a6cf285..0000000000
--- a/changes/ticket28924
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (compilation):
-    - When possible, place our warning flags in a separate file, to avoid
-      flooding verbose build logs to an unacceptable amount. Closes ticket
-      28924.
diff --git a/changes/ticket28970 b/changes/ticket28970
deleted file mode 100644
index 138c575fcc..0000000000
--- a/changes/ticket28970
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (clietn, hidden service v3):
-    - Fix a BUG() assertion that occurs within a very small race window between
-      a client intro circuit opens and its descriptor that gets cleaned up from
-      the cache. The circuit is now closed which will trigger a re-fetch of the
-      descriptor and continue the HS connection. Fixes bug 28970; bugfix on
-      0.3.2.1-alpha.
diff --git a/changes/ticket28973 b/changes/ticket28973
deleted file mode 100644
index b1d208ee51..0000000000
--- a/changes/ticket28973
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor features (OpenSSL bug workaround):
-    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
-      key export function from handling long labels.  When this bug
-      is detected, Tor will disable TLS 1.3.  We recommend upgrading to
-      a version of OpenSSL without this bug when it becomes available.
-      Closes ticket 28973.
diff --git a/changes/ticket29026 b/changes/ticket29026
deleted file mode 100644
index 1db873dfcf..0000000000
--- a/changes/ticket29026
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (compilation):
-    - Compile correctly when OpenSSL is built with engine support
-      disabled, or with deprecated APIs disabled. Closes ticket
-      29026. Patches from "Mangix".
diff --git a/changes/ticket29160 b/changes/ticket29160
deleted file mode 100644
index 8e11183064..0000000000
--- a/changes/ticket29160
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (tests):
-    - Do not log an error-level message if we fail to find an IPv6
-      network interface from the unit tests. Fixes bug 29160; bugfix on
-      0.2.7.3-rc.
diff --git a/changes/ticket29168 b/changes/ticket29168
deleted file mode 100644
index 65c5232f65..0000000000
--- a/changes/ticket29168
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (cell scheduler, KIST):
-    - Make KIST to always take into account the outbuf length when computing
-      what we can actually put in the outbuf. This could lead to the outbuf
-      being filled up and thus a possible memory DoS vector. TROVE-2019-001.
-      Fixes bug 29168; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket29435 b/changes/ticket29435
deleted file mode 100644
index d48ae98e4b..0000000000
--- a/changes/ticket29435
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (testing):
-    - Fix our gcov wrapper script to look for object files at the
-      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket29617 b/changes/ticket29617
deleted file mode 100644
index 4d50ea9627..0000000000
--- a/changes/ticket29617
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (out-of-memory handler):
-    - When purging the DNS cache because of an out-of-memory condition,
-      try purging just the older entries at first.  Previously, we would
-      purge the whole thing. Fixes bug 29617; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket29702 b/changes/ticket29702
deleted file mode 100644
index e1cc1f867b..0000000000
--- a/changes/ticket29702
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Testing:
-    - Specify torrc paths (with empty files) when launching tor in
-      integration tests; refrain from reading user and system torrcs.
-      Resolves issue 29702.
diff --git a/changes/ticket29806 b/changes/ticket29806
deleted file mode 100644
index 6afefd4c04..0000000000
--- a/changes/ticket29806
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor features (bandwidth authority):
-    - Make bandwidth authorities to ignore relays that are reported in the
-      bandwidth file with the key-value "vote=0".
-      This change allows to report the relays that were not measured due
-      some failure and diagnose the reasons without the bandwidth being included in the
-      bandwidth authorities vote.
-      Closes ticket 29806.
diff --git a/changes/ticket29962 b/changes/ticket29962
deleted file mode 100644
index e36cc0cf9a..0000000000
--- a/changes/ticket29962
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (continuous integration):
-    - On Travis Rust builds, cleanup Rust registry and refrain from caching
-      target/ directory to speed up builds. Resolves issue 29962.
diff --git a/changes/ticket30117 b/changes/ticket30117
deleted file mode 100644
index 5b6e6dabf7..0000000000
--- a/changes/ticket30117
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, tell timelimit to use stem's backtrace signals. And launch
-      python directly from timelimit, so python receives the signals from
-      timelimit, rather than make. Closes ticket 30117.
diff --git a/changes/ticket30213 b/changes/ticket30213
deleted file mode 100644
index acb7614807..0000000000
--- a/changes/ticket30213
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (continuous integration):
-    - Remove sudo configuration lines from .travis.yml as they are no longer
-      needed with current Travis build environment. Resolves issue 30213.
diff --git a/changes/ticket30234 b/changes/ticket30234
deleted file mode 100644
index 5a0076bad2..0000000000
--- a/changes/ticket30234
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, show stem's tor log after failure. Closes ticket 30234.
diff --git a/changes/ticket30454 b/changes/ticket30454
deleted file mode 100644
index 77c45d0feb..0000000000
--- a/changes/ticket30454
+++ /dev/null
@@ -1,10 +0,0 @@
-  o Major bugfixes (hidden service v3):
-    - An intro point could try to send an INTRODUCE_ACK with a status code
-      that it wasn't able to encode leading to a hard assert() of the relay.
-      Fortunately, that specific code path can not be reached thus this issue
-      can't be triggered. We've consolidated the ABI values into trunnel now.
-      Fixes bug 30454; bugfix on 0.3.0.1-alpha.
-    - HSv3 client will now be able to properly handle unknown status code from
-      a INTRODUCE_ACK cell (nack) even if they do not know it. The NACK
-      behavior will stay the same. This will allow us to extend status code if
-      we want in the future without breaking the normal client behavior.
diff --git a/changes/ticket30591 b/changes/ticket30591
deleted file mode 100644
index f97c024009..0000000000
--- a/changes/ticket30591
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, make stem log a controller trace to the console. And tail
-      stem's tor log after failure. Closes ticket 30591.
diff --git a/changes/ticket30694 b/changes/ticket30694
deleted file mode 100644
index 70dbf6481a..0000000000
--- a/changes/ticket30694
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, only run the stem tests that use a tor binary.
-      Closes ticket 30694.
diff --git a/changes/ticket30860 b/changes/ticket30860
deleted file mode 100644
index b946f735c4..0000000000
--- a/changes/ticket30860
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing:
-    - Run the chutney IPv6 networks as part of Travis CI.
-      Closes ticket 30860.
diff --git a/changes/ticket30871 b/changes/ticket30871
deleted file mode 100644
index 81c076bb02..0000000000
--- a/changes/ticket30871
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (circuit build, guard):
-    - When considering upgrading circuits from "waiting for guard" to "open",
-      always ignore the ones that are mark for close. Else, we can end up in
-      the situation where a subsystem is notified of that circuit opening but
-      still marked for close leading to undesirable behavior. Fixes bug 30871;
-      bugfix on 0.3.0.1-alpha.
diff --git a/changes/ticket31372_appveyor b/changes/ticket31372_appveyor
deleted file mode 100644
index e7bb03182e..0000000000
--- a/changes/ticket31372_appveyor
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (continuous integration):
-    - When building on Appveyor, pass the "-k" flag to make, so that
-      we are informed of all compilation failures, not just the first
-      one or two. Closes part of ticket 31372.
diff --git a/changes/ticket31372_travis b/changes/ticket31372_travis
deleted file mode 100644
index 403869b2ed..0000000000
--- a/changes/ticket31372_travis
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (continuous integration):
-    - When building on Travis, pass the "-k" flag to make, so that
-      we are informed of all compilation failures, not just the first
-      one or two. Closes part of ticket 31372.
diff --git a/changes/ticket31374 b/changes/ticket31374
deleted file mode 100644
index e8eef9cd49..0000000000
--- a/changes/ticket31374
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation warning):
-    - Fix a compilation warning on Windows about casting a function
-      pointer for GetTickCount64(). Fixes bug 31374; bugfix on
-      0.2.9.1-alpha.
diff --git a/changes/ticket31406 b/changes/ticket31406
deleted file mode 100644
index 0ebe6f6c47..0000000000
--- a/changes/ticket31406
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (directory authority):
-    - A new IP address the directory authority "dizum" has been changed. Closes
-      ticket 31406;
diff --git a/changes/ticket31466 b/changes/ticket31466
deleted file mode 100644
index e535b4502e..0000000000
--- a/changes/ticket31466
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Rate-limit our the logging message about the obsolete .exit notation.
-      Previously, there was no limit on this warning, which could potentially
-      be triggered many times by a hostile website. Fixes bug 31466;
-      bugfix on 0.2.2.1-alpha.
diff --git a/changes/ticket31548 b/changes/ticket31548
deleted file mode 100644
index fef0b5d01f..0000000000
--- a/changes/ticket31548
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Major bugfixes (hidden service v3):
-    - Make onion service always use the exact amount of configured intro points
-      (or less due to node exlusion). Before, a service could sometimes pick
-      more intro points than configured with the
-      HiddenServiceNumIntroductionPoints option. Fixes bug 31548; bugfix on
-      0.3.2.1-alpha.
-
diff --git a/changes/ticket31554 b/changes/ticket31554
deleted file mode 100644
index 73f4159ff3..0000000000
--- a/changes/ticket31554
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (stem tests):
-    - Change "make test-stem" so it only runs the stem tests that use tor.
-      This change makes test-stem faster and more reliable.
-      Closes ticket 31554.
diff --git a/changes/ticket31576 b/changes/ticket31576
new file mode 100644
index 0000000000..ab984cf3d4
--- /dev/null
+++ b/changes/ticket31576
@@ -0,0 +1,3 @@
+  o Removed features:
+    - Stop shipping contrib/dist/rc.subr file, as it is not being used on
+      FreeBSD anymore. Closes issue 31576.
diff --git a/changes/ticket31634 b/changes/ticket31634
new file mode 100644
index 0000000000..2777595036
--- /dev/null
+++ b/changes/ticket31634
@@ -0,0 +1,4 @@
+  o Minor features (testing, architeture):
+    - Our test scripts now double-check that subsystem initialization order
+      is consistent with the inter-module dependencies established by our
+      .may_include files.  Implements ticket 31634.
diff --git a/changes/ticket31673 b/changes/ticket31673
deleted file mode 100644
index 3b2bb4a46e..0000000000
--- a/changes/ticket31673
+++ /dev/null
@@ -1,3 +0,0 @@
-  o New system requirements (build system):
-    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows system.
-      Closes 31673;
diff --git a/changes/ticket31687_1 b/changes/ticket31687_1
deleted file mode 100644
index 2f4d440974..0000000000
--- a/changes/ticket31687_1
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Suppress spurious float-conversion warnings from GCC when calling
-      floating-point classifier functions on FreeBSD.  Fixes part of bug
-      31687; bugfix on 0.3.1.5-alpha.
diff --git a/changes/ticket31687_2 b/changes/ticket31687_2
deleted file mode 100644
index eadc698275..0000000000
--- a/changes/ticket31687_2
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6):
-    - When extracting an IPv6 address from a PF-based proxy, verify
-      that we are actually configured to receive an IPv6 address,
-      and log an internal error if not. Fixes part of bug 31687;
-      bugfix on 0.2.3.4-alpha.
diff --git a/changes/ticket31812 b/changes/ticket31812
new file mode 100644
index 0000000000..869e494892
--- /dev/null
+++ b/changes/ticket31812
@@ -0,0 +1,4 @@
+  o Documentation:
+    - Replace most http:// URLs in our code and documentation with https://
+      URLs.  (We have left unchanged the code in src/ext/, and the text in
+      LICENSE.)  Closes ticket 31812. Patch from Jeremy Rand.
diff --git a/changes/ticket31859 b/changes/ticket31859
deleted file mode 100644
index dbc591e00b..0000000000
--- a/changes/ticket31859
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing:
-    - Simplify the Travis CI build matrix, and optimise for build time.
-      Closes ticket 31859.
diff --git a/changes/ticket31919_bionic b/changes/ticket31919_bionic
deleted file mode 100644
index eb41644555..0000000000
--- a/changes/ticket31919_bionic
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features (continuous integration):
-    - Use Ubuntu Bionic images for our Travis CI builds, so we can get
-      a recent version of coccinelle. But leave chutney on Ubuntu Trusty,
-      until we can fix some Bionic permissions issues (see ticket 32240).
-      Related to ticket 31919.
diff --git a/changes/ticket32058 b/changes/ticket32058
deleted file mode 100644
index b40bcda416..0000000000
--- a/changes/ticket32058
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (mainloop, periodic events):
-    - Periodic events enabled flag was not unset properly when shutting down tor
-      cleanly. This had the side effect to not re-enable periodic events when
-      tor_api.h is used to relaunch tor after a shutdown. Fixes bug 32058;
-      bugfix on 0.3.3.1-alpha.
diff --git a/changes/ticket32086 b/changes/ticket32086
deleted file mode 100644
index b9312c2bea..0000000000
--- a/changes/ticket32086
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing:
-    - Use Windows Server 2019 instead of Windows Server 2016 in our
-      Appveyor builds. Closes ticket 32086.
diff --git a/changes/ticket32088 b/changes/ticket32088
new file mode 100644
index 0000000000..0d4fc74754
--- /dev/null
+++ b/changes/ticket32088
@@ -0,0 +1,13 @@
+  o Major features (Proposal 310, performance + security):
+    - Implements Proposal 310 - Bandaid on guard selection.
+      Proposal 310 solves a load-balancing issue within Prop271 which strongly
+      impact experimental research with Shadow.
+      Security improvement: Proposal 310 prevents any newly Guard relay to
+      have a chance to get into the primary list of older Tor clients,
+      except if the N first sampled guards of these clients are unreachable.
+      Implements recommendation from 32088.
+      
+      Proposal 310 is linked to the CLAPS project researching optimal
+      client location-aware path selections. This project is a collaboration
+      between the UCLouvain Crypto Group, the U.S. Naval Research Laboratory and
+      Princeton University.
diff --git a/changes/ticket32143 b/changes/ticket32143
new file mode 100644
index 0000000000..7f8a809ba5
--- /dev/null
+++ b/changes/ticket32143
@@ -0,0 +1,4 @@
+  o Minor features (continuous integration):
+    - Run unit-test and integration test (Stem, Chutney) jobs with
+      ALL_BUGS_ARE_FATAL macro being enabled on Travis and Appveyor.
+      Resolves ticket 32143.
diff --git a/changes/ticket32240 b/changes/ticket32240
deleted file mode 100644
index 35cc3df27e..0000000000
--- a/changes/ticket32240
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Testing:
-    - Turn off Tor's Sandbox in Chutney jobs, and run those jobs on Ubuntu
-      Bionic. Turning off the Sandbox is a work-around, until we fix the
-      sandbox errors in 32722. Closes ticket 32240.
diff --git a/changes/ticket32241 b/changes/ticket32241
deleted file mode 100644
index 4243cec175..0000000000
--- a/changes/ticket32241
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, use Xcode 11.2 on macOS 10.14. Closes ticket 32241.
diff --git a/changes/ticket32242 b/changes/ticket32242
deleted file mode 100644
index d63d5a586e..0000000000
--- a/changes/ticket32242
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Testing (continuous integration):
-    - Use zstd in our Travis Linux builds. Closes ticket 32242.
diff --git a/changes/ticket32407 b/changes/ticket32407
deleted file mode 100644
index badb09abfe..0000000000
--- a/changes/ticket32407
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (crash):
-    - When running Tor with an option like --verify-config or --dump-config
-      that does not start the event loop, avoid crashing if we try to exit
-      early because of an error. Fixes bug 32407; bugfix on 0.3.3.1-alpha.
diff --git a/changes/ticket32500 b/changes/ticket32500
deleted file mode 100644
index 2c0f35df72..0000000000
--- a/changes/ticket32500
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Testing:
-    - Require C99 standards-conforming code in Travis CI, but allow GNU gcc
-      extensions. Also activates clang's -Wtypedef-redefinition warnings.
-      Build some jobs with -std=gnu99, and some jobs without.
-      Closes ticket 32500.
diff --git a/changes/ticket32542 b/changes/ticket32542
new file mode 100644
index 0000000000..c52335b059
--- /dev/null
+++ b/changes/ticket32542
@@ -0,0 +1,3 @@
+  o Minor feature (onion service client, SOCKS5):
+    - Add 3 new SocksPort ExtendedErrors (F2, F3, F7) that reports back new type
+      of onion service connection failures. Closes ticket 32542.
diff --git a/changes/ticket32622 b/changes/ticket32622
new file mode 100644
index 0000000000..1c663567fd
--- /dev/null
+++ b/changes/ticket32622
@@ -0,0 +1,5 @@
+  o Minor features (bootstrap reporting):
+    - Report more detailed reasons for bootstrap failure when the failure
+      happens due to a TLS error. Previously we would just call these errors
+      "MISC" when they happened during read, and "DONE" when they
+      happened during any other TLS operation. Closes ticket 32622.
diff --git a/changes/ticket32629 b/changes/ticket32629
deleted file mode 100644
index 740746c572..0000000000
--- a/changes/ticket32629
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Testing:
-    - Re-enable the Travis CI macOS Chutney build, but allow the job to finish
-      before it finishes, because the Travis macOS jobs are slow.
-      Closes ticket 32629.
diff --git a/changes/ticket32696 b/changes/ticket32696
new file mode 100644
index 0000000000..8f56fc394e
--- /dev/null
+++ b/changes/ticket32696
@@ -0,0 +1,7 @@
+  o Minor features (directory authority):
+    - Authorities now recommend protocol versions that are supported
+      by Tor 0.3.5 and later. (Earlier versions of Tor have been
+      deprecated since January of this year.)  This recommendation
+      will cause older clients and relays to give a warning on startup,
+      or when they download a consensus directory.
+      Closes ticket 32696.
diff --git a/changes/ticket32720 b/changes/ticket32720
new file mode 100644
index 0000000000..87c540b7ff
--- /dev/null
+++ b/changes/ticket32720
@@ -0,0 +1,4 @@
+  o Minor features (directory):
+    - Remember the number of bytes we have downloaded for each directory
+      purpose while bootstrapping, and while fully bootstrapped. Log this
+      information as part of the heartbeat message. Closes ticket 32720.
diff --git a/changes/ticket32873 b/changes/ticket32873
new file mode 100644
index 0000000000..65ea1f64ad
--- /dev/null
+++ b/changes/ticket32873
@@ -0,0 +1,6 @@
+  o Minor features (control port):
+    - Return a descriptive error message from the 'GETINFO
+      status/fresh-relay-descs' command on the control port.
+      Previously, we returned a generic error of "Error
+      generating descriptor". Closes ticket 32873. Patch by
+      Neel Chauhan.
diff --git a/changes/ticket32888 b/changes/ticket32888
new file mode 100644
index 0000000000..ce7fb40b30
--- /dev/null
+++ b/changes/ticket32888
@@ -0,0 +1,4 @@
+  o Minor features (logging):
+    - When trying to find our own address, add debug-level logging
+      to report the sources of candidate addresses.  Closes ticket
+      32888.
diff --git a/changes/ticket32905 b/changes/ticket32905
new file mode 100644
index 0000000000..6f420ec693
--- /dev/null
+++ b/changes/ticket32905
@@ -0,0 +1,6 @@
+  o Removed features:
+    - Remove the ClientAutoIPv6ORPort option. This option attempted
+      to randomly choose between IPv4 and IPv6 for client connections,
+      and isn't a true implementation of Happy Eyeballs. Often, this
+      option failed on IPv4-only or IPv6-only connections. Closes
+      ticket 32905. Patch by Neel Chauhan.
diff --git a/changes/ticket32994 b/changes/ticket32994
new file mode 100644
index 0000000000..43a32afa78
--- /dev/null
+++ b/changes/ticket32994
@@ -0,0 +1,3 @@
+  o Code simplification and refactoring:
+    - Rewrite port_parse_config() to use the default port flags from
+      port_cfg_new(). Closes ticket 32994. Patch by MrSquanchee.
diff --git a/changes/ticket33014 b/changes/ticket33014
new file mode 100644
index 0000000000..885051d9cf
--- /dev/null
+++ b/changes/ticket33014
@@ -0,0 +1,3 @@
+  o Code simplification and refactoring (onion service):
+    - Refactor configuration parsing to use the new config subsystem code.
+      Closes ticket 33014.
diff --git a/changes/ticket33075 b/changes/ticket33075
deleted file mode 100644
index 69698d90b3..0000000000
--- a/changes/ticket33075
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Testing:
-    - Stop allowing failures on the Travis CI stem tests job. It looks like all
-      the stem hangs we were seeing are now fixed, but let's make sure we see
-      them if they happen again. Closes ticket 33075.
diff --git a/changes/ticket33119 b/changes/ticket33119
deleted file mode 100644
index 11c20bc7a2..0000000000
--- a/changes/ticket33119
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Major bugfixes (security, denial-of-service):
-    - Fix a denial-of-service bug that could be used by anyone to consume a
-      bunch of CPU on any Tor relay or authority, or by directories to
-      consume a bunch of CPU on clients or hidden services. Because
-      of the potential for CPU consumption to introduce observable
-      timing patterns, we are treating this as a high-severity security
-      issue.  Fixes bug 33119; bugfix on 0.2.1.5-alpha. We are also tracking
-      this issue as TROVE-2020-002.
diff --git a/changes/ticket33192 b/changes/ticket33192
new file mode 100644
index 0000000000..97f976226b
--- /dev/null
+++ b/changes/ticket33192
@@ -0,0 +1,5 @@
+  o Minor feature (python):
+    - Stop assuming that /usr/bin/python exists. Instead of using a
+      hardcoded path in scripts that still use Python 2, use /usr/bin/env,
+      similarly to the scripts that use Python 3. Fixes bug 33192; bugfix
+      on 0.4.2.
\ No newline at end of file
diff --git a/changes/ticket33212 b/changes/ticket33212
deleted file mode 100644
index aeb09e0c67..0000000000
--- a/changes/ticket33212
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (rust, build):
-    - Fix a syntax warning given by newer versions of Rust, and creating
-      problems for our continuous integration.
-      Fixes bug 33212; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket33275 b/changes/ticket33275
new file mode 100644
index 0000000000..bff3a7a3ad
--- /dev/null
+++ b/changes/ticket33275
@@ -0,0 +1,5 @@
+  o Documentation (manpage):
+    - Alphabetize the Denial of Service Mitigation Options, Directory
+      Authority Server Options, Hidden Service Options, and Testing
+      Network Options sections of the tor(1) manpage.  Closes ticket
+      33275.  Work by Swati Thacker as part of Google Season of Docs.
diff --git a/changes/ticket33280 b/changes/ticket33280
new file mode 100644
index 0000000000..b90c3086ea
--- /dev/null
+++ b/changes/ticket33280
@@ -0,0 +1,4 @@
+  o Testing:
+    - Add test-network-ipv4 and test-network-ipv6 jobs to the Makefile.
+      These jobs run the IPv4-only and dual-stack chutney flavours from
+      test-network-all. Closes ticket 33280.
diff --git a/changes/ticket33300 b/changes/ticket33300
new file mode 100644
index 0000000000..9b0bdce372
--- /dev/null
+++ b/changes/ticket33300
@@ -0,0 +1,3 @@
+  o Testing:
+    - Add a basic IPv6 test to "make test-network". This test only runs when
+      the local machine has an IPv6 stack. Closes ticket 33300.
diff --git a/changes/ticket33303 b/changes/ticket33303
new file mode 100644
index 0000000000..b7ac7b5067
--- /dev/null
+++ b/changes/ticket33303
@@ -0,0 +1,4 @@
+  o Testing:
+    - Run the test-network-ipv6 Makefile target in the Travis CI IPv6 chutney
+      job. This job runs on macOS, so it's a bit slow.
+      Closes ticket 33303.
diff --git a/changes/ticket33316 b/changes/ticket33316
new file mode 100644
index 0000000000..25b0444078
--- /dev/null
+++ b/changes/ticket33316
@@ -0,0 +1,15 @@
+  o Minor bugfixes (initialization):
+    - Initialize the subsystems in our code in an order more closely
+      corresponding to their dependencies, so that every system is
+      initialized before the ones that (theoretically) depend on it.
+      Fixes bug 33316; bugfix on 0.4.0.1-alpha.
+
+  o Minor features (tests):
+    - Initialize all subsystems at the beginning of our unit test harness,
+      to avoid crashes due to uninitialized subsystems.
+      Follow-up from ticket 33316.
+
+  o Code simplification and refactoring:
+    - Merge the orconn and ocirc events into the "core" subsystem, which
+      manages or connections and origin circuits.  Previously they
+      were isolated in subsystems of their own.
diff --git a/changes/ticket33334 b/changes/ticket33334
new file mode 100644
index 0000000000..ada3cb284c
--- /dev/null
+++ b/changes/ticket33334
@@ -0,0 +1,5 @@
+  o Testing:
+    - Test v3 onion services to tor's mixed IPv4 chutney network. And add a
+      mixed IPv6 chutney network. These networks are used in the
+      test-network-all, test-network-ipv4, and test-network-ipv6 make targets.
+      Closes ticket 33334.
diff --git a/changes/ticket33339 b/changes/ticket33339
new file mode 100644
index 0000000000..75ccb3546f
--- /dev/null
+++ b/changes/ticket33339
@@ -0,0 +1,3 @@
+  o Minor feature (developer tools):
+    - Add a script to help check the alphabetical ordering of option
+      names in a manpage.  Closes ticket 33339.
diff --git a/changes/ticket33346 b/changes/ticket33346
new file mode 100644
index 0000000000..acbbae5169
--- /dev/null
+++ b/changes/ticket33346
@@ -0,0 +1,3 @@
+  o Minor features (linux seccomp2 sandbox):
+    - Permit the unlinkat() syscall, which some Libc implementations
+      use to implement unlink(). Closes ticket 33346.
diff --git a/changes/ticket33349 b/changes/ticket33349
new file mode 100644
index 0000000000..0458a72c8d
--- /dev/null
+++ b/changes/ticket33349
@@ -0,0 +1,4 @@
+  o Code simplification and refactoring:
+    - Updated comments in 'scheduler.c' to reflect old code changes,
+      and simplified the scheduler channel state change code. Closes
+      ticket 33349.
diff --git a/changes/ticket33361 b/changes/ticket33361
deleted file mode 100644
index bc9715d6a1..0000000000
--- a/changes/ticket33361
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfix (relay, configuration):
-    - Now warn if the ContactInfo field is not set and mention that the relay
-      might get rejected if so. Fixes bug 33361; bugfix on 0.1.1.10-alpha.
diff --git a/changes/ticket33366 b/changes/ticket33366
new file mode 100644
index 0000000000..1310c493c2
--- /dev/null
+++ b/changes/ticket33366
@@ -0,0 +1,3 @@
+  o Minor features (compilation size):
+    - Most Server-side DNS code is now disabled when building without
+      support for relay mode. Closes ticket 33366.
diff --git a/changes/ticket33368 b/changes/ticket33368
new file mode 100644
index 0000000000..ecc6f66f4e
--- /dev/null
+++ b/changes/ticket33368
@@ -0,0 +1,3 @@
+  o Minor features (client-only compilation):
+    - Disable more code related to the ext_orport protocol when compiling
+      without support for relay mode. Closes ticket 33368.
diff --git a/changes/ticket33369 b/changes/ticket33369
new file mode 100644
index 0000000000..c55335c5b7
--- /dev/null
+++ b/changes/ticket33369
@@ -0,0 +1,4 @@
+  o Documentation (manpage):
+    - Add cross reference links and a table of contents to the HTML
+      tor manpage.  Closes ticket 33369.  Work by Swati Thacker as
+      part of Google Season of Docs.
diff --git a/changes/ticket33370 b/changes/ticket33370
new file mode 100644
index 0000000000..41e03357f0
--- /dev/null
+++ b/changes/ticket33370
@@ -0,0 +1,3 @@
+  o Minor features (client-only compilation):
+    - Disable more of our self-testing code when support for relay mode is
+      disabled. Closes ticket 33370.
diff --git a/changes/ticket33400 b/changes/ticket33400
new file mode 100644
index 0000000000..7603890765
--- /dev/null
+++ b/changes/ticket33400
@@ -0,0 +1,3 @@
+  o Minor feature (onion service v3):
+    - Log at INFO level why the service can not upload its descriptor(s). Closes
+      ticket 33400; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket33436 b/changes/ticket33436
new file mode 100644
index 0000000000..69b5545c6d
--- /dev/null
+++ b/changes/ticket33436
@@ -0,0 +1,4 @@
+  o Minor features (directory authority, shared random):
+    - Refactor more authority-only parts of the shared-random scheduling code
+      to reside in the dirauth module, and to be disabled when compiling with
+      --disable-module-dirauth.  Closes ticket 33436.
diff --git a/changes/ticket33451 b/changes/ticket33451
new file mode 100644
index 0000000000..74dd6d1ad8
--- /dev/null
+++ b/changes/ticket33451
@@ -0,0 +1,3 @@
+  o Minor features (developer tools):
+    - Add a script ("git-install-tools.sh") to install git hooks and helper
+      scripts.  Closes ticket 33451.
diff --git a/changes/ticket33458 b/changes/ticket33458
new file mode 100644
index 0000000000..885c6dc505
--- /dev/null
+++ b/changes/ticket33458
@@ -0,0 +1,4 @@
+  o Minor bugfix (onion service v3):
+    - When cleaning the client descriptor cache, an attempt at closing circuits
+      for a non decrypted descriptor (lacking client authorization) lead to an
+      assert(). Fixes bug 33458; bugfix on 0.4.2.1-alpha.
diff --git a/changes/ticket33491 b/changes/ticket33491
deleted file mode 100644
index 595ea863ea..0000000000
--- a/changes/ticket33491
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (DoS defenses, bridges, pluggable transport):
-    - DoS subsystem was not given the transport name of the client connection
-      when tor is a bridge and thus failing to find the GeoIP cache entry for
-      that client address. This resulted in failing to apply DoS defenses on
-      bridges with a pluggable transport. Fixes bug 33491; bugfix on
-      0.3.3.2-alpha.
diff --git a/changes/ticket33633 b/changes/ticket33633
new file mode 100644
index 0000000000..de030a6000
--- /dev/null
+++ b/changes/ticket33633
@@ -0,0 +1,6 @@
+  o Code simplification and refactoring:
+    - Move the circuit extend code to the relay module.
+      Split the circuit extend function into smaller functions.
+      Closes ticket 33633.
+    - Move LOG_PROTOCOL_WARN to app/config.c. Resolves a dependency inversion.
+      Closes ticket 33633.
diff --git a/changes/ticket33642 b/changes/ticket33642
new file mode 100644
index 0000000000..b81edf7613
--- /dev/null
+++ b/changes/ticket33642
@@ -0,0 +1,4 @@
+  o Minor features (developer tooling):
+    - Refrain from listing all .a files that are generated by Tor build in
+      .gitignore. Add a single wildcard *.a entry that covers all of them for
+      present and future. Closes ticket 33642.
diff --git a/changes/ticket33643 b/changes/ticket33643
deleted file mode 100644
index 7fddab74eb..0000000000
--- a/changes/ticket33643
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features (testing):
-    - The unit tests now support a "TOR_SKIP_TESTCASES" environment variable
-      to specify a list of space-separated test cases that should not be
-      executed. We will use this to disable certain tests that are failing on
-      Appveyor because of mismatched OpenSSL libraries. Part of ticket 33643.
diff --git a/changes/ticket33643_part2 b/changes/ticket33643_part2
deleted file mode 100644
index 28193d2af5..0000000000
--- a/changes/ticket33643_part2
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing (CI):
-    - On appveyor, skip the crypto/openssl_version test, which is failing
-      because of a mismatched library installation. Fix for 33643.
diff --git a/changes/ticket33679 b/changes/ticket33679
new file mode 100644
index 0000000000..d37842d065
--- /dev/null
+++ b/changes/ticket33679
@@ -0,0 +1,4 @@
+  o Minor features (IPv6 Support, address.c):
+    - Adds IPv6 support to tor_addr_is_valid(). Adds tests for the
+      above changes and tor_addr_is_null(). Closes ticket 33679.
+      Patch by MrSquanchee.
diff --git a/changes/ticket33778 b/changes/ticket33778
new file mode 100644
index 0000000000..a33c647a6e
--- /dev/null
+++ b/changes/ticket33778
@@ -0,0 +1,3 @@
+  o Documentation (manpage):
+    - Updated the options set by TestingTorNetwork in the man page.
+      Closes ticket 33778.
diff --git a/changes/ticket33779 b/changes/ticket33779
new file mode 100644
index 0000000000..d4bc769ebb
--- /dev/null
+++ b/changes/ticket33779
@@ -0,0 +1,3 @@
+  o Minor bugfixes (onion service, logging):
+    - Typo in a log info level when PublishHidServDescriptors is set to 0.
+      Fixes bug 33779; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket33788 b/changes/ticket33788
new file mode 100644
index 0000000000..236c056623
--- /dev/null
+++ b/changes/ticket33788
@@ -0,0 +1,4 @@
+  o Minor features (code safety):
+    - Check for failures of tor_inet_ntop() and tor_inet_ntoa() functions in
+      DNS and IP address processing code and adjust codepaths to make them
+      less likely to crash entire Tor instance. Resolves issue 33788.
diff --git a/changes/ticket33789 b/changes/ticket33789
new file mode 100644
index 0000000000..a7e69793e6
--- /dev/null
+++ b/changes/ticket33789
@@ -0,0 +1,4 @@
+  o Code simplification and refactoring (relay address):
+    - Move a series of functions related to address resolving into their own
+      files. Closes ticket 33789.
+
diff --git a/changes/ticket33796 b/changes/ticket33796
new file mode 100644
index 0000000000..9a98bf2d9a
--- /dev/null
+++ b/changes/ticket33796
@@ -0,0 +1,7 @@
+  o Removed features (IPv6, revert):
+    - Revert the client port prefer IPv6 feature because it breaks the
+      torsocks use case. The SOCKS resolve command is lacking a mechanism to
+      ask for a specific address family (v4 or v6) thus prioritizing IPv6 when
+      an IPv4 address is asked on the resolve SOCKS interface resulting in a
+      failure. Tor Browser explicitly set PreferIPv6 so this should not affect
+      the majority of our users. Closes ticket 33796; bugfix on 0.4.4.1-alpha.
diff --git a/changes/ticket33817 b/changes/ticket33817
new file mode 100644
index 0000000000..9c22d084eb
--- /dev/null
+++ b/changes/ticket33817
@@ -0,0 +1,12 @@
+  o Major features (IPv6, relay):
+    - Relays may extend circuits over IPv6, if the relay has an IPv6 ORPort,
+      and the client supplies the other relay's IPv6 ORPort in the EXTEND2
+      cell. IPv6 extends will be used by the relay IPv6 ORPort self-tests in
+      33222. Closes ticket 33817.
+    - Consider IPv6-only EXTEND2 cells valid on relays. Log a protocol warning
+      if the IPv4 or IPv6 address is an internal address, and internal
+      addresses are not allowed. But continue to use the other address, if it
+      is valid. Closes ticket 33817.
+    - If a relay can extend over IPv4 and IPv6, it chooses between them
+      uniformly at random. Closes ticket 33817.
+    - Re-use existing IPv6 connections for circuit extends. Closes ticket 33817.
diff --git a/changes/ticket33873 b/changes/ticket33873
new file mode 100644
index 0000000000..c45191181a
--- /dev/null
+++ b/changes/ticket33873
@@ -0,0 +1,4 @@
+  o Minor bugfix (SOCKS, onion service client):
+    - Also detect bad v3 onion service address of the wrong length when
+      returning the F6 ExtendedErrors code. Fixes bug 33873; bugfix on
+      0.4.3.1-alpha.
diff --git a/changes/ticket33901 b/changes/ticket33901
new file mode 100644
index 0000000000..b824cc5b07
--- /dev/null
+++ b/changes/ticket33901
@@ -0,0 +1,4 @@
+  o Minor features (IPv6, relay):
+    - Allow clients and relays to send dual-stack and IPv6-only EXTEND2 cells.
+      Parse dual-stack and IPv6-only EXTEND2 cells on relays.
+      Closes ticket 33901.
diff --git a/changes/ticket33956 b/changes/ticket33956
new file mode 100644
index 0000000000..7ad802797d
--- /dev/null
+++ b/changes/ticket33956
@@ -0,0 +1,5 @@
+  o Code simplification and refactoring:
+    - Define and use a new constant TOR_ADDRPORT_BUF_LEN which is like
+      TOR_ADDR_BUF_LEN but includes enough space for an IP address,
+      brackets, seperating colon, and port number. Closes ticket 33956.
+      Patch by Neel Chauhan.
diff --git a/changes/ticket34087 b/changes/ticket34087
new file mode 100644
index 0000000000..16990c305a
--- /dev/null
+++ b/changes/ticket34087
@@ -0,0 +1,3 @@
+  o Minor bugfix (onion service v3 client):
+    - Remove a BUG() warning that can occur naturally. Fixes bug 34087; bugfix
+      on 0.3.2.1-alpha.
diff --git a/changes/ticket34211 b/changes/ticket34211
new file mode 100644
index 0000000000..b454873abf
--- /dev/null
+++ b/changes/ticket34211
@@ -0,0 +1,3 @@
+  o Minor features (windows):
+    - Add support for console control signals like Ctrl+C in Windows
+      Closes ticket 34211. Patch from Damon Harris (TheDcoder).
diff --git a/changes/ticket34255_043 b/changes/ticket34255_043
new file mode 100644
index 0000000000..5cfec1d48d
--- /dev/null
+++ b/changes/ticket34255_043
@@ -0,0 +1,3 @@
+  o Documentation:
+    - Fix several doxygen warnings related to imbalanced groups.
+      Closes ticket 34255.
diff --git a/changes/ticket34382 b/changes/ticket34382
new file mode 100644
index 0000000000..0bdfe22a5e
--- /dev/null
+++ b/changes/ticket34382
@@ -0,0 +1,6 @@
+  o Minor features (Linux seccomp2 sandbox, compilation):
+    - Allow Tor to build on platforms where it doesn't know how to
+      report which syscall had caused the linux seccomp2 sandbox
+      to fail. This change should make the sandbox code more portable
+      to less common Linux architectures.
+      Closes ticket 34382.
diff --git a/changes/ticket40005 b/changes/ticket40005
new file mode 100644
index 0000000000..12727e0a06
--- /dev/null
+++ b/changes/ticket40005
@@ -0,0 +1,3 @@
+  o Minor bugfix (control port, onion service):
+    - Consistently use 'address' in "Invalid v3 address" response to
+      ONION_CLIENT_AUTH commands. Fixes bug 40005; bugfix on 0.4.3.1-alpha.
diff --git a/changes/ticket40030 b/changes/ticket40030
new file mode 100644
index 0000000000..c5f3ca4ff9
--- /dev/null
+++ b/changes/ticket40030
@@ -0,0 +1,7 @@
+  o Removed features:
+    - Our "check-local" test target no longer tries to use the Coccinelle
+      semantic patching tool parse all the C files.  While it is a good idea
+      to try to make sure Coccinelle works on our C before we run a
+      Coccinelle patch, doing so on every test run has proven to be disruptive.
+      You can still run this tool manually with "make check-cocci". Closes
+      ticket 40030.
diff --git a/changes/ticket40089 b/changes/ticket40089
new file mode 100644
index 0000000000..121e8e9820
--- /dev/null
+++ b/changes/ticket40089
@@ -0,0 +1,4 @@
+  o Minor features (control port):
+    - If a ClientName was specified in ONION_CLIENT_AUTH_ADD for an
+      onion service, display it when we use ONION_CLIENT_AUTH_VIEW
+      on it. Closes ticket 40089. Patch by Neel Chauhan.
diff --git a/changes/ticket40109 b/changes/ticket40109
new file mode 100644
index 0000000000..d99db65aa4
--- /dev/null
+++ b/changes/ticket40109
@@ -0,0 +1,6 @@
+  o Major bugfixes (onion services, DoS):
+    - The consensus parameters for the onion service DoS defenses was
+      overwriting the circuit parameters that could have been set by the service
+      operator using HiddenServiceEnableIntroDoSDefense. Fixes bug 40109; bugfix
+      on 0.4.2.1-alpha.
+
diff --git a/changes/ticket40162 b/changes/ticket40162
new file mode 100644
index 0000000000..093042f9af
--- /dev/null
+++ b/changes/ticket40162
@@ -0,0 +1,7 @@
+  o Minor features (authorities):
+    - Authorities now list a different set of protocols as required and
+      recommended.  These lists are chosen so that only truly recommended
+      and/or required protocols are included, and so that clients using 0.2.9
+      or later will continue to work (even though they are not supported),
+      whereas only relays running 0.3.5 or later will meet the requirements.
+      Closes ticket 40162.
diff --git a/changes/ticket40164 b/changes/ticket40164
new file mode 100644
index 0000000000..c96118d0a4
--- /dev/null
+++ b/changes/ticket40164
@@ -0,0 +1,6 @@
+  o Minor features (directory authority):
+    - Make it possible to specify multiple ConsensusParams torrc lines.
+      Now directory authority operators can for example put the main
+      ConsensusParams config in one torrc file and then add to it from
+      a different torrc file. Closes ticket 40164.
+
diff --git a/changes/ticket40345 b/changes/ticket40345
new file mode 100644
index 0000000000..246e4a86af
--- /dev/null
+++ b/changes/ticket40345
@@ -0,0 +1,5 @@
+  o Minor bugfixes (channel, DoS):
+    - Fix a possible non fatal assertion BUG() due to a too early free of a
+      string when noting down the client connection for the DoS defenses
+      subsystem. Fixes bug 40345; bugfix on 0.4.3.4-rc
+
diff --git a/changes/ticket6198 b/changes/ticket6198
new file mode 100644
index 0000000000..7f3fdf2fa7
--- /dev/null
+++ b/changes/ticket6198
@@ -0,0 +1,3 @@
+  o Minor features (defense in depth):
+    - Wipe more data from connection address fields before returning them to
+      the memory heap. Closes ticket 6198.
diff --git a/changes/trove_2020_003 b/changes/trove_2020_003
deleted file mode 100644
index aa1a8f1c78..0000000000
--- a/changes/trove_2020_003
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (onion services v3):
-  - Fix assertion failure that could result from a corrupted ADD_ONION control
-    port command. Found by Saibato. Fixes bug 33137; bugfix on
-    0.3.3.1-alpha. This issue is also being tracked as TROVE-2020-003.
diff --git a/configure.ac b/configure.ac
index 89ccbc5807..1527d580b0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4,10 +4,21 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_PREREQ([2.63])
-AC_INIT([tor],[0.3.5.15-dev])
+AC_INIT([tor],[0.4.4.9])
 AC_CONFIG_SRCDIR([src/app/main/tor_main.c])
 AC_CONFIG_MACRO_DIR([m4])
 
+configure_flags="$*"
+
+# DO NOT EDIT THIS DEFINITION BY HAND UNLESS YOU KNOW WHAT YOU'RE DOING.
+#
+# The update_versions.py script updates this definition when the
+# version number changes.  Tor uses it to make sure that it
+# only shuts down for missing "required protocols" when those protocols
+# are listed as required by a consensus after this date.
+AC_DEFINE(APPROX_RELEASE_DATE, ["2021-06-07"], # for 0.4.4.9
+          [Approximate date when this software was released. (Updated when the version changes.)])
+
 # "foreign" means we don't follow GNU package layout standards
 # "1.11" means we require automake version 1.11 or newer
 # "subdir-objects" means put .o files in the same directory as the .c files
@@ -20,6 +31,18 @@ AC_USE_SYSTEM_EXTENSIONS
 AC_CANONICAL_HOST
 
 PKG_PROG_PKG_CONFIG
+if test "x$PKG_CONFIG" = "x" ; then
+    pkg_config_user_action="install pkg-config, and check the PKG_CONFIG_PATH environment variable"
+    AC_MSG_NOTICE([Some libraries need pkg-config, including systemd, nss, lzma, zstd, and custom mallocs.])
+    AC_MSG_NOTICE([To use those libraries, $pkg_config_user_action.])
+else
+    pkg_config_user_action="check the PKG_CONFIG_PATH environment variable"
+fi
+
+if test "x$PKG_CONFIG_PATH" = "x" && test "x$prefix" != "xNONE" && test "$host" != "$build"; then
+   export PKG_CONFIG_PATH=$prefix/lib/pkgconfig
+   AC_MSG_NOTICE([set PKG_CONFIG_PATH=$PKG_CONFIG_PATH to support cross-compiling])
+fi
 
 AC_ARG_ENABLE(openbsd-malloc,
    AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD.  Linux only. Deprecated: see --with-malloc]))
@@ -58,6 +81,24 @@ AC_ARG_ENABLE(nss,
 AC_ARG_ENABLE(pic,
    AS_HELP_STRING(--enable-pic, [Build Tor's binaries as position-independent code, suitable to link as a library.]))
 
+AC_ARG_ENABLE(missing-doc-warnings,
+   AS_HELP_STRING(--enable-missing-doc-warnings, [Tell doxygen to warn about missing documentation. Makes doxygen warnings nonfatal.]))
+
+if test "$enable_missing_doc_warnings" = "yes"; then
+   DOXYGEN_FATAL_WARNINGS=NO
+   DOXYGEN_WARN_ON_MISSING=YES
+elif test "$enable_fatal_warnings" = "yes"; then
+   # Fatal warnings from doxygen are nice, but not if we're warning about
+   # missing documentation.
+   DOXYGEN_FATAL_WARNINGS=YES
+   DOXYGEN_WARN_ON_MISSING=NO
+else
+   DOXYGEN_FATAL_WARNINGS=NO
+   DOXYGEN_WARN_ON_MISSING=NO
+fi
+AC_SUBST(DOXYGEN_FATAL_WARNINGS)
+AC_SUBST(DOXYGEN_WARN_ON_MISSING)
+
 if test "x$enable_coverage" != "xyes" -a "x$enable_asserts_in_tests" = "xno" ; then
     AC_MSG_ERROR([Can't disable assertions outside of coverage build])
 fi
@@ -96,6 +137,12 @@ if test "$enable_memory_sentinels" = "no"; then
            [Defined if we're turning off memory safety code to look for bugs])
 fi
 
+AC_ARG_ENABLE(manpage,
+              AS_HELP_STRING(--disable-manpage, [Disable manpage generation.]))
+
+AC_ARG_ENABLE(html-manual,
+              AS_HELP_STRING(--disable-html-manual, [Disable HTML documentation.]))
+
 AC_ARG_ENABLE(asciidoc,
      AS_HELP_STRING(--disable-asciidoc, [don't use asciidoc (disables building of manpages)]),
      [case "${enableval}" in
@@ -145,13 +192,13 @@ if test "x$have_systemd" = "xyes"; then
     TOR_SYSTEMD_CFLAGS="${SYSTEMD_CFLAGS}"
     TOR_SYSTEMD_LIBS="${SYSTEMD_LIBS}"
     PKG_CHECK_MODULES(LIBSYSTEMD209, [libsystemd >= 209],
-         [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or more])], [])
+         [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or greater])], [])
 fi
 AC_SUBST(TOR_SYSTEMD_CFLAGS)
 AC_SUBST(TOR_SYSTEMD_LIBS)
 
 if test "x$enable_systemd" = "xyes" -a "x$have_systemd" != "xyes" ; then
-    AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found])
+    AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found, $pkg_config_user_action, or set SYSTEMD_CFLAGS and SYSTEMD_LIBS.])
 fi
 
 case "$host" in
@@ -181,6 +228,13 @@ if test "x$enable_expensive_hardening" = "xyes" || test "x$enable_fragile_harden
   AC_DEFINE(DEBUG_SMARTLIST, 1, [Enable smartlist debugging])
 fi
 
+AC_ARG_ENABLE(all-bugs-are-fatal,
+   AS_HELP_STRING(--enable-all-bugs-are-fatal, [force all soft asserts in Tor codebase (tor_assert_nonfatal(), BUG(), etc.) to act as hard asserts (tor_assert() and equivalents); makes Tor fragile; only recommended for dev builds]))
+
+if test "x$enable_all_bugs_are_fatal" = "xyes"; then
+  AC_DEFINE(ALL_BUGS_ARE_FATAL, 1, [All assert failures are fatal])
+fi
+
 dnl Linker hardening options
 dnl Currently these options are ELF specific - you can't use this with MacOSX
 AC_ARG_ENABLE(linker-hardening,
@@ -231,16 +285,33 @@ dnl Tor modules options. These options are namespaced with --disable-module-XXX
 dnl ---
 
 dnl All our modules.
-m4_define(MODULES, dirauth)
+m4_define(MODULES, relay dirauth dircache)
+
+dnl Relay module.
+AC_ARG_ENABLE([module-relay],
+              AS_HELP_STRING([--disable-module-relay],
+                             [Build tor without the Relay modules: tor can not run as a relay, bridge, or authority. Implies --disable-module-dirauth]))
+AM_CONDITIONAL(BUILD_MODULE_RELAY, [test "x$enable_module_relay" != "xno"])
+AM_COND_IF(BUILD_MODULE_RELAY,
+           AC_DEFINE([HAVE_MODULE_RELAY], [1],
+                     [Compile with Relay feature support]))
+
+dnl Dircache module.  (This cannot be enabled or disabled independently of
+dnl the relay module.  It is not listed by --list-modules for this reason.)
+AM_CONDITIONAL(BUILD_MODULE_DIRCACHE,
+               [test "x$enable_module_relay" != "xno"])
+AM_COND_IF(BUILD_MODULE_DIRCACHE,
+           AC_DEFINE([HAVE_MODULE_DIRCACHE], [1],
+                     [Compile with directory cache support]))
 
 dnl Directory Authority module.
 AC_ARG_ENABLE([module-dirauth],
               AS_HELP_STRING([--disable-module-dirauth],
-                             [Do not build tor with the dirauth module]),
-              [], dnl Action if-given
-              AC_DEFINE([HAVE_MODULE_DIRAUTH], [1],
-                        [Compile with Directory Authority feature support]))
-AM_CONDITIONAL(BUILD_MODULE_DIRAUTH, [test "x$enable_module_dirauth" != "xno"])
+                             [Build tor without the Directory Authority module: tor can not run as a directory authority or bridge authority]))
+AM_CONDITIONAL(BUILD_MODULE_DIRAUTH,[test "x$enable_module_dirauth" != "xno" && test "x$enable_module_relay" != "xno"])
+AM_COND_IF(BUILD_MODULE_DIRAUTH,
+           AC_DEFINE([HAVE_MODULE_DIRAUTH], [1],
+                     [Compile with Directory Authority feature support]))
 
 dnl Helper variables.
 TOR_MODULES_ALL_ENABLED=
@@ -290,19 +361,26 @@ AC_PATH_PROG([ASCIIDOC], [asciidoc], none)
 AC_PATH_PROGS([A2X], [a2x a2x.py], none)
 
 AM_CONDITIONAL(USE_ASCIIDOC, test "x$asciidoc" = "xtrue")
+AM_CONDITIONAL(BUILD_MANPAGE, [test "x$enable_manpage" != "xno"])
+AM_CONDITIONAL(BUILD_HTML_DOCS, [test "x$enable_html_manual" != "xno"])
 
 AM_PROG_CC_C_O
 AC_PROG_CC_C99
 
-AC_ARG_VAR([PYTHON], [path to Python binary])
-AC_CHECK_PROGS(PYTHON, [ \
+AC_CACHE_CHECK([for Python 3], [tor_cv_PYTHON],
+   [AC_PATH_PROGS_FEATURE_CHECK([PYTHON], [ \
 	python3 \
 	python3.8 python3.7 python3.6 python3.5 python3.4 \
-	python \
-	python2 python2.7])
+	python ],
+   [["$ac_path_PYTHON" -c 'import sys; sys.exit(sys.version_info[0]<3)' && tor_cv_PYTHON="$ac_path_PYTHON" ac_path_PYTHON_found=:]] )])
+AC_SUBST([PYTHON], [$tor_cv_PYTHON])
+
+PYTHON="$tor_cv_PYTHON"
+
 if test "x$PYTHON" = "x"; then
-  AC_MSG_WARN([Python unavailable; some tests will not be run.])
+  AC_MSG_WARN([Python 3 unavailable; some tests will not be run.])
 fi
+
 AM_CONDITIONAL(USEPYTHON, [test "x$PYTHON" != "x"])
 
 dnl List all external rust crates we depend on here. Include the version
@@ -519,15 +597,6 @@ if test "x$enable_rust" = "xyes"; then
     fi
   fi
 
-  dnl This is a workaround for #46797
-  dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ).  Once the
-  dnl upstream bug is fixed, we can remove this workaround.
-  case "$host_os" in
-      darwin*)
-        TOR_RUST_EXTRA_LIBS="-lresolv"
-	;;
-  esac
-
   dnl For now both MSVC and MinGW rust libraries will output static libs with
   dnl the MSVC naming convention.
   if test "$bwin32" = "true"; then
@@ -566,14 +635,12 @@ if test "x$enable_rust" = "xyes"; then
   if test "x$RUSTC_VERSION_MAJOR" = "x" -o "x$RUSTC_VERSION_MINOR" = "x"; then
     AC_MSG_ERROR([rustc version couldn't be identified])
   fi
-  if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 14; then
-    AC_MSG_ERROR([rustc must be at least version 1.14])
+  if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 31; then
+    AC_MSG_ERROR([rustc must be at least version 1.31.0])
   fi
   AC_MSG_RESULT([$RUSTC_VERSION])
 fi
 
-AC_SUBST(TOR_RUST_EXTRA_LIBS)
-
 AC_SEARCH_LIBS(socket, [socket network])
 AC_SEARCH_LIBS(gethostbyname, [nsl])
 AC_SEARCH_LIBS(dlopen, [dl])
@@ -605,6 +672,7 @@ AC_CHECK_FUNCS(
 	explicit_bzero \
 	timingsafe_memcmp \
 	flock \
+	fsync \
 	ftime \
 	get_current_dir_name \
 	getaddrinfo \
@@ -623,8 +691,10 @@ AC_CHECK_FUNCS(
 	llround \
 	localtime_r \
 	lround \
+	madvise \
 	memmem \
 	memset_s \
+	minherit \
 	mmap \
 	pipe \
 	pipe2 \
@@ -828,6 +898,8 @@ fi
 dnl Now check for particular libevent functions.
 AC_CHECK_FUNCS([evutil_secure_rng_set_urandom_device_file \
                 evutil_secure_rng_add_bytes \
+                evdns_base_get_nameserver_addr \
+
 ])
 
 LIBS="$save_LIBS"
@@ -876,7 +948,7 @@ if test "x$enable_nss" = "xyes"; then
   PKG_CHECK_MODULES(NSS,
      [nss],
      [have_nss=yes],
-     [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it.])])
+     [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it, $pkg_config_user_action, or set NSS_CFLAGS and NSS_LIBS.])])
   AC_SUBST(NSS_CFLAGS)
   AC_SUBST(NSS_LIBS)
 fi
@@ -980,21 +1052,24 @@ AC_CHECK_MEMBERS([struct ssl_method_st.get_cipher_by_char], , ,
 [#include <openssl/ssl.h>
 ])
 
+dnl OpenSSL functions which we might not have.  In theory, we could just
+dnl check the openssl version number, but in practice that gets pretty
+dnl confusing with LibreSSL, OpenSSL, and various distributions' patches
+dnl to them.
 AC_CHECK_FUNCS([ \
 		ERR_load_KDF_strings \
+		EVP_PBE_scrypt \
+		EVP_sha3_256 \
+		SSL_CIPHER_find \
+		SSL_CTX_set1_groups_list \
+		SSL_CTX_set_security_level \
 		SSL_SESSION_get_master_key \
+		SSL_get_client_ciphers \
+		SSL_get_client_random \
 		SSL_get_server_random \
-                SSL_get_client_ciphers \
-                SSL_get_client_random \
-                SSL_CTX_set1_groups_list \
-		SSL_CIPHER_find \
-                SSL_CTX_set_security_level \
-		TLS_method
+		TLS_method \
 	       ])
 
-dnl Check if OpenSSL has scrypt implementation.
-AC_CHECK_FUNCS([ EVP_PBE_scrypt ])
-
 dnl Check if OpenSSL structures are opaque
 AC_CHECK_MEMBERS([SSL.state], , ,
 [#include <openssl/ssl.h>
@@ -1006,6 +1081,15 @@ AC_CHECK_SIZEOF(SHA_CTX, , [AC_INCLUDES_DEFAULT()
 
 fi # enable_nss
 
+dnl We will someday make KECCAK_TINY optional, but for now we still need
+dnl it for SHAKE, since OpenSSL's SHAKE can't be squeezed more than
+dnl once.  See comment in the definition of crypto_xof_t.
+
+dnl AM_CONDITIONAL(BUILD_KECCAK_TINY,
+dnl   test "x$ac_cv_func_EVP_sha3_256" != "xyes")
+
+AM_CONDITIONAL(BUILD_KECCAK_TINY, true)
+
 dnl ======================================================================
 dnl Can we use KIST?
 
@@ -1073,10 +1157,10 @@ dnl Where we do we find lzma?
 AC_ARG_ENABLE(lzma,
       AS_HELP_STRING(--enable-lzma, [enable support for the LZMA compression scheme.]),
       [case "${enableval}" in
-        "yes") lzma=true ;;
-        "no")  lzma=false ;;
+        "yes") ;;
+        "no")  ;;
         * ) AC_MSG_ERROR(bad value for --enable-lzma) ;;
-      esac], [lzma=auto])
+      esac], [enable_lzma=auto])
 
 if test "x$enable_lzma" = "xno"; then
     have_lzma=no;
@@ -1087,7 +1171,7 @@ else
                       have_lzma=no)
 
     if test "x$have_lzma" = "xno" ; then
-        AC_MSG_WARN([Unable to find liblzma.])
+        AC_MSG_WARN([Unable to find liblzma, $pkg_config_user_action, or set LZMA_CFLAGS and LZMA_LIBS.])
     fi
 fi
 
@@ -1105,10 +1189,10 @@ dnl Where we do we find zstd?
 AC_ARG_ENABLE(zstd,
       AS_HELP_STRING(--enable-zstd, [enable support for the Zstandard compression scheme.]),
       [case "${enableval}" in
-        "yes") zstd=true ;;
-        "no")  zstd=false ;;
+        "yes") ;;
+        "no")  ;;
         * ) AC_MSG_ERROR(bad value for --enable-zstd) ;;
-      esac], [zstd=auto])
+      esac], [enable_zstd=auto])
 
 if test "x$enable_zstd" = "xno"; then
     have_zstd=no;
@@ -1119,7 +1203,7 @@ else
                       have_zstd=no)
 
     if test "x$have_zstd" = "xno" ; then
-        AC_MSG_WARN([Unable to find libzstd.])
+        AC_MSG_WARN([Unable to find libzstd, $pkg_config_user_action, or set ZSTD_CFLAGS and ZSTD_LIBS.])
     fi
 fi
 
@@ -1207,6 +1291,17 @@ m4_ifdef([AS_VAR_IF],[
        TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check")
     fi
     TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true)
+
+   AC_MSG_CHECKING([whether we can run hardened binaries])
+   AC_RUN_IFELSE([AC_LANG_PROGRAM([], [return 0;])],
+        [AC_MSG_RESULT([yes])],
+        [AC_MSG_RESULT([no])
+         AC_MSG_ERROR([dnl
+ We can link with compiler hardening options, but we can't run with them.
+ That's a bad sign! If you must, you can pass --disable-gcc-hardening to
+ configure, but it would be better to figure out what the underlying problem
+ is.])],
+        [AC_MSG_RESULT([cross])])
 fi
 
 if test "$fragile_hardening" = "yes"; then
@@ -1229,6 +1324,11 @@ if test "$fragile_hardening" = "yes"; then
       AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*])
     fi
 
+   TOR_TRY_COMPILE_WITH_CFLAGS([-fno-sanitize=float-divide-by-zero], also_link, CFLAGS_UBSAN="-fno-sanitize=float-divide-by-zero", true)
+    if test "$tor_cv_cflags__fno_sanitize_float_divide_by_zero" = "yes" && test "$tor_can_link__fno_sanitize_float_divide_by_zero" != "yes"; then
+      AC_MSG_ERROR([The compiler supports -fno-sanitize=float-divide-by-zero, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*])
+    fi
+
 TOR_CHECK_CFLAGS([-fno-omit-frame-pointer])
 fi
 
@@ -1483,6 +1583,7 @@ AC_CHECK_HEADERS([errno.h \
 		  inttypes.h \
 		  limits.h \
 		  linux/types.h \
+		  mach/vm_inherit.h \
 		  machine/limits.h \
 		  malloc.h \
 		  malloc/malloc.h \
@@ -1618,6 +1719,7 @@ AC_CHECK_MEMBERS([struct timeval.tv_sec], , ,
 AC_CHECK_SIZEOF(char)
 AC_CHECK_SIZEOF(short)
 AC_CHECK_SIZEOF(int)
+AC_CHECK_SIZEOF(unsigned int)
 AC_CHECK_SIZEOF(long)
 AC_CHECK_SIZEOF(long long)
 AC_CHECK_SIZEOF(__int64)
@@ -1880,8 +1982,7 @@ if test "$tor_cv_uint8_uchar" = "no"; then
 fi
 
 AC_ARG_WITH(tcmalloc,
-AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library. Deprecated; see --with-malloc]),
-[ tcmalloc=yes ], [ tcmalloc=no ])
+AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library. Deprecated; see --with-malloc]))
 
 default_malloc=system
 
@@ -1890,7 +1991,7 @@ if test "x$enable_openbsd_malloc" = "xyes" ; then
   default_malloc=openbsd
 fi
 
-if test "x$tcmalloc" = "xyes"; then
+if test "x$with_tcmalloc" = "xyes"; then
   AC_MSG_NOTICE([The --with-tcmalloc argument is deprecated; use --with-malloc=tcmalloc instead.])
   default_malloc=tcmalloc
 fi
@@ -1908,7 +2009,7 @@ AS_CASE([$malloc],
 			have_tcmalloc=no)
 
       if test "x$have_tcmalloc" = "xno" ; then
-          AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc.])
+          AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc, $pkg_config_user_action, or set TCMALLOC_CFLAGS and TCMALLOC_LIBS.])
       fi
 
       CFLAGS="$CFLAGS $TCMALLOC_CFLAGS"
@@ -1921,8 +2022,8 @@ AS_CASE([$malloc],
 			have_jemalloc=yes,
 			have_jemalloc=no)
 
-      if test "x$have_tcmalloc" = "xno" ; then
-          AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc.])
+      if test "x$have_jemalloc" = "xno" ; then
+          AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc, $pkg_config_user_action, or set JEMALLOC_CFLAGS and JEMALLOC_LIBS.])
       fi
 
       CFLAGS="$CFLAGS $JEMALLOC_CFLAGS"
@@ -2468,16 +2569,12 @@ AC_CONFIG_FILES([
 	Doxyfile
 	Makefile
 	config.rust
-	contrib/dist/suse/tor.sh
 	contrib/operator-tools/tor.logrotate
-	contrib/dist/tor.sh
-	contrib/dist/torctl
 	contrib/dist/tor.service
 	src/config/torrc.sample
 	src/config/torrc.minimal
 	src/rust/.cargo/config
 	scripts/maint/checkOptionDocs.pl
-	scripts/maint/updateVersions.pl
         warning_flags
 ])
 
@@ -2515,3 +2612,141 @@ https://trac.torproject.org/projects/tor/wiki/doc/TorFragileHardening
 fi
 
 AC_OUTPUT
+
+#
+# Mini-report on what will be built.
+#
+
+PPRINT_INIT
+PPRINT_SET_INDENT(1)
+PPRINT_SET_TS(65)
+
+AS_ECHO
+AS_ECHO("Tor Version: ${PPRINT_COLOR_BLDBLU}Tor $PACKAGE_VERSION $PPRINT_COLOR_RST")
+AS_ECHO
+
+PPRINT_SUBTITLE([Build Features])
+
+PPRINT_PROP_STRING([Compiler], [$CC])
+PPRINT_PROP_STRING([Host OS], [$host_os])
+AS_ECHO
+
+test "x$enable_fatal_warnings" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([Warnings are fatal (--enable-fatal-warnings)], $value)
+
+test "x$enable_rust" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([Rust support (--enable-rust)], $value)
+
+test "x$enable_cargo_online_mode" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([Cargo Online Fetch (--enable-cargo-online-mode)], $value)
+
+test "x$enable_android" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([Android support (--enable-android)], $value)
+
+AS_ECHO
+PPRINT_SUBTITLE([Static Build])
+
+test "x$enable_static_tor" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([tor (--enable-static-tor)], $value)
+
+if test "x$enable_static_libevent" = "xyes"; then
+  PPRINT_PROP_STRING([libevent], [$TOR_LIBDIR_libevent])
+else
+  PPRINT_PROP_BOOL([libevent (--enable-static-libevent)], $value)
+fi
+
+if test "x$enable_static_openssl" = "xyes"; then
+  PPRINT_PROP_STRING([libssl], [$TOR_LIBDIR_openssl])
+else
+  PPRINT_PROP_BOOL([libssl (--enable-static-openssl)], $value)
+fi
+
+if test "x$enable_static_zlib" = "xyes"; then
+  PPRINT_PROP_STRING([zlib1g], [$TOR_LIBDIR_zlib])
+else
+  PPRINT_PROP_BOOL([zlib1g (--enable-static-zlib)], $value)
+fi
+
+AS_ECHO
+PPRINT_SUBTITLE([Optional Libraries])
+
+test "x$enable_nss" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([libnss (--enable-nss)], $value)
+
+test "x$enable_seccomp" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([libseccomp (--disable-seccomp)], $value)
+
+test "x$enable_libscrypt" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([libscrypt (--disable-libscrypt)], $value)
+
+test "x$enable_systemd" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([Systemd support (--enable-systemd)], $value)
+
+test "x$enable_lzma" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([liblzma (--enable-lzma)], $value)
+
+test "x$enable_zstd" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([libzstd (--enable-zstd)], $value)
+
+AS_ECHO
+PPRINT_SUBTITLE([Hardening])
+
+test "x$enable_gcc_hardening" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([Compiler Hardening (--disable-gcc-hardening)], $value)
+
+test "x$enable_linker_hardening" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([Linker Hardening (--disable-linker-hardening)], $value)
+
+test "x$fragile_hardening" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([Fragile Hardening (--enable-fragile-hardening, dev only)], $value)
+
+AS_ECHO
+PPRINT_SUBTITLE([Modules])
+
+m4_foreach_w([mname], MODULES,
+  [
+    AM_COND_IF(m4_join([], [BUILD_MODULE_], m4_toupper([]mname[])), value=1, value=0)
+    PPRINT_PROP_BOOL([mname (--disable-module-mname)], $value)
+  ]
+)
+
+AS_ECHO
+PPRINT_SUBTITLE([Documentation])
+
+test "x$enable_asciidoc" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([AsciiDoc (--disable-asciidoc)], $value)
+
+test "x$enable_manpage" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([Man Pages (--disable-manpage)], $value)
+
+test "x$enable_html_manual" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([HTML Manual (--disable-html-manual)], $value)
+
+AS_ECHO
+PPRINT_SUBTITLE([Tests])
+
+test "x$enable_unittests" != "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([Unit tests (--disable-unittests)], $value)
+
+test "x$enable_asserts_in_tests" = "xno" && value=1 || value=0
+PPRINT_PROP_BOOL([assert()s enabled (--enable-asserts-in-tests, dev only)], $value)
+
+test "x$enable_coverage" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([Code Coverage (--enable-coverage)], $value)
+
+test "x$enable_libfuzzer" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([libFuzzer support (--enable-libfuzzer)], $value)
+
+test "x$enable_oss_fuzz" = "xyes" && value=1 || value=0
+PPRINT_PROP_BOOL([OSS-Fuzz support (--enable-oss-fuzz)], $value)
+
+AS_ECHO
+PPRINT_SUBTITLE([Install Directories])
+
+report_mandir="`eval eval echo $mandir`"
+PPRINT_PROP_STRING([Binaries], [$BINDIR])
+PPRINT_PROP_STRING([Configuration], [$CONFDIR])
+PPRINT_PROP_STRING([Man Pages], [$report_mandir])
+
+AS_ECHO
+AS_ECHO(["Configure Line: ./configure $configure_flags"])
diff --git a/contrib/README b/contrib/README
index 3a94bb5016..a56065f97d 100644
--- a/contrib/README
+++ b/contrib/README
@@ -29,13 +29,11 @@ should probably use src/tools/tor-resolve instead.
 dist/ -- Scripts and files for use when packaging Tor
 -----------------------------------------------------
 
-torctl, rc.subr, and tor.sh are init scripts for use with SysV-style init
+torctl and tor.sh are init scripts for use with SysV-style init
 tools.  Everybody likes to write init scripts differently, it seems.
 
 tor.service is a sample service file for use with systemd.
 
-The suse/ subdirectory contains files used by the suse distribution.
-
 operator-tools/ -- Tools for Tor relay operators
 ------------------------------------------------
 
diff --git a/contrib/clang/sanitize_blacklist.txt b/contrib/clang/sanitize_blacklist.txt
index c71cc4d878..2ce787975b 100644
--- a/contrib/clang/sanitize_blacklist.txt
+++ b/contrib/clang/sanitize_blacklist.txt
@@ -1,6 +1,6 @@
 # clang sanitizer special case list
-# syntax specified in http://clang.llvm.org/docs/SanitizerSpecialCaseList.html
-# for more info see http://clang.llvm.org/docs/AddressSanitizer.html
+# syntax specified in https://clang.llvm.org/docs/SanitizerSpecialCaseList.html
+# for more info see https://clang.llvm.org/docs/AddressSanitizer.html
 
 #
 # Tor notes: This file is obsolete!
diff --git a/contrib/client-tools/tor-resolve.py b/contrib/client-tools/tor-resolve.py
index 47ae1a0c38..85e7d2d8b4 100755
--- a/contrib/client-tools/tor-resolve.py
+++ b/contrib/client-tools/tor-resolve.py
@@ -1,4 +1,9 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 import socket
 import struct
@@ -82,7 +87,7 @@ def parseHostAndPort(h):
         try:
             port = int(h[i+1:])
         except ValueError:
-            print "Bad hostname %r"%h
+            print("Bad hostname %r"%h)
             sys.exit(1)
     elif h:
         try:
@@ -118,15 +123,15 @@ def resolve(hostname, sockshost, socksport, socksver=4, reverse=0):
             return None
         answer += more
         result = parse(answer)
-    print "Got answer",result
+    print("Got answer",result)
     m = s.recv(1)
     if m:
-        print "Got extra data too: %r"%m
+        print("Got extra data too: %r"%m)
     return result
 
 if __name__ == '__main__':
     if len(sys.argv) not in (2,3,4):
-        print "Syntax: resolve.py [-4|-5] hostname [sockshost:socksport]"
+        print("Syntax: resolve.py [-4|-5] hostname [sockshost:socksport]")
         sys.exit(0)
     socksver = 4
     reverse = 0
@@ -141,7 +146,7 @@ if __name__ == '__main__':
             break
 
     if len(sys.argv) >= 4:
-        print "Syntax: resolve.py [-x] [-4|-5] hostname [sockshost:socksport]"
+        print("Syntax: resolve.py [-x] [-4|-5] hostname [sockshost:socksport]")
         sys.exit(0)
     if len(sys.argv) == 3:
         sh,sp = parseHostAndPort(sys.argv[2])
diff --git a/contrib/client-tools/torify b/contrib/client-tools/torify
index 54acfed654..ac4c9b5c7f 100755
--- a/contrib/client-tools/torify
+++ b/contrib/client-tools/torify
@@ -53,7 +53,7 @@ pathfind() {
 
 if pathfind torsocks; then
     exec torsocks "$@"
-    echo "$0: Failed to exec torsocks $@" >&2
+    echo "$0: Failed to exec torsocks $*" >&2
     exit 1
 else
     echo "$0: torsocks not found in your PATH.  Perhaps it isn't installed?  (tsocks is no longer supported, for security reasons.)" >&2
diff --git a/contrib/dirauth-tools/nagios-check-tor-authority-cert b/contrib/dirauth-tools/nagios-check-tor-authority-cert
index 46dc7284b7..75ff479a53 100755
--- a/contrib/dirauth-tools/nagios-check-tor-authority-cert
+++ b/contrib/dirauth-tools/nagios-check-tor-authority-cert
@@ -49,12 +49,12 @@ DIRSERVERS="$DIRSERVERS 80.190.246.100:80"      # gabelmoo
 DIRSERVERS="$DIRSERVERS 194.109.206.212:80"     # dizum
 DIRSERVERS="$DIRSERVERS 213.73.91.31:80"        # dannenberg
 
-TMPFILE="`tempfile`"
+TMPFILE=$(mktemp)
 trap 'rm -f "$TMPFILE"' 0
 
 for dirserver in $DIRSERVERS; do
-	wget -q -O "$TMPFILE" "http://$dirserver/tor/keys/fp/$identity"
-	if [ "$?" = 0 ]; then
+	if wget -q -O "$TMPFILE" "http://$dirserver/tor/keys/fp/$identity"
+        then
 		break
 	else
 		cat /dev/null > "$TMPFILE"
@@ -74,10 +74,10 @@ now=$(date +%s)
 if [ "$now" -ge "$expiryunix" ]; then
 	echo "CRITICAL: Certificate expired $expirydate (authority $identity)."
 	exit 2
-elif [ "$(( $now + 7*24*60*60 ))" -ge "$expiryunix" ]; then
+elif [ "$(( now + 7*24*60*60 ))" -ge "$expiryunix" ]; then
 	echo "CRITICAL: Certificate expires $expirydate (authority $identity)."
 	exit 2
-elif [ "$(( $now + 30*24*60*60 ))" -ge "$expiryunix" ]; then
+elif [ "$(( now + 30*24*60*60 ))" -ge "$expiryunix" ]; then
 	echo "WARNING: Certificate expires $expirydate (authority $identity)."
 	exit 1
 else
diff --git a/contrib/dist/rc.subr b/contrib/dist/rc.subr
deleted file mode 100644
index d757e89528..0000000000
--- a/contrib/dist/rc.subr
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh
-# $FreeBSD: ports/security/tor-devel/files/tor.in,v 1.1 2006/02/17 22:21:25 mnag Exp $
-#
-# (rc.subr written by Peter Thoenen for Net/FreeBSD)
-#
-# REQUIRE: NETWORKING SERVERS USR
-# BEFORE: LOGIN
-#
-# Add the following lines to /etc/rc.conf to enable tor
-#
-# tor_enable (bool):    Set to "NO" by default
-#                       Set it to "YES" to enable tor
-# tor_conf (str):       Points to your tor conf file
-#                       Default: /usr/local/etc/tor/torrc
-# tor_user (str):       Tor Daemon user. Default _tor
-#
-
-. /etc/rc.subr
-
-name="tor"
-rcvar=${name}_enable
-
-load_rc_config ${name}
-
-: ${tor_enable="NO"}
-: ${tor_conf="/usr/local/etc/tor/torrc"}
-: ${tor_user="_tor"}
-: ${tor_pidfile="/var/run/tor/tor.pid"}
-: ${tor_logfile="/var/log/tor"}
-: ${tor_datadir="/var/run/tor"}
-
-required_files=${tor_conf}
-required_dirs=${tor_datadir}
-command="/usr/local/bin/${name}"
-command_args="-f ${tor_conf} --pidfile ${tor_pidfile} --runasdaemon 1 --datadirectory ${tor_datadir} --user ${tor_user}"
-extra_commands="log"
-log_cmd="${name}_log"
-
-tor_log() {
-        cat ${tor_logfile}
-}
-
-run_rc_command "$1"
diff --git a/contrib/dist/suse/tor.sh.in b/contrib/dist/suse/tor.sh.in
deleted file mode 100644
index b7e9005eb5..0000000000
--- a/contrib/dist/suse/tor.sh.in
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2006-2007 Andrew Lewman
-#
-# tor    The Onion Router
-#
-# Startup/shutdown script for tor. This is a wrapper around torctl;
-# torctl does the actual work in a relatively system-independent, or at least
-# distribution-independent, way, and this script deals with fitting the
-# whole thing into the conventions of the particular system at hand.
-#
-# These next couple of lines "declare" tor for the "chkconfig" program,
-# originally from SGI, used on Red Hat/Fedora and probably elsewhere.
-#
-# chkconfig: 2345 90 10
-# description: Onion Router - A low-latency anonymous proxy
-#
-
-### BEGIN INIT INFO
-# Provides: tor
-# Required-Start: $remote_fs $network
-# Required-Stop: $remote_fs $network
-# Default-Start: 3 5
-# Default-Stop: 0 1 2 6
-# Short-Description: Start the tor daemon
-# Description:  Start the tor daemon:  the anon-proxy server
-### END INIT INFO
-
-. /etc/rc.status
-
-# Shell functions sourced from /etc/rc.status:
-#      rc_check         check and set local and overall rc status
-#      rc_status        check and set local and overall rc status
-#      rc_status -v     ditto but be verbose in local rc status
-#      rc_status -v -r  ditto and clear the local rc status
-#      rc_failed        set local and overall rc status to failed
-#      rc_reset         clear local rc status (overall remains)
-#      rc_exit          exit appropriate to overall rc status
-
-# First reset status of this service
-rc_reset
-
-# Increase open file descriptors a reasonable amount
-ulimit -n 8192
-
-TORCTL=@BINDIR@/torctl
-
-# torctl will use these environment variables
-TORUSER=@TORUSER@
-export TORUSER
-TORGROUP=@TORGROUP@
-export TORGROUP
-
-TOR_DAEMON_PID_DIR="@LOCALSTATEDIR@/run/tor"
-
-if [ -x /bin/su ] ; then
-    SUPROG=/bin/su
-elif [ -x /sbin/su ] ; then
-    SUPROG=/sbin/su
-elif [ -x /usr/bin/su ] ; then
-    SUPROG=/usr/bin/su
-elif [ -x /usr/sbin/su ] ; then
-    SUPROG=/usr/sbin/su
-else
-    SUPROG=/bin/su
-fi
-
-case "$1" in
-
-    start)
-    echo "Starting tor daemon"
-
-    if [ ! -d $TOR_DAEMON_PID_DIR ] ; then
-        mkdir -p $TOR_DAEMON_PID_DIR
-        chown $TORUSER:$TORGROUP $TOR_DAEMON_PID_DIR
-    fi
-
-    ## Start daemon with startproc(8). If this fails
-    ## the echo return value is set appropriate.
-
-    startproc -f $TORCTL start
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    stop)
-    echo "Stopping tor daemon" 
-    startproc -f $TORCTL stop
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    restart)
-    echo "Restarting tor daemon" 
-    startproc -f $TORCTL restart
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    reload)
-    echo "Reloading tor daemon" 
-    startproc -f $TORCTL reload
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    status)
-    startproc -f $TORCTL status
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    *)
-    echo "Usage: $0 (start|stop|restart|reload|status)"
-    RETVAL=1
-esac
-
-rc_exit
diff --git a/contrib/dist/tor.sh.in b/contrib/dist/tor.sh.in
deleted file mode 100644
index 92f890681f..0000000000
--- a/contrib/dist/tor.sh.in
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/sh
-#
-# tor    The Onion Router
-#
-# Startup/shutdown script for tor. This is a wrapper around torctl;
-# torctl does the actual work in a relatively system-independent, or at least
-# distribution-independent, way, and this script deals with fitting the
-# whole thing into the conventions of the particular system at hand.
-# This particular script is written for Red Hat/Fedora Linux, and may
-# also work on Mandrake, but not SuSE.
-#
-# These next couple of lines "declare" tor for the "chkconfig" program,
-# originally from SGI, used on Red Hat/Fedora and probably elsewhere.
-#
-# chkconfig: 2345 90 10
-# description: Onion Router - A low-latency anonymous proxy
-#
-
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-DAEMON=/usr/sbin/tor
-NAME=tor
-DESC="tor daemon"
-TORPIDDIR=/var/run/tor
-TORPID=$TORPIDDIR/tor.pid
-WAITFORDAEMON=60
-ARGS=""
-
-# Library functions
-if [ -f /etc/rc.d/init.d/functions ]; then
-   . /etc/rc.d/init.d/functions
-elif [ -f /etc/init.d/functions ]; then
-   . /etc/init.d/functions
-fi
-
-TORCTL=@BINDIR@/torctl
-
-# torctl will use these environment variables
-TORUSER=@TORUSER@
-export TORUSER
-
-if [ -x /bin/su ] ; then
-    SUPROG=/bin/su
-elif [ -x /sbin/su ] ; then
-    SUPROG=/sbin/su
-elif [ -x /usr/bin/su ] ; then
-    SUPROG=/usr/bin/su
-elif [ -x /usr/sbin/su ] ; then
-    SUPROG=/usr/sbin/su
-else
-    SUPROG=/bin/su
-fi
-
-# Raise ulimit based on number of file descriptors available (thanks, Debian)
-
-if [ -r /proc/sys/fs/file-max ]; then
-	system_max=`cat /proc/sys/fs/file-max`
-	if [ "$system_max" -gt "80000" ] ; then
-		MAX_FILEDESCRIPTORS=32768
-	elif [ "$system_max" -gt "40000" ] ; then
-		MAX_FILEDESCRIPTORS=16384
-	elif [ "$system_max" -gt "10000" ] ; then
-		MAX_FILEDESCRIPTORS=8192
-	else
-		MAX_FILEDESCRIPTORS=1024
-		cat << EOF
-
-Warning: Your system has very few filedescriptors available in total.
-
-Maybe you should try raising that by adding 'fs.file-max=100000' to your
-/etc/sysctl.conf file.  Feel free to pick any number that you deem appropriate.
-Then run 'sysctl -p'.  See /proc/sys/fs/file-max for the current value, and
-file-nr in the same directory for how many of those are used at the moment.
-
-EOF
-	fi
-else
-	MAX_FILEDESCRIPTORS=8192
-fi
-
-NICE=""
-
-case "$1" in
-
-    start)
-	if [ -n "$MAX_FILEDESCRIPTORS" ]; then
-		echo -n "Raising maximum number of filedescriptors (ulimit -n) to $MAX_FILEDESCRIPTORS"
-		if ulimit -n "$MAX_FILEDESCRIPTORS" ; then
-			echo "."
-		else
-			echo ": FAILED."
-		fi
-	fi
-
-    action $"Starting tor:" $TORCTL start
-    RETVAL=$?
-    ;;
-
-    stop)
-    action $"Stopping tor:" $TORCTL stop
-    RETVAL=$?
-    ;;
-
-    restart)
-    action $"Restarting tor:" $TORCTL restart
-    RETVAL=$?
-    ;;
-
-    reload)
-    action $"Reloading tor:" $TORCTL reload
-    RETVAL=$?
-    ;;
-
-    status)
-    $TORCTL status
-    RETVAL=$?
-    ;;
-
-    *)
-    echo "Usage: $0 (start|stop|restart|reload|status)"
-    RETVAL=1
-esac
-
-exit $RETVAL
diff --git a/contrib/dist/torctl.in b/contrib/dist/torctl.in
deleted file mode 100644
index 4cc137da46..0000000000
--- a/contrib/dist/torctl.in
+++ /dev/null
@@ -1,195 +0,0 @@
-#!/bin/sh
-#
-# TOR control script designed to allow an easy command line interface
-# to controlling The Onion Router
-#
-# The exit codes returned are:
-#       0 - operation completed successfully. For "status", tor running.
-#       1 - For "status", tor not running.
-#       2 - Command not supported
-#       3 - Could not be started or reloaded
-#       4 - Could not be stopped
-#       5 -
-#       6 -
-#       7 -
-#       8 -
-#
-# When multiple arguments are given, only the error from the _last_
-# one is reported.
-#
-#
-# |||||||||||||||||||| START CONFIGURATION SECTION  ||||||||||||||||||||
-# --------------------                              --------------------
-# Name of the executable
-EXEC=tor
-#
-# the path to your binary, including options if necessary
-TORBIN="@BINDIR@/$EXEC"
-#
-# the path to the configuration file
-TORCONF="@CONFDIR@/torrc"
-#
-# the path to your PID file
-PIDFILE="@LOCALSTATEDIR@/run/tor/tor.pid"
-#
-# The path to the log file
-LOGFILE="@LOCALSTATEDIR@/log/tor/tor.log"
-#
-# The path to the datadirectory
-TORDATA="@LOCALSTATEDIR@/lib/tor"
-#
-TORARGS="--pidfile $PIDFILE --log \"notice file $LOGFILE\" --runasdaemon 1"
-TORARGS="$TORARGS --datadirectory $TORDATA"
-
-# If user name is set in the environment, then use it;
-# otherwise run as the invoking user (or whatever user the config
-# file says)... unless the invoking user is root. The idea here is to
-# let an unprivileged user run tor for her own use using this script,
-# while still providing for it to be used as a system daemon.
-if [ "x`id -u`" = "x0" ]; then
-    TORUSER=@TORUSER@
-fi
-
-if [ "x$TORUSER" != "x" ]; then
-    TORARGS="$TORARGS --user $TORUSER"
-fi
-
-# We no longer wrap the Tor daemon startup in an su when running as
-# root, because it's too painful to make the use of su portable.
-# Just let the daemon set the UID and GID.
-START="$TORBIN -f $TORCONF $TORARGS"
-
-#
-# --------------------                              --------------------
-# ||||||||||||||||||||   END CONFIGURATION SECTION  ||||||||||||||||||||
-
-ERROR=0
-ARGV="$@"
-if [ "x$ARGV" = "x" ] ; then
-    ARGS="help"
-fi
-
-checkIfRunning ( ) {
-    # check for pidfile
-    PID=unknown
-    if [ -f $PIDFILE ] ; then
-        PID=`/bin/cat $PIDFILE`
-        if [ "x$PID" != "x" ] ; then
-            if kill -0 $PID 2>/dev/null ; then
-               STATUS="$EXEC (pid $PID) running"
-               RUNNING=1
-            else
-                STATUS="PID file ($PIDFILE) present, but $EXEC ($PID) not running"
-                RUNNING=0
-            fi
-        else
-            STATUS="$EXEC (pid $PID?) not running"
-            RUNNING=0
-        fi
-    else
-       STATUS="$EXEC apparently not running (no pid file)"
-       RUNNING=0
-    fi
-    return
-}
-
-for ARG in $@ $ARGS
-do
-    checkIfRunning
-
-    case $ARG in
-    start)
-        if [ $RUNNING -eq 1 ]; then
-            echo "$0 $ARG: $EXEC (pid $PID) already running"
-            continue
-        fi
-        if eval "$START" ; then
-            echo "$0 $ARG: $EXEC started"
-            # Make sure it stayed up!
-            /bin/sleep 1
-            checkIfRunning
-            if [ $RUNNING -eq 0 ]; then
-                echo "$0 $ARG: $EXEC (pid $PID) quit unexpectedly"
-            fi
-        else
-            echo "$0 $ARG: $EXEC could not be started"
-            ERROR=3
-        fi
-        ;;
-    stop)
-        if [ $RUNNING -eq 0 ]; then
-            echo "$0 $ARG: $STATUS"
-            continue
-        fi
-        if kill -15 $PID ; then
-            echo "$0 $ARG: $EXEC stopped"
-        else
-            /bin/sleep 1
-            if kill -9 $PID ; then
-                echo "$0 $ARG: $EXEC stopped"
-            else
-                echo "$0 $ARG: $EXEC could not be stopped"
-                ERROR=4
-            fi
-        fi
-        # Make sure it really died!
-        /bin/sleep 1
-        checkIfRunning
-        if [ $RUNNING -eq 1 ]; then
-            echo "$0 $ARG: $EXEC (pid $PID) unexpectedly still running"
-            ERROR=4
-        fi
-        ;;
-    restart)
-        $0 stop start
-        ;;
-    reload)
-        if [ $RUNNING -eq 0 ]; then
-            echo "$0 $ARG: $STATUS"
-            continue
-        fi
-        if kill -1 $PID; then
-            /bin/sleep 1
-            echo "$EXEC (PID $PID) reloaded"
-        else
-            echo "Can't reload $EXEC"
-            ERROR=3
-        fi
-        ;;
-    status)
-        echo $STATUS
-        if [ $RUNNING -eq 1 ]; then
-            ERROR=0
-        else
-            ERROR=1
-        fi
-        ;;
-    log)
-        cat $LOGFILE
-        ;;
-    help)
-        echo "usage: $0 (start|stop|restart|status|help)"
-        /bin/cat <<EOF
-
-start      - start $EXEC
-stop       - stop $EXEC
-restart    - stop and restart $EXEC if running or start if not running
-reload     - cause the running process to reinitialize itself
-status     - tell whether $EXEC is running or not
-log        - display the contents of the log file
-help       - this text
-
-EOF
-        ERROR=0
-        ;;
-    *)
-        $0 help
-        ERROR=2
-        ;;
-
-    esac
-
-done
-
-exit $ERROR
-
diff --git a/contrib/include.am b/contrib/include.am
index a23e82d6da..60783dc439 100644
--- a/contrib/include.am
+++ b/contrib/include.am
@@ -2,12 +2,7 @@
 EXTRA_DIST+= \
 	contrib/README					\
 	contrib/client-tools/torify			\
-	contrib/dist/rc.subr				\
-	contrib/dist/suse/tor.sh.in			\
-	contrib/dist/tor.sh				\
-	contrib/dist/torctl				\
 	contrib/dist/tor.service.in			\
-	contrib/operator-tools/linux-tor-prio.sh	\
 	contrib/operator-tools/tor-exit-notice.html	\
 	contrib/or-tools/exitlist			\
 	contrib/win32build/tor-mingw.nsi.in		\
diff --git a/contrib/operator-tools/linux-tor-prio.sh b/contrib/operator-tools/linux-tor-prio.sh
deleted file mode 100644
index 30ea5fc659..0000000000
--- a/contrib/operator-tools/linux-tor-prio.sh
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/bin/bash
-# Written by Marco Bonetti & Mike Perry
-# Based on instructions from Dan Singletary's ADSL BW Management HOWTO:
-# http://www.faqs.org/docs/Linux-HOWTO/ADSL-Bandwidth-Management-HOWTO.html
-# This script is Public Domain.
-
-############################### README #################################
-
-# This script provides prioritization of Tor traffic below other
-# traffic on a Linux server. It has two modes of operation: UID based 
-# and IP based. 
-
-# UID BASED PRIORITIZATION
-#
-# The UID based method requires that Tor be launched from 
-# a specific user ID. The "User" Tor config setting is
-# insufficient, as it sets the UID after the socket is created.
-# Here is a C wrapper you can use to execute Tor and drop privs before 
-# it creates any sockets. 
-#
-# Compile with:
-# gcc -DUID=`id -u tor` -DGID=`id -g tor` tor_wrap.c -o tor_wrap
-#
-# #include <unistd.h>
-# int main(int argc, char **argv) {
-# if(initgroups("tor", GID) == -1) { perror("initgroups"); return 1; }
-# if(setresgid(GID, GID, GID) == -1) { perror("setresgid"); return 1; }
-# if(setresuid(UID, UID, UID) == -1) { perror("setresuid"); return 1; }
-# execl("/bin/tor", "/bin/tor", "-f", "/etc/tor/torrc", NULL);
-# perror("execl"); return 1;
-# }
-
-# IP BASED PRIORITIZATION
-#
-# The IP setting requires that a separate IP address be dedicated to Tor. 
-# Your Torrc should be set to bind to this IP for "OutboundBindAddress", 
-# "ListenAddress", and "Address".
-
-# GENERAL USAGE
-#
-# You should also tune the individual connection rate parameters below
-# to your individual connection. In particular, you should leave *some* 
-# minimum amount of bandwidth for Tor, so that Tor users are not 
-# completely choked out when you use your server's bandwidth. 30% is 
-# probably a reasonable choice. More is better of course.
-#
-# To start the shaping, run it as: 
-#   ./linux-tor-prio.sh 
-#
-# To get status information (useful to verify packets are getting marked
-# and prioritized), run:
-#   ./linux-tor-prio.sh status
-#
-# And to stop prioritization:
-#   ./linux-tor-prio.sh stop
-#
-########################################################################
-
-# BEGIN USER TUNABLE PARAMETERS
-
-DEV=eth0
-
-# NOTE! You must START Tor under this UID. Using the Tor User
-# config setting is NOT sufficient. See above.
-TOR_UID=$(id -u tor)
-
-# If the UID mechanism doesn't work for you, you can set this parameter
-# instead. If set, it will take precedence over the UID setting. Note that
-# you need multiple IPs with one specifically devoted to Tor for this to
-# work.
-#TOR_IP="42.42.42.42"
-
-# Average ping to most places on the net, milliseconds
-RTT_LATENCY=40
-
-# RATE_UP must be less than your connection's upload capacity in
-# kbits/sec. If it is larger, then the bottleneck will be at your
-# router's queue, which you do not control. This will cause congestion
-# and a revert to normal TCP fairness no matter what the queing
-# priority is.
-RATE_UP=5000
-
-# RATE_UP_TOR is the minimum speed your Tor connections will have in
-# kbits/sec.  They will have at least this much bandwidth for upload.
-# In general, you probably shouldn't set this too low, or else Tor
-# users who use your node will be completely choked out whenever your
-# machine does any other network activity. That is not very fun.
-RATE_UP_TOR=1500
-
-# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor traffic in
-# kbits/sec.
-RATE_UP_TOR_CEIL=5000
-
-CHAIN=OUTPUT
-#CHAIN=PREROUTING
-#CHAIN=POSTROUTING
-
-MTU=1500
-AVG_PKT=900 # should be more like 600 for non-exit nodes
-
-# END USER TUNABLE PARAMETERS
-
-
-
-# The queue size should be no larger than your bandwidth-delay
-# product. This is RT latency*bandwidth/MTU/2
-
-BDP=$(expr $RTT_LATENCY \* $RATE_UP / $AVG_PKT)
-
-# Further research indicates that the BDP calculations should use
-# RTT/sqrt(n) where n is the expected number of active connections..
-
-BDP=$(expr $BDP / 4)
-
-if [ "$1" = "status" ]
-then
-	echo "[qdisc]"
-	tc -s qdisc show dev $DEV
-	tc -s qdisc show dev imq0
-	echo "[class]"
-	tc -s class show dev $DEV
-	tc -s class show dev imq0
-	echo "[filter]"
-	tc -s filter show dev $DEV
-	tc -s filter show dev imq0
-	echo "[iptables]"
-	iptables -t mangle -L TORSHAPER-OUT -v -x 2> /dev/null
-	exit
-fi
-
-
-# Reset everything to a known state (cleared)
-tc qdisc del dev $DEV root 2> /dev/null > /dev/null
-tc qdisc del dev imq0 root 2> /dev/null > /dev/null
-iptables -t mangle -D POSTROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -D PREROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -D OUTPUT -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -F TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -X TORSHAPER-OUT 2> /dev/null > /dev/null
-ip link set imq0 down 2> /dev/null > /dev/null
-rmmod imq 2> /dev/null > /dev/null
-
-if [ "$1" = "stop" ]
-then
-	echo "Shaping removed on $DEV."
-	exit
-fi
-
-# Outbound Shaping (limits total bandwidth to RATE_UP)
-
-ip link set dev $DEV qlen $BDP
-
-# Add HTB root qdisc, default is high prio
-tc qdisc add dev $DEV root handle 1: htb default 20
-
-# Add main rate limit class
-tc class add dev $DEV parent 1: classid 1:1 htb rate ${RATE_UP}kbit
-
-# Create the two classes, giving Tor at least RATE_UP_TOR kbit and capping
-# total upstream at RATE_UP so the queue is under our control.
-tc class add dev $DEV parent 1:1 classid 1:20 htb rate $(expr $RATE_UP - $RATE_UP_TOR)kbit ceil ${RATE_UP}kbit prio 0
-tc class add dev $DEV parent 1:1 classid 1:21 htb rate $[$RATE_UP_TOR]kbit ceil ${RATE_UP_TOR_CEIL}kbit prio 10
-
-# Start up pfifo
-tc qdisc add dev $DEV parent 1:20 handle 20: pfifo limit $BDP
-tc qdisc add dev $DEV parent 1:21 handle 21: pfifo limit $BDP
-
-# filter traffic into classes by fwmark
-tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 20 fw flowid 1:20
-tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 21 fw flowid 1:21
-
-# add TORSHAPER-OUT chain to the mangle table in iptables
-iptables -t mangle -N TORSHAPER-OUT
-iptables -t mangle -I $CHAIN -o $DEV -j TORSHAPER-OUT
-
-
-# Set firewall marks
-# Low priority to Tor
-if [ ""$TOR_IP == "" ]
-then
-	echo "Using UID-based QoS. UID $TOR_UID marked as low priority."
-	iptables -t mangle -A TORSHAPER-OUT -m owner --uid-owner $TOR_UID -j MARK --set-mark 21
-else
-	echo "Using IP-based QoS. $TOR_IP marked as low priority."
-	iptables -t mangle -A TORSHAPER-OUT -s $TOR_IP -j MARK --set-mark 21
-fi
-
-# High prio for everything else
-iptables -t mangle -A TORSHAPER-OUT -m mark --mark 0 -j MARK --set-mark 20
-
-echo "Outbound shaping added to $DEV.  Rate for Tor upload at least: ${RATE_UP_TOR}Kbyte/sec."
-
diff --git a/contrib/operator-tools/tor-exit-notice.html b/contrib/operator-tools/tor-exit-notice.html
index f0f9a6344c..278d3de502 100644
--- a/contrib/operator-tools/tor-exit-notice.html
+++ b/contrib/operator-tools/tor-exit-notice.html
@@ -254,7 +254,7 @@ seize this router will accomplish nothing.</p>
 <p>
 Furthermore, this machine also serves as a carrier of email, which means that
 its contents are further protected under the ECPA. <a
-href="http://www.law.cornell.edu/uscode/text/18/2707">18
+href="https://www.law.cornell.edu/uscode/text/18/2707">18
 USC 2707</a> explicitly allows for civil remedies ($1000/account
 <i><b>plus</b></i>  legal fees)
 in the event of a seizure executed without good faith or probable cause (it
@@ -272,7 +272,7 @@ used to violate the DMCA, please be aware that this machine does not host or
 contain any illegal content. Also be aware that network infrastructure
 maintainers are not liable for the type of content that passes over their
 equipment, in accordance with <a
-href="http://www.law.cornell.edu/uscode/text/17/512">DMCA
+href="https://www.law.cornell.edu/uscode/text/17/512">DMCA
 "safe harbor" provisions</a>. In other words, you will have just as much luck
 sending a takedown notice to the Internet backbone providers. Please consult
 <a href="https://www.torproject.org/eff/tor-dmca-response">EFF's prepared
diff --git a/contrib/or-tools/check-tor b/contrib/or-tools/check-tor
deleted file mode 100755
index e981a35fcc..0000000000
--- a/contrib/or-tools/check-tor
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/sh
-
-## Originally written by Peter Palfrader.
-
-## This script lets you quickly check if a given router (by nickname)
-## will let you do a TLS handshake, or will let you download a directory.
-
-## Usage: check-tor nickname
-
-#set -x
-
-router="$1"
-dirserver="http://belegost.seul.org:80/tor/"
-
-lines=$( wget -q $dirserver --proxy=off -O - | grep -A5 '^router '"$router"' ' )
-line=$( echo "$lines" | head -n1 )
-
-if [ -z "$line" ]; then
-	echo "Not found" >&2
-	exit 1
-fi
-
-echo "$lines"
-echo
-
-ipor=$( echo "$line" | awk '{printf "%s:%s", $3, $4}' )
-
-op=$( echo "$line" | awk '{printf $6}' )
-ipop=$( echo "$line" | awk '{printf "%s:%s", $3, $6}' )
-
-echo
-echo ">>" openssl s_client -connect "$ipor"
-timeout 5 openssl s_client -connect "$ipor" < /dev/null
-if [ "$op" != "0" ]; then
-	echo
-	echo ">>" wget --proxy=off -O - http://$ipop/tor/
-	timeout 5 wget --proxy=off -O - http://$ipop/tor/ | head -n3
-fi
-
-echo
-echo -n "$router "; echo "$lines" | grep 'fingerprint' | sed -e 's/^opt //' -e 's/^fingerprint //';
diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in
index 234971e178..151127df7f 100644
--- a/contrib/win32build/tor-mingw.nsi.in
+++ b/contrib/win32build/tor-mingw.nsi.in
@@ -8,7 +8,7 @@
 !include "LogicLib.nsh"
 !include "FileFunc.nsh"
 !insertmacro GetParameters
-!define VERSION "0.3.5.15-dev"
+!define VERSION "0.4.4.9"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"
diff --git a/doc/HACKING/CircuitPaddingDevelopment.md b/doc/HACKING/CircuitPaddingDevelopment.md
new file mode 100644
index 0000000000..a4e65697b8
--- /dev/null
+++ b/doc/HACKING/CircuitPaddingDevelopment.md
@@ -0,0 +1,1234 @@
+# Circuit Padding Developer Documentation
+
+This document is written for researchers who wish to prototype and evaluate circuit-level padding defenses in Tor.
+
+Written by Mike Perry and George Kadianakis.
+
+# Table of Contents
+
+- [0. Background](#0-background)
+- [1. Introduction](#1-introduction)
+    - [1.1. System Overview](#11-system-overview)
+    - [1.2. Layering Model](#12-layering-model)
+    - [1.3. Computation Model](#13-computation-model)
+    - [1.4. Deployment Constraints](#14-other-deployment-constraints)
+- [2. Creating New Padding Machines](#2-creating-new-padding-machines)
+    - [2.1. Registering a New Padding Machine](#21-registering-a-new-padding-machine)
+    - [2.2. Machine Activation and Shutdown](#22-machine-activation-and-shutdown)
+- [3. Specifying Padding Machines](#3-specifying-padding-machines)
+    - [3.1. Padding Machine States](#31-padding-machine-states)
+    - [3.2. Padding Machine State Transitions](#32-padding-machine-state-transitions)
+    - [3.3. Specifying Per-State Padding](#33-specifying-per-state-padding)
+    - [3.4. Specifying Precise Cell Counts](#34-specifying-precise-cell-counts)
+    - [3.5. Specifying Overhead Limits](#35-specifying-overhead-limits)
+- [4. Evaluating Padding Machines](#4-evaluating-padding-machines)
+    - [4.1. Pure Simulation](#41-pure-simulation)
+    - [4.2. Testing in Chutney](#42-testing-in-chutney)
+    - [4.3. Testing in Shadow](#43-testing-in-shadow)
+    - [4.4. Testing on the Live Network](#44-testing-on-the-live-network)
+- [5. Example Padding Machines](#5-example-padding-machines)
+    - [5.1. Deployed Circuit Setup Machines](#51-deployed-circuit-setup-machines)
+    - [5.2. Adaptive Padding Early](#52-adaptive-padding-early)
+    - [5.3. Sketch of Tamaraw](#53-sketch-of-tamaraw)
+    - [5.4. Other Padding Machines](#54-other-padding-machines)
+- [6. Framework Implementation Details](#6-framework-implementation-details)
+    - [6.1. Memory Allocation Conventions](#61-memory-allocation-conventions)
+    - [6.2. Machine Application Events](#62-machine-application-events)
+    - [6.3. Internal Machine Events](#63-internal-machine-events)
+- [7. Future Features and Optimizations](#7-future-features-and-optimizations)
+    - [7.1. Load Balancing and Flow Control](#71-load-balancing-and-flow-control)
+    - [7.2. Timing and Queuing Optimizations](#72-timing-and-queuing-optimizations)
+    - [7.3. Better Machine Negotiation](#73-better-machine-negotiation)
+    - [7.4. Probabilistic State Transitions](#74-probabilistic-state-transitions)
+    - [7.5. More Complex Pattern Recognition](#75-more-complex-pattern-recognition)
+- [8. Open Research Problems](#8-open-research-problems)
+    - [8.1. Onion Service Circuit Setup](#81-onion-service-circuit-setup)
+    - [8.2. Onion Service Fingerprinting](#82-onion-service-fingerprinting)
+    - [8.3. Open World Fingerprinting](#83-open-world-fingerprinting)
+    - [8.4. Protocol Usage Fingerprinting](#84-protocol-usage-fingerprinting)
+    - [8.5. Datagram Transport Side Channels](#85-datagram-transport-side-channels)
+- [9. Must Read Papers](#9-must-read-papers)
+
+
+## 0. Background
+
+Tor supports both connection-level and circuit-level padding, and both
+systems are live on the network today. The connection-level padding behavior
+is described in [section 2 of
+padding-spec.txt](https://github.com/torproject/torspec/blob/master/padding-spec.txt#L47). The
+circuit-level padding behavior is described in [section 3 of
+padding-spec.txt](https://github.com/torproject/torspec/blob/master/padding-spec.txt#L282).
+
+These two systems are orthogonal and should not be confused. The
+connection-level padding system is only active while the TLS connection is
+otherwise idle. Moreover, it regards circuit-level padding as normal data
+traffic, and hence while the circuit-level padding system is actively padding,
+the connection-level padding system will not add any additional overhead.
+
+While the currently deployed circuit-level padding behavior is quite simple,
+it is built on a flexible framework. This framework supports the description
+of event-driven finite state machine by filling in fields of a simple C
+structure, and is designed to support any delay-free statistically shaped
+cover traffic on individual circuits, with cover traffic flowing to and from a
+node of the implementor's choice (Guard, Middle, Exit, Rendezvous, etc).
+
+This class of system was first proposed in
+[Timing analysis in low-latency mix networks: attacks and defenses](https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf)
+by Shmatikov and Wang, and extended for the website traffic fingerprinting
+domain by Juarez et al. in
+[Toward an Efficient Website Fingerprinting Defense](http://arxiv.org/pdf/1512.00524). The
+framework also supports fixed parameterized probability distributions, as
+used in [APE](https://www.cs.kau.se/pulls/hot/thebasketcase-ape/) by Tobias
+Pulls, and many other features.
+
+This document describes how to use Tor's circuit padding framework to
+implement and deploy novel delay-free cover traffic defenses.
+
+## 1. Introduction
+
+The circuit padding framework is the official way to implement padding
+defenses in Tor. It may be used in combination with application-layer
+defenses, and/or obfuscation defenses, or on its own.
+
+Its current design should be enough to deploy most defenses without
+modification, but you can extend it to [provide new
+features](#7-future-features-and-optimizations) as well.
+
+### 1.1. System Overview
+
+Circuit-level padding can occur between Tor clients and relays at any hop of
+one of the client's circuits. Both parties need to support the same padding
+mechanisms for the system to work, and the client must enable it.
+
+We added a padding negotiation relay cell to the Tor protocol that clients use
+to ask a relay to start padding, as well as a torrc directive for researchers
+to pin their clients' relay selection to the subset of Tor nodes that
+implement their custom defenses, to support ethical live network testing and
+evaluation.
+
+Circuit-level padding is performed by 'padding machines'. A padding machine is
+a finite state machine. Every state specifies a different form of
+padding style, or stage of padding, in terms of inter-packet timings and total
+packet counts.
+
+Padding state machines are specified by filling in fields of a C structure,
+which specifies the transitions between padding states based on various events,
+probability distributions of inter-packet delays, and the conditions under
+which padding machines should be applied to circuits.
+
+This compact C structure representation is designed to function as a
+microlanguage, which can be compiled down into a
+bitstring that [can be tuned](#13-computation-model) using various
+optimization methods (such as gradient descent, GAs, or GANs), either in
+bitstring form or C struct form.
+
+The event driven, self-contained nature of this framework is also designed to
+make [evaluation](#4-evaluating-padding-machines) both expedient and rigorously
+reproducible.
+
+This document covers the engineering steps to write, test, and deploy a
+padding machine, as well as how to extend the framework to support new machine
+features.
+
+If you prefer to learn by example, you may want to skip to either the
+[QuickStart Guide](CircuitPaddingQuickStart.md), and/or [Section
+5](#5-example-padding-machines) for example machines to get you up and running
+quickly.
+
+### 1.2. Layering Model
+
+The circuit padding framework is designed to provide one layer in a layered
+system of interchangeable components.
+
+The circuit padding framework operates at the Tor circuit layer. It only deals
+with the inter-cell timings and quantity of cells sent on a circuit. It can
+insert cells on a circuit in arbitrary patterns, and in response to arbitrary
+conditions, but it cannot delay cells. It also does not deal with packet
+sizes, how cells are packed into TLS records, or ways that the Tor protocol
+might be recognized on the wire.
+
+The problem of differentiating Tor traffic from non-Tor traffic based on
+TCP/TLS packet sizes, initial handshake patterns, and DPI characteristics is the
+domain of [pluggable
+transports](https://trac.torproject.org/projects/tor/wiki/doc/AChildsGardenOfPluggableTransports),
+which may optionally be used in conjunction with this framework (or without
+it).
+
+This document focuses primarily on the circuit padding framework's cover
+traffic features, and will only briefly touch on the potential obfuscation and
+application layer coupling points of the framework. Explicit layer coupling 
+points can be created by adding either new [machine application
+events](#62-machine-application-events) or new [internal machine
+events](#63-internal-machine-events) to the circuit padding framework, so that
+your padding machines can react to events from other layers.
+
+### 1.3. Computation Model
+
+The circuit padding framework is designed to support succinctly specified
+defenses that can be tuned through [computer-assisted
+optimization](#4-evaluating-padding-machines).
+
+We chose to generalize the original [Adaptive Padding 2-state
+design](https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf) into an
+event-driven state machine because state machines are the simplest form of
+sequence recognition devices from [automata
+theory](https://en.wikipedia.org/wiki/Finite-state_machine).
+
+Most importantly: this framing allows cover traffic defenses to be modeled as
+an optimization problem search space, expressed as fields of a C structure
+(which is simultaneously a compact opaque bitstring as well as a symbolic
+vector in an abstract feature space). This kind of space is particularly well
+suited to search by gradient descent, GAs, and GANs. 
+
+When performing this optimization search, each padding machine should have a
+fitness function, which will allow two padding machines to be compared for
+relative effectiveness. Optimization searches work best if this fitness can be
+represented as a single number, for example the total amount by which it
+reduces the [Balanced
+Accuracy](https://en.wikipedia.org/wiki/Precision_and_recall#Imbalanced_Data)
+of an adversary's classifier, divided by an amount of traffic overhead. 
+
+Before you begin the optimization phase for your defense, you should
+also carefully consider the [features and
+optimizations](#7-future-features-and-optimizations) that we suspect will be
+useful, and also see if you can come up with any more. You should similarly be
+sure to restrict your search space to avoid areas of the bitstring/feature
+vector that you are sure you will not need. For example, some
+[applications](#8-open-research-problems) may not need the histogram
+accounting used by Adaptive Padding, but might need to add other forms of
+[pattern recognition](#75-more-complex-pattern-recognition) to react to
+sequences that resemble HTTP GET and HTTP POST.
+
+### 1.4. Other Deployment Constraints
+
+The framework has some limitations that are the result of deliberate
+choices. We are unlikely to deploy defenses that ignore these limitations.
+
+In particular, we have deliberately not provided any mechanism to delay actual
+user traffic, even though we are keenly aware that if we were to support
+additional delay, defenses would be able to have [more success with less
+bandwidth
+overhead](https://freedom.cs.purdue.edu/anonymity/trilemma/index.html).
+
+In the website traffic fingerprinting domain, [provably optimal
+defenses](https://www.cypherpunks.ca/~iang/pubs/webfingerprint-ccs14.pdf)
+achieve their bandwidth overhead bounds by ensuring that a non-empty queue is
+maintained, by rate limiting traffic below the actual throughput of a circuit.
+For optimal results, this queue must avoid draining to empty, and yet it
+must also be drained fast enough to avoid tremendous queue overhead in fast
+Tor relays, which carry hundreds of thousands of circuits simultaneously.
+
+Unfortunately, Tor's end-to-end flow control is not congestion control. Its
+window sizes are currently fixed. This means there is no signal when queuing
+occurs, and thus no ability to limit queue size through pushback. This means
+there is currently no way to do the fine-grained queue management necessary to
+create such a queue and rate limit traffic effectively enough to keep this
+queue from draining to empty, without also risking that aggregate queuing
+would cause out-of-memory conditions on fast relays.
+
+It may be possible to create a congestion control algorithm that can support
+such fine grained queue management, but this is a [deeply unsolved area of
+research](https://lists.torproject.org/pipermail/tor-dev/2018-November/013562.html).
+
+Even beyond these major technical hurdles, additional latency is also
+unappealing to the wider Internet community, for the simple reason that
+bandwidth [continues to increase
+exponentially](https://ipcarrier.blogspot.com/2014/02/bandwidth-growth-nearly-what-one-would.html)
+where as the speed of light is fixed. Significant engineering effort has been
+devoted to optimizations that reduce the effect of latency on Internet
+protocols. To go against this trend would ensure our irrelevance to the wider
+conversation about traffic analysis defenses for low latency Internet protocols.
+
+On the other hand, through [load
+balancing](https://gitweb.torproject.org/torspec.git/tree/proposals/265-load-balancing-with-overhead.txt)
+and [circuit multiplexing strategies](https://bugs.torproject.org/29494), we
+believe it is possible to add significant bandwidth overhead in the form of
+cover traffic, without significantly impacting end-user performance.
+
+For these reasons, we believe the trade-off should be in favor of adding more
+cover traffic, rather than imposing queuing memory overhead and queuing delay.
+
+As a last resort for narrowly scoped application domains (such as
+shaping Tor service-side onion service traffic to look like other websites or
+different application-layer protocols), delay *may* be added at the
+[application layer](https://petsymposium.org/2017/papers/issue2/paper54-2017-2-source.pdf).
+Any additional cover traffic required by such defenses should still be
+added at the circuit padding layer using this framework, to provide
+engineering efficiency through loose layer coupling and component re-use, as
+well as to provide additional gains against [low
+resolution](https://github.com/torproject/torspec/blob/master/padding-spec.txt#L47)
+end-to-end traffic correlation.
+
+Because such delay-based defenses will impact performance significantly more
+than simply adding cover traffic, they must be optional, and negotiated by
+only specific application layer endpoints that want them. This will have
+consequences for anonymity sets and base rates, if such traffic shaping and
+additional cover traffic is not very carefully constructed.
+
+In terms of acceptable overhead, because Tor onion services
+[currently use](https://metrics.torproject.org/hidserv-rend-relayed-cells.html)
+less than 1% of the
+[total consumed bandwidth](https://metrics.torproject.org/bandwidth-flags.html)
+of the Tor network, and because onion services exist to provide higher
+security as compared to Tor Exit traffic, they are an attractive target for
+higher-overhead defenses. We encourage researchers to target this use case
+for defenses that require more overhead, and/or for the deployment of
+optional negotiated application-layer delays on either the server or the
+client side.
+
+## 2. Creating New Padding Machines
+
+This section explains how to use the existing mechanisms in Tor to define a
+new circuit padding machine.  We assume here that you know C, and are at
+least somewhat familiar with Tor development.  For more information on Tor
+development in general, see the other files in doc/HACKING/ in a recent Tor
+distribution.
+
+Again, if you prefer to learn by example, you may want to skip to either the
+[QuickStart Guide](CircuitPaddingQuickStart.md), and/or [Section
+5](#5-example-padding-machines) for example machines to get up and running
+quickly.
+
+To create a new padding machine, you must:
+
+  1. Define your machine using the fields of a heap-allocated
+     `circpad_machine_spec_t` C structure.
+
+  2. Register this object in the global list of available padding machines,
+     using `circpad_register_padding_machine()`.
+
+  3. Ensure that your machine is properly negotiated under your desired
+     circuit conditions.
+
+### 2.1. Registering a New Padding Machine
+
+Again, a circuit padding machine is designed to be specified entirely as a [single
+C structure](#13-computation-model).
+
+Your machine definitions should go into their own functions in
+[circuitpadding_machines.c](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding_machines.c). For
+details on all of the fields involved in specifying a padding machine, see
+[Section 3](#3-specifying-padding-machines).
+
+You must register your machine in `circpad_machines_init()` in
+[circuitpadding.c](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding.c). To
+add a new padding machine specification, you must allocate a
+`circpad_machine_spec_t` on the heap with `tor_malloc_zero()`, give it a
+human readable name string, and a machine number equivalent to the number of
+machines in the list, and register the structure using
+`circpad_register_padding_machine()`.
+
+Each machine must have a client instance and a relay instance. Register your
+client-side machine instance in the `origin_padding_machines` list, and your
+relay side machine instance in the `relay_padding_machines` list. Once you
+have registered your instance, you do not need to worry about deallocation;
+this is handled for you automatically.
+
+Both machine lists use registration order to signal machine precedence for a
+given `machine_idx` slot on a circuit. This means that machines that are
+registered last are checked for activation *before* machines that are
+registered first. (This reverse precedence ordering allows us to
+deprecate older machines simply by adding new ones after them.)
+
+### 2.2. Machine Activation and Shutdown
+
+After a machine has been successfully registered with the framework, it will
+be instantiated on any client-side circuits that support it. Only client-side
+circuits may initiate padding machines, but either clients or relays may shut
+down padding machines.
+
+#### 2.2.1. Machine Application Conditions
+
+The
+[circpad_machine_conditions_t conditions field](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L641)
+of your `circpad_machine_spec_t` machine definition instance controls the
+conditions under which your machine will be attached and enabled on a Tor
+circuit, and when it gets shut down.
+
+*All* of your explicitly specified conditions in
+`circpad_machine_spec_t.conditions` *must* be met for the machine to be
+applied to a circuit. If *any* condition ceases to be met, then the machine
+is shut down.  (This is checked on every event that arrives, even if the
+condition is unrelated to the event.)
+Another way to look at this is that
+all specified conditions must evaluate to true for the entire duration that
+your machine is running. If any are false, your machine does not run (or
+stops running and shuts down).
+
+In particular, as part of the
+[circpad_machine_conditions_t structure](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding.h#L149),
+the circuit padding subsystem gives the developer the option to enable a
+machine based on:
+  - The
+    [length](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L157)
+    on the circuit (via the `min_hops` field).
+  - The
+    [current state](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L174)
+    of the circuit, such as streams, relay_early, etc. (via the
+    `circpad_circuit_state_t state_mask` field).
+  - The
+    [purpose](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L178)
+    (i.e. type) of the circuit (via the `circpad_purpose_mask_t purpose_mask`
+    field).
+
+This condition mechanism is the preferred way to determine if a machine should
+apply to a circuit. For information about potentially useful conditions that
+we have considered but have not yet implemented, see [Section
+7.3](#73-better-machine-negotiation). We will happily accept patches for those
+conditions, or any for other additional conditions that are needed for your
+use case.
+
+#### 2.2.2. Detecting and Negotiating Machine Support
+
+When a new machine specification is added to Tor (or removed from Tor), you
+should bump the Padding subprotocol version in `src/core/or/protover.c` and
+`src/rust/protover/protover.rs`, add a field to `protover_summary_flags_t` in
+`or.h`, and set this field in `memoize_protover_summary()` in versions.c. This
+new field must then be checked in `circpad_node_supports_padding()` in
+`circuitpadding.c`.
+
+Note that this protocol version update and associated support check is not
+necessary if your experiments will *only* be using your own relays that
+support your own padding machines. This can be accomplished by using the
+`MiddleNodes` directive; see [Section 4](#4-evaluating-padding-machines) for more information.
+
+If the protocol support check passes for the circuit, then the client sends a
+`RELAY_COMMAND_PADDING_NEGOTIATE` cell towards the
+`circpad_machine_spec_t.target_hop` relay, and immediately enables the
+padding machine, and may begin sending padding. (The framework does not wait
+for the `RELAY_COMMAND_PADDING_NEGOTIATED` response to begin padding,
+so that we can
+switch between machines rapidly.)
+
+#### 2.2.3. Machine Shutdown Mechanisms
+
+Padding machines can be shut down on a circuit in three main ways:
+  1. During a `circpad_machine_event` callback, when
+     `circpad_machine_spec_t.conditions` no longer applies (client side)
+  2. After a transition to the CIRCPAD_STATE_END, if
+     `circpad_machine_spec_t.should_negotiate_end` is set (client or relay
+     side)
+  3. If there is a `RELAY_COMMAND_PADDING_NEGOTIATED` error response from the
+     relay during negotiation.
+
+Each of these cases causes the originating node to send a relay cell towards
+the other side, indicating that shutdown has occurred. The client side sends
+`RELAY_COMMAND_PADDING_NEGOTIATE`, and the relay side sends
+`RELAY_COMMAND_PADDING_NEGOTIATED`.
+
+Because padding from malicious exit nodes can be used to construct active
+timing-based side channels to malicious guard nodes, the client checks that
+padding-related cells only come from relays with active padding machines.
+For this reason, when a client decides to shut down a padding machine,
+the framework frees the mutable `circuit_t.padding_info`, but leaves the
+`circuit_t.padding_machine` pointer set until the
+`RELAY_COMMAND_PADDING_NEGOTIATED` response comes back, to ensure that any
+remaining in-flight padding packets are recognized a valid. Tor does
+not yet close circuits due to violation of this property, but the
+[vanguards addon component "bandguard"](https://github.com/mikeperry-tor/vanguards/blob/master/README_TECHNICAL.md#the-bandguards-subsystem)
+does.
+
+As an optimization, a client may replace a machine with another, by
+sending a `RELAY_COMMAND_PADDING_NEGOTIATE` cell to shut down a machine, and
+immediately sending a `RELAY_COMMAND_PADDING_NEGOTIATE` to start a new machine
+in the same index, without waiting for the response from the first negotiate
+cell.
+
+Unfortunately, there is a known bug as a consequence of this optimization. If
+your machine depends on repeated shutdown and restart of the same machine
+number on the same circuit, please see [Bug
+30922](https://bugs.torproject.org/30992). Depending on your use case, we may
+need to fix that bug or help you find a workaround. See also [Section
+6.1.3](#613-deallocation-and-shutdown) for some more technical details on this
+mechanism.
+
+
+## 3. Specifying Padding Machines
+
+By now, you should understand how to register, negotiate, and control the
+lifetime of your padding machine, but you still don't know how to make it do
+anything yet. This section should help you understand how to specify how your
+machine reacts to events and adds padding to the wire.
+
+If you prefer to learn by example first instead, you may wish to skip to
+[Section 5](#5-example-padding-machines).
+
+
+A padding machine is specified by filling in an instance of
+[circpad_machine_spec_t](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L605). Instances
+of this structure specify the precise functionality of a machine: it's
+what the circuit padding developer is called to write. These instances
+are created only at startup, and are referenced via `const` pointers during
+normal operation.
+
+In this section we will go through the most important elements of this
+structure.
+
+### 3.1. Padding Machine States
+
+A padding machine is a finite state machine where each state
+specifies a different style of padding.
+
+As an example of a simple padding machine, you could have a state machine
+with the following states: `[START] -> [SETUP] -> [HTTP] -> [END]` where the
+`[SETUP]` state pads in a way that obfuscates the ''circuit setup'' of Tor,
+and the `[HTTP]` state pads in a way that emulates a simple HTTP session. Of
+course, padding machines can be more complicated than that, with dozens of
+states and non-trivial transitions.
+
+Padding developers encode the machine states in the
+[circpad_machine_spec_t structure](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L655). Each
+machine state is described by a
+[circpad_state_t structure](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L273)
+and each such structure specifies the style and amount of padding to be sent,
+as well as the possible state transitions.
+
+The function `circpad_machine_states_init()` must be used for allocating and
+initializing the `circpad_machine_spec_t.states` array before states and
+state transitions can be defined, as some of the state object has non-zero
+default values.
+
+### 3.2. Padding Machine State Transitions
+
+As described above, padding machines can have multiple states, to
+support different forms of padding. Machines can transition between
+states based on events that occur either on the circuit level or on
+the machine level.
+
+State transitions are specified using the
+[next_state field](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding.h#L381)
+of the `circpad_state_t` structure. As a simple example, to transition
+from state `A` to state `B` when event `E` occurs, you would use the
+following code: `A.next_state[E] = B`.
+
+#### 3.2.1. State Transition Events
+
+Here we will go through
+[the various events](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding.h#L30)
+that can be used to transition between states:
+
+* Circuit-level events
+  * `CIRCPAD_EVENT_NONPADDING_RECV`: A non-padding cell is received
+  * `CIRCPAD_EVENT_NONPADDING_SENT`: A non-adding cell is sent
+  * `CIRCPAD_EVENT_PADDING_SENT`: A padding cell is sent
+  * `CIRCPAD_EVENT_PADDING_RECV`: A padding cell is received
+* Machine-level events
+  * `CIRCPAD_EVENT_INFINITY`: Tried to schedule padding using the ''infinity bin''.
+  * `CIRCPAD_EVENT_BINS_EMPTY`: All histogram bins are empty (out of tokens)
+  * `CIRCPAD_EVENT_LENGTH_COUNT`: State has used all its padding capacity (see `length_dist` below)
+
+### 3.3. Specifying Per-State Padding
+
+Each state of a padding machine specifies either:
+  * A padding histogram describing inter-transmission delays between cells;
+d     OR
+  * A parameterized delay probability distribution for inter-transmission
+    delays between cells.
+
+Either mechanism specifies essentially the *minimum inter-transmission time*
+distribution. If non-padding traffic does not get transmitted from this
+endpoint before the delay value sampled from this distribution expires, a
+padding packet is sent.
+
+The choice between histograms and probability distributions can be subtle. A
+rule of thumb is that probability distributions are easy to specify and
+consume very little memory, but might not be able to describe certain types
+of complex padding logic. Histograms, in contrast, can support precise
+packet-count oriented or multimodal delay schemes, and can use token removal
+logic to reduce overhead and shape the total padding+non-padding inter-packet
+delay distribution towards an overall target distribution.
+
+We suggest that you start with a probability distribution if possible, and
+you move to a histogram-based approach only if a probability distribution
+does not suit your needs.
+
+#### 3.3.1. Padding Probability Distributions
+
+The easiest, most compact way to schedule padding using a machine state is to
+use a probability distribution that specifies the possible delays. That can
+be done
+[using the circpad_state_t fields](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L339)
+`iat_dist`, `dist_max_sample_usec` and `dist_added_shift_usec`.
+
+The Tor circuit padding framework
+[supports multiple types](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L214)
+of probability distributions, and the developer should use the
+[circpad_distribution_t structure](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L240)
+to specify them as well as the required parameters.
+
+#### 3.3.2. Padding Histograms
+
+A more advanced way to schedule padding is to use a ''padding
+histogram''. The main advantages of a histogram are that it allows you to
+specify distributions that are not easily parameterized in closed form, or
+require specific packet counts at particular time intervals. Histograms also
+allow you to make use of an optional traffic minimization and shaping
+optimization called *token removal*, which is central to the original
+[Adaptive Padding](https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf)
+concept.
+
+If a histogram is used by a state (as opposed to a fixed parameterized
+distribution), then the developer must use the
+[histogram-related fields](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L285)
+of the `circpad_state_t` structure.
+
+The width of a histogram bin specifies the range of inter-packet delay times,
+whereas its height specifies the amount of tokens in each bin. To sample a
+padding delay from a histogram, we first randomly pick a bin (weighted by the
+amount of tokens in each bin) and then sample a delay from within that bin by
+picking a uniformly random delay using the width of the bin as the range.
+
+Each histogram also has an ''infinity bin'' as its final bin.  If the
+''infinity bin'' is chosen,
+we don't schedule any padding (i.e., we schedule padding with
+infinite delay). If the developer does not want infinite delay, they
+should not give any tokens to the ''infinity bin''.
+
+If a token removal strategy is specified (via the
+`circpad_state_t.token_removal` field), each time padding is sent using a
+histogram, the padding machine will remove a token from the appropriate
+histogram bin whenever this endpoint sends *either a padding packet or a
+non-padding packet*. The different removal strategies govern what to do when
+the bin corresponding to the current inter-packet delay is empty.
+
+Token removal is optional. It is useful if you want to do things like specify
+a burst should be at least N packets long, and you only want to add padding
+packets if there are not enough non-padding packets. The cost of doing token
+removal is additional memory allocations for making per-circuit copies of
+your histogram that can be modified.
+
+### 3.4. Specifying Precise Cell Counts
+
+Padding machines should be able to specify the exact amount of padding they
+send. For histogram-based machines this can be done using a specific amount
+of tokens, but another (and perhaps easier) way to do this, is to use the
+[length_dist field](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.h#L362)
+of the `circpad_state_t` structure.
+
+The `length_dist` field is basically a probability distribution similar to the
+padding probability distributions, which applies to a specific machine state
+and specifies the amount of padding we are willing to send during that state.
+This value gets sampled when we transition to that state (TODO document this
+in the code).
+
+### 3.5. Specifying Overhead Limits
+
+Separately from the length counts, it is possible to rate limit the overhead
+percentage of padding at both the global level across all machines, and on a
+per-machine basis.
+
+At the global level, the overhead percentage of all circuit padding machines
+as compared to total traffic can be limited through the Tor consensus
+parameter `circpad_global_max_padding_pct`. This overhead is defined as the
+percentage of padding cells *sent* out of the sum of non padding and padding
+cells *sent*, and is applied *only after* at least
+`circpad_global_allowed_cells` padding cells are sent by that relay or client
+(to allow for small bursts of pure padding on otherwise idle or freshly
+restarted relays). When both of these limits are hit by a relay or client, no
+further padding cells will be sent, until sufficient non-padding traffic is
+sent to cause the percentage of padding traffic to fall back below the
+threshold.
+
+Additionally, each individual padding machine can rate limit itself by
+filling in the fields `circpad_machine_spec_t.max_padding_percent` and
+`circpad_machine_spec_t.allowed_padding_count`, which behave identically to
+the consensus parameters, but only apply to that specific machine.
+
+## 4. Evaluating Padding Machines
+
+One of the goals of the circuit padding framework is to provide improved
+evaluation and scientific reproducibility for lower cost. This includes both
+the [choice](#13-computation-model) of the compact C structure representation
+(which has an easy-to-produce bitstring representation for optimization by
+gradient descent, GAs, or GANs), as well as rapid prototyping and evaluation.
+
+So far, whenever evaluation cost has been a barrier, each research group has
+developed their own ad-hoc packet-level simulators of various padding
+mechanisms for evaluating website fingerprinting attacks and defenses. The
+process typically involves doing a crawl of Alexa top sites over Tor, and
+recording the Tor cell count and timing information for each page in the
+trace. These traces are then fed to simulations of defenses, which output
+modified trace files.
+
+Because no standardized simulation and evaluation mechanism exists, it is
+often hard to tell if independent implementations of various attacks and
+defenses are in fact true-to-form or even properly calibrated for direct
+comparison, and discrepancies in results across the literature suggests
+this is not always so.
+
+Our preferred outcome with this framework is that machines are tuned
+and optimized on a tracing simulator, but that the final results come from
+an actual live network test of the defense. The traces from this final crawl
+should be preserved as artifacts to be run on the simulator and reproduced
+on the live network by future papers, ideally in journal venues that have an
+artifact preservation policy.
+
+### 4.1. Pure Simulation
+
+When doing initial tuning of padding machines, especially in adversarial
+settings, variations of a padding machine defense may need to be applied to
+network activity hundreds or even millions of times. The wall-clock time
+required to do this kind of tuning using live testing or even Shadow network
+emulation may often be prohibitive.
+
+To help address this, and to better standardize results, Tobias Pulls has
+implemented a [circpad machine trace simulator](https://github.com/pylls/circpad-sim),
+which uses Tor's unit test framework to simulate applying padding machines to
+circuit packet traces via a combination of Tor patches and python scripts. This
+simulator can be used to record traces from clients, Guards, Middles, Exits,
+and any other hop in the path, only for circuits that are run by the
+researcher. This makes it possible to safely record baseline traces and
+ultimately even mount passive attacks on the live network, without impacting
+or recording any normal user traffic.
+
+In this way, a live crawl of the Alexa top sites could be performed once, to
+produce a standard "undefended" corpus. Padding machines can be then quickly
+evaluated and tuned on these simulated traces in a standardized way, and then
+the results can then be [reproduced on the live Tor
+network](#44-Testing-on-the-Live-Network) with the machines running on your own relays.
+
+Please be mindful of the Limitations section of the simulator documentation,
+however, to ensure that you are aware of the edge cases and timing
+approximations that are introduced by this approach.
+
+### 4.2. Testing in Chutney
+
+The Tor Project provides a tool called
+[Chutney](https://github.com/torproject/chutney/) which makes it very easy to
+setup private Tor networks. While getting it work for the first time might
+take you some time of doc reading, the final result is well worth it for the
+following reasons:
+
+- You control all the relays and hence you have greater control and debugging
+  capabilities.
+- You control all the relays and hence you can toggle padding support on/off
+  at will.
+- You don't need to be cautious about overhead or damaging the real Tor
+  network during testing.
+- You don't even need to be online; you can do all your testing offline over
+  localhost.
+
+A final word of warning here is that since Chutney runs over localhost, the
+packet latencies and delays are completely different from the real Tor
+network, so if your padding machines rely on real network timings you will
+get different results on Chutney. You can work around this by using a
+different set of delays if Chutney is used, or by moving your padding
+machines to the real network when you want to do latency-related testing.
+
+### 4.3. Testing in Shadow
+
+[Shadow](https://shadow.github.io/) is an environment for running entire Tor
+network simulations, similar to Chutney, but designed to be both more memory
+efficient, as well as provide an accurate Tor network topology and latency
+model.
+
+While Shadow is significantly more memory efficient than Chutney, and can make
+use of extremely accurate Tor network capacity and latency models, it will not
+be as fast or efficient as the [circpad trace simulator](https://github.com/pylls/circpad-sim),
+if you need to do many many iterations of an experiment to tune your defense.
+
+### 4.4. Testing on the Live Network
+
+Live network testing is the gold standard for verifying that any attack or
+defense is behaving as expected, to minimize the influence of simplifying
+assumptions.
+
+However, it is not ethical, or necessarily possible, to run high-resolution
+traffic analysis attacks on the entire Tor network. But it is both ethical
+and possible to run small scale experiments that target only your own
+clients, who will only use your own Tor relays that support your new padding
+machines.
+
+We provide the `MiddleNodes` torrc directive to enable this, which will allow
+you to specify the fingerprints and/or IP netmasks of relays to be used in
+the second hop position. Options to restrict other hops also exist, if your
+padding system is padding to a different hop. The `HSLayer2Nodes` option
+overrides the `MiddleNodes` option for onion service circuits, if both are
+set. (The
+[vanguards addon](https://github.com/mikeperry-tor/vanguards/README_TECHNICAL.md)
+will set `HSLayer2Nodes`.)
+
+When you run your own clients, and use MiddleNodes to restrict your clients
+to use your relays, you can perform live network evaluations of a defense
+applied to whatever traffic crawl or activity your clients do.
+
+## 5. Example Padding Machines
+
+### 5.1. Deployed Circuit Setup Machines
+
+Tor currently has two padding machines enabled by default, which aim to hide
+certain features of the client-side onion service circuit setup protocol. For
+more details on their precise goal and function, please see
+[proposal 302](https://github.com/torproject/torspec/blob/master/proposals/302-padding-machines-for-onion-clients.txt)
+. In this section we will go over the code of those machines to clarify some
+of the engineering parts:
+
+#### 5.1.1. Overview
+
+The codebase of proposal 302 can be found in
+[circuitpadding_machines.c](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding_machines.c)
+and specifies four padding machines:
+
+- The [client-side introduction](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L60) circuit machine.
+- The [relay-side introduction](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L146) circuit machine.
+- The [client-side rendezvous](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L257) circuit machine
+- The [relay-side rendezvous](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L374) circuit machine.
+
+Each of those machines has its own setup function, and
+[they are all initialized](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding.c#L2718)
+by the circuit padding framework. To understand more about them, please
+carefully read the individual setup function for each machine which are
+fairly well documented. Each function goes through the following steps:
+- Machine initialization
+  - Give it a [name](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L70)
+  - Specify [which hop](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L73) the padding should go to
+  - Specify whether it should be [client-side](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L75) or relay-side.
+- Specify for [which types of circuits](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L78) the machine should apply
+- Specify whether the circuits should be [kept alive](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding_machines.c#L112) until the machine finishes padding.
+- Sets [padding limits](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L116) to avoid too much overhead in case of bugs or errors.
+- Setup [machine states](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L120)
+   - Specify [state transitions](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L123).
+- Finally [register the machine](https://github.com/torproject/tor/blob/35e978da61efa04af9a5ab2399dff863bc6fb20a/src/core/or/circuitpadding_machines.c#L137) to the global machine list
+
+### 5.2. Adaptive Padding Early
+
+[Adaptive Padding Early](https://www.cs.kau.se/pulls/hot/thebasketcase-ape/)
+is a variant of Adaptive Padding/WTF-PAD that does not use histograms or token
+removal to shift padding distributions, but instead uses fixed parameterized
+distributions to specify inter-packet timing thresholds for burst and gap
+inter-packet delays.
+
+Tobias Pulls's [QuickStart Guide](CircuitPaddingQuickStart.md) describes how
+to get this machine up and running, and has links to branches with a working
+implementation.
+
+### 5.3. Sketch of Tamaraw
+
+The [Tamaraw defense
+paper](https://www.cypherpunks.ca/~iang/pubs/webfingerprint-ccs14.pdf) is the
+only defense to date that provides a proof of optimality for the finite-length
+website traffic fingerprinting domain. These bounds assume that a defense is
+able to perform a full, arbitrary transform of a trace that is under a fixed
+number of packets in length.
+
+The key insight to understand Tamaraw's optimality is that it achieves one
+such optimal transform by delaying traffic below a circuit's throughput. By
+doing this, it creates a queue that is rarely empty, allowing it to produce
+a provably optimal transform with minimal overhead. As [Section
+1.4](#14-other-deployment-constraints) explains, this queue
+cannot be maintained on the live Tor network without risk of out-of-memory
+conditions at relays.
+
+However, if the queue is not maintained in the Tor network, but instead by the
+application layer, it could be deployed by websites that opt in to using it.
+
+In this case, the application layer component would do *optional* constant
+rate shaping, negotiated between a web browser and a website. The Circuit
+Padding Framework can then easily fill in any missing gaps of cover traffic
+packets, and also ensure that only a fixed length number of packets are sent
+in total.
+
+However, for such a defense to be safe, additional care must be taken to
+ensure that the resulting traffic pattern still has a large
+anonymity/confusion set with other traces on the live network.
+
+Accomplishing this is an unsolved problem.
+
+### 5.4. Other Padding Machines
+
+Our partners in this project at RIT have produced a couple prototypes, based
+on their published research designs
+[REB and RBB](https://www.researchgate.net/publication/329743510_UNDERSTANDING_FEATURE_DISCOVERY_IN_WEBSITE_FINGERPRINTING_ATTACKS).
+
+As [their writeup
+explains](https://github.com/notem/tor-rbp-padding-machine-doc), because RBB
+uses delay, the circuit padding machine that they made is a no-delay version.
+
+They also ran into an issue with the 0-delay timing workaround for [bug
+31653](https://bugs.torproject.org/31653). Keep an eye on that bug for updates
+with improved workarounds/fixes.
+
+Their code is [available on github](https://github.com/notem/tor/tree/circuit_padding_rbp_machine).
+
+
+## 6. Framework Implementation Details
+
+If you need to add additional events, conditions, or other features to the
+circuit padding framework, then this section is for you.
+
+### 6.1. Memory Allocation Conventions
+
+If the existing circuit padding features are sufficient for your needs, then
+you do not need to worry about memory management or pointer lifespans. The
+circuit padding framework should take care of this for you automatically.
+
+However, if you need to add new padding machine features to support your
+padding machines, then it will be helpful to understand how circuits
+correspond to the global machine definitions, and how mutable padding machine
+memory is managed.
+
+#### 6.1.1. Circuits and Padding Machines
+
+In Tor, the
+[circuit_t structure](https://github.com/torproject/tor/blob/master/src/core/or/circuit_st.h)
+is the superclass structure for circuit-related state that is used on both
+clients and relays. On clients, the actual datatype of the object pointed to
+by `circuit_t *` is the subclass structure
+[origin_circuit_t](https://github.com/torproject/tor/blob/master/src/core/or/origin_circuit_st.h). The
+macros `CIRCUIT_IS_ORIGIN()` and `TO_ORIGIN_CIRCUIT()` are used to determine
+if a circuit is a client-side (origin) circuit and to cast the pointer safely
+to `origin_circuit_t *`.
+
+Because circuit padding machines can be present at both clients and relays,
+the circuit padding fields are stored in the `circuit_t *` superclass
+structure. Notice that there are actually two sets of circuit padding fields:
+a `const circpad_machine_spec_t *` array, and a `circpad_machine_runtime_t *`
+array. Each of these arrays holds at most two elements, as there can be at
+most two padding machines on each circuit.
+
+The `const circpad_machine_spec_t *` points to a globally allocated machine
+specification. These machine specifications are
+allocated and set up during Tor program startup, in `circpad_machines_init()`
+in
+[circuitpadding.c](https://github.com/torproject/tor/blob/master/src/core/or/circuitpadding.c). Because
+the machine specification object is shared by all circuits, it must not be
+modified or freed until program exit (by `circpad_machines_free()`). The
+`const` qualifier should enforce this at compile time.
+
+The `circpad_machine_runtime_t *` array member points to the mutable runtime
+information for machine specification at that same array index. This runtime
+structure keeps track of the current machine state, packet counts, and other
+information that must be updated as the machine operates. When a padding
+machine is successfully negotiated `circpad_setup_machine_on_circ()` allocates
+the associated runtime information.
+
+#### 6.1.2. Histogram Management
+
+If a `circpad_state_t` of a machine specifies a `token_removal` strategy
+other than `CIRCPAD_TOKEN_REMOVAL_NONE`, then every time
+there is a state transition into this state, `circpad_machine_setup_tokens()`
+will copy the read-only `circpad_state_t.histogram` array into a mutable
+version at `circpad_machine_runtime_t.histogram`. This mutable copy is used
+to decrement the histogram bin accounts as packets are sent, as per the
+specified token removal strategy.
+
+When the state machine transitions out of this state, the mutable histogram copy is freed
+by this same `circpad_machine_setup_tokens()` function.
+
+#### 6.1.3. Deallocation and Shutdown
+
+As an optimization, padding machines can be swapped in and out by the client
+without waiting a full round trip for the relay machine to shut down.
+
+Internally, this is accomplished by immediately freeing the heap-allocated
+`circuit_t.padding_info` field corresponding to that machine, but still preserving the
+`circuit_t.padding_machine` pointer to the global padding machine
+specification until the response comes back from the relay. Once the response
+comes back, that `circuit_t.padding_machine` pointer is set to NULL, if the
+response machine number matches the current machine present.
+
+Because of this partial shutdown condition, we have two macros for iterating
+over machines. `FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN()` is used to iterate
+over machines that have both a `circuit_t.padding_info` slot and a
+`circuit_t.padding_machine` slot occupied. `FOR_EACH_CIRCUIT_MACHINE_BEGIN()`
+is used when we need to iterate over all machines that are either active or
+are simply waiting for a response to a shutdown request.
+
+If the machine is replaced instead of just shut down, then the client frees
+the `circuit_t.padding_info`, and then sets the `circuit_t.padding_machine`
+and `circuit_t.padding_info` fields for this next machine immediately. This is
+done in `circpad_add_matching_machines()`. In this case, since the new machine
+should have a different machine number, the shut down response from the relay
+is silently discarded, since it will not match the new machine number.
+
+If this sequence of machine teardown and spin-up happens rapidly enough for
+the same machine number (as opposed to different machines), then a race
+condition can happen. This is
+[known bug #30992](https://bugs.torproject.org/30992).
+
+When the relay side decides to shut down a machine, it sends a
+`RELAY_COMMAND_PADDING_NEGOTIATED` towards the client. If this cell matches the
+current machine number on the client, that machine is torn down, by freeing
+the `circuit_t.padding_info` slot and immediately setting
+`circuit_t.padding_machine` slot to NULL.
+
+Additionally, if Tor decides to close a circuit forcibly due to error before
+the padding machine is shut down, then `circuit_t.padding_info` is still
+properly freed by the call to `circpad_circuit_free_all_machineinfos()`
+in `circuit_free_()`.
+
+### 6.2. Machine Application Events
+
+The framework checks client-side origin circuits to see if padding machines
+should be activated or terminated during specific event callbacks in
+`circuitpadding.c`. We list these event callbacks here only for reference. You
+should not modify any of these callbacks to get your machine to run; instead,
+you should use the `circpad_machine_spec_t.conditions` field.
+
+However, you may add new event callbacks if you need other activation events,
+for example to provide obfuscation-layer or application-layer signaling. Any
+new event callbacks should behave exactly like the existing callbacks.
+
+During each of these event callbacks, the framework checks to see if any
+current running padding machines have conditions that no longer apply as a
+result of the event, and shuts those machines down. Then, it checks to see if
+any new padding machines should be activated as a result of the event, based
+on their circuit application conditions. **Remember: Machines are checked in
+reverse order in the machine list. This means that later, more recently added
+machines take precedence over older, earlier entries in each list.**
+
+Both of these checks are performed using the machine application conditions
+that you specify in your machine's `circpad_machine_spec_t.conditions` field.
+
+The machine application event callbacks are prefixed by `circpad_machine_event_` by convention in circuitpadding.c. As of this writing, these callbacks are:
+
+  - `circpad_machine_event_circ_added_hop()`: Called whenever a new hop is
+    added to a circuit.
+  - `circpad_machine_event_circ_built()`: Called when a circuit has completed
+    construction and is
+    opened. <!-- open != ready for traffic. Which do we mean? -nickm -->
+  - `circpad_machine_event_circ_purpose_changed()`: Called when a circuit
+    changes purpose.
+  - `circpad_machine_event_circ_has_no_relay_early()`: Called when a circuit
+    runs out of `RELAY_EARLY` cells.
+  - `circpad_machine_event_circ_has_streams()`: Called when a circuit gets a
+    stream attached.
+  - `circpad_machine_event_circ_has_no_streams()`: Called when the last
+    stream is detached from a circuit.
+
+### 6.3. Internal Machine Events
+
+To provide for some additional capabilities beyond simple finite state machine
+behavior, the circuit padding machines also have internal events that they
+emit to themselves when packet count length limits are hit, when the Infinity
+bin is sampled, and when the histogram bins are emptied of all tokens.
+
+These events are implemented as `circpad_internal_event_*` functions in
+`circuitpadding.c`, which are called from various areas that determine when
+the events should occur.
+
+While the conditions that trigger these internal events to be called may be
+complex, they are processed by the state machine definitions in a nearly
+identical manner as the cell processing events, with the exception that they
+are sent to the current machine only, rather than all machines on the circuit.
+
+
+## 7. Future Features and Optimizations
+
+While implementing the circuit padding framework, our goal was to deploy a
+system that obscured client-side onion service circuit setup and supported
+deployment of WTF-PAD and/or APE. Along the way, we noticed several features
+that might prove useful to others, but were not essential to implement
+immediately. We do not have immediate plans to implement these ideas, but we
+would gladly accept patches that do so.
+
+The following list gives an overview of these improvements, but as this
+document ages, it may become stale. The canonical list of improvements that
+researchers may find useful is tagged in our bugtracker with
+[circpad-researchers](https://trac.torproject.org/projects/tor/query?keywords=~circpad-researchers),
+and the list of improvements that are known to be necessary for some research
+areas are tagged with
+[circpad-researchers-want](https://trac.torproject.org/projects/tor/query?keywords=~circpad-researchers-want).
+
+Please consult those lists for the latest status of these issues. Note that
+not all fixes will be backported to all Tor versions, so be mindful of which
+Tor releases receive which fixes as you conduct your experiments.
+
+### 7.1. Load Balancing and Flow Control
+
+Fortunately, non-Exit bandwidth is already plentiful and exceeds the Exit
+capacity, and we anticipate that if we inform our relay operator community of
+the need for non-Exit bandwidth to satisfy padding overhead requirements,
+they will be able to provide that with relative ease.
+
+Unfortunately, padding machines that have large quantities of overhead will
+require changes to our load balancing system to account for this
+overhead. The necessary changes are documented in
+[Proposal 265](https://gitweb.torproject.org/torspec.git/tree/proposals/265-load-balancing-with-overhead.txt).
+
+Additionally, padding cells are not currently subjected to flow control. For
+high amounts of padding, we may want to change this. See [ticket
+31782](https://bugs.torproject.org/31782) for details.
+
+### 7.2. Timing and Queuing Optimizations
+
+The circuitpadding framework has some timing related issues that may impact
+results. If high-resolution timestamps are fed to opaque deep learning
+trainers, those training models may end up able to differentiate padding
+traffic from non-padding traffic due to these timing bugs.
+
+The circuit padding cell event callbacks come from post-decryption points in
+the cell processing codepath, and from the pre-queue points in the cell send
+codepath. This means that our cell events will not reflect the actual time
+when packets are read or sent on the wire. This is much worse in the send
+direction, as the circuitmux queue, channel outbuf, and kernel TCP buffer will
+impose significant additional delay between when we currently report that a
+packet was sent, and when it actually hits the wire.
+
+[Ticket 29494](https://bugs.torproject.org/29494) has a more detailed
+description of this problem, and an experimental branch that changes the cell
+event callback locations to be from circuitmux post-queue, which with KIST,
+should be an accurate reflection of when they are actually sent on the wire.
+
+If your padding machine and problem space depends on very accurate notions of
+relay-side packet timing, please try that branch and let us know on the
+ticket if you need any further assistance fixing it up.
+
+Additionally, with these changes, it will be possible to provide further
+overhead reducing optimizations by letting machines specify flags to indicate
+that padding should not be sent if there are any cells pending in the cell
+queue, for doing things like extending cell bursts more accurately and with
+less overhead.
+
+However, even if we solve the queuing issues, Tor's current timers are not as
+precise as some padding designs may require. We will still have issues of
+timing precision to solve. [Ticket 31653](https://bugs.torproject.org/31653)
+describes an issue the circuit padding system has with sending 0-delay padding
+cells, and [ticket 32670](https://bugs.torproject.org/32670) describes a
+libevent timer accuracy issue, which causes callbacks to vary up to 10ms from
+their scheduled time, even in absence of load.
+
+All of these issues strongly suggest that you either truncate the resolution
+of any timers you feed to your classifier, or that you omit timestamps
+entirely from the classification problem until these issues are addressed.
+
+### 7.3. Better Machine Negotiation
+
+Circuit padding is applied to circuits through machine conditions.
+
+The following machine conditions may be useful for some use cases, but have
+not been implemented yet:
+  * [Exit Policy-based Stream Conditions](https://bugs.torproject.org/29083)
+  * [Probability to apply machine/Cointoss condition](https://bugs.torproject.org/30092)
+  * [Probability distributions for launching new padding circuit(s)](https://bugs.torproject.org/31783)
+  * [More flexible purpose handling](https://bugs.torproject.org/32040)
+
+Additionally, the following features may help to obscure that padding is being
+negotiated, and/or streamline that negotiation process:
+  * [Always send negotiation cell on all circuits](https://bugs.torproject.org/30172)
+  * [Better shutdown handling](https://bugs.torproject.org/30992)
+  * [Preference-ordered negotiation menu](https://bugs.torproject.org/30348)
+
+### 7.4. Probabilistic State Transitions
+
+Right now, the state machine transitions are fully deterministic. However,
+one could imagine a state machine that uses probabilistic transitions between
+states to simulate a random walk or Hidden Markov Model traversal across
+several pages.
+
+The simplest way to implement this is to make  the `circpad_state_t.next_state` array
+into an array of structs that have a next state field, and a probability to
+transition to that state.
+
+If you need this feature, please see [ticket
+31787](https://bugs.torproject.org/31787) for more details.
+
+### 7.5. More Complex Pattern Recognition
+
+State machines are extremely efficient sequence recognition devices. But they
+are not great pattern recognition devices. This is one of the reasons why
+[Adaptive Padding](https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf)
+used state machines in combination with histograms, to model the target
+distribution of interpacket delays for transmitted packets.
+
+However, there currently is no such optimization for reaction to patterns of
+*received* traffic. There may also be cases where defenses must react to more
+complex patterns of sent traffic than can be expressed by our current
+histogram and length count events.
+
+For example: if you wish your machine to react to a certain count of incoming
+cells in a row, right now you have to have a state for each cell, and use the
+infinity bin to timeout of the sequence in each state. We could make this more
+compact if each state had an arrival cell counter and inter-cell timeout. Or
+we could make more complex mechanisms to recognize certain patterns of arrival
+traffic in a state.
+
+The best way to build recognition primitives like this into the framework is
+to add additional [Internal Machine Events](#63-internal-machine-events) for
+the pattern in question.
+
+As another simple example, a useful additional event might be to transition
+whenever any of your histogram bins are empty, rather than all of them. To do
+this, you would add `CIRCPAD_EVENT_ANY_BIN_EMPTY` to the enum
+`circpad_event_t` in `circuitpadding.h`. You would then create a function
+`circuitpadding_internal_event_any_bin_empty()`, which would work just like
+`circuitpadding_internal_event_bin_empty()`, and also be called from
+`check_machine_token_supply()` in `circuitpadding.c` but with the check for
+each bin being zero instead of the total. With this change, new machines could
+react to this new event in the same way as any other.
+
+If you have any other ideas that may be useful, please comment on [ticket
+32680](https://bugs.torproject.org/32680).
+
+
+## 8. Open Research Problems
+
+### 8.1. Onion Service Circuit Setup
+
+Our circuit setup padding does not address timing-based features, only
+packet counts. Deep learning can probably see this.
+
+However, before going too deep down the timing rabithole, we may need to make
+[some improvements to Tor](#72-timing-and-queuing-optimizations). Please
+comment on those tickets if you need this.
+
+### 8.2. Onion Service Fingerprinting
+
+We have done nothing to obscure the service side of onion service circuit
+setup. Because service-side onion services will have the reverse traffic byte
+counts as normal clients, they will likely need some kind of [hybrid
+application layer traffic shaping](#53-sketch-of-tamaraw), in addition to
+simple circuit setup obfuscation.
+
+Fingerprinting in
+[combination](https://github.com/mikeperry-tor/vanguards/blob/master/README_SECURITY.md)
+with
+[vanguards](https://github.com/mikeperry-tor/vanguards/blob/master/README_TECHNICAL.md)
+ia also an open issue.
+
+### 8.3. Open World Fingerprinting
+
+Similarly, Open World/clearweb website fingerprinting defenses remain
+an unsolved problem from the practicality point of view. The original WTF-PAD
+defense was never tuned, and it is showing accuracy issues against deep
+learning attacks.
+
+### 8.4. Protocol Usage Fingerprinting
+
+Traffic Fingerprinting to determine the protocol in use by a client has not
+been studied, either from the attack or defense point of view.
+
+### 8.5. Datagram Transport Side Channels
+
+Padding can reduce the accuracy of dropped-cell side channels in such
+transports, but we don't know [how to measure
+this](https://lists.torproject.org/pipermail/tor-dev/2018-November/013562.html).
+
+## 9. Must Read Papers
+
+These are by far the most important papers in the space, to date:
+
+ - [Tamaraw](https://www.cypherpunks.ca/~iang/pubs/webfingerprint-ccs14.pdf)
+ - [Bayes, Not Naive](https://www.petsymposium.org/2017/papers/issue4/paper50-2017-4-source.pdf)
+ - [Anonymity Trilemma](https://eprint.iacr.org/2017/954.pdf)
+ - [WTF-PAD](http://arxiv.org/pdf/1512.00524)
+
+Except for WTF-PAD, these papers were selected because they point towards
+optimality bounds that can be benchmarked against.
+
+We cite them even though we are skeptical that provably optimal defenses can
+be constructed, at least not without trivial or impractical transforms (such as
+those that can be created with unbounded queue capacity, or stored knowledge
+of traces for every possible HTTP trace on the Internet).
+
+We also are not demanding an optimality or security proof for every defense.
+
+Instead, we cite the above as benchmarks. We believe the space, especially the
+open-world case, to be more akin to an optimization problem, where a
+WTF-PAD-like defense must be tuned through an optimizer to produce results
+comparable to provably optimal but practically unrealizable defenses, through
+rigorous adversarial evaluation.
+
+## A. Acknowledgments
+
+This research was supported in part by NSF grants CNS-1619454 and CNS-1526306.
diff --git a/doc/HACKING/CircuitPaddingQuickStart.md b/doc/HACKING/CircuitPaddingQuickStart.md
new file mode 100644
index 0000000000..2780b5c6ea
--- /dev/null
+++ b/doc/HACKING/CircuitPaddingQuickStart.md
@@ -0,0 +1,266 @@
+# A Padding Machine from Scratch
+
+A quickstart guide by Tobias Pulls.
+
+This document describes the process of building a "padding machine" in tor's new
+circuit padding framework from scratch. Notes were taken as part of porting
+[Adaptive Padding Early
+(APE)](https://www.cs.kau.se/pulls/hot/thebasketcase-ape/) from basket2 to the
+circuit padding framework. The goal is just to document the process and provide
+useful pointers along the way, not create a useful machine. 
+
+The quick and dirty plan is to:
+1. clone and compile tor
+2. use newly built tor in TB and at small (non-exit) relay we run
+3. add a bare-bones APE padding machine
+4. run the machine, inspect logs for activity
+5. port APE's state machine without thinking much about parameters
+
+## Clone and compile tor
+
+```bash
+git clone https://git.torproject.org/tor.git
+cd tor
+git checkout tor-0.4.1.5
+```
+Above we use the tag for tor-0.4.1.5 where the circuit padding framework was
+released. Note that this version of the framework is missing many features and
+fixes that have since been merged to origin/master. If you need the newest
+framework features, you should use that master instead.
+
+```bash
+sh autogen.sh 
+./configure
+make
+```
+When you run `./configure` you'll be told of missing dependencies and packages
+to install on debian-based distributions. Important: if you plan to run `tor` on
+a relay as part of the real Tor network and your server runs a distribution that
+uses systemd, then I'd recommend that you `apt install dpkg dpkg-dev
+libevent-dev libssl-dev asciidoc quilt dh-apparmor libseccomp-dev dh-systemd
+libsystemd-dev pkg-config dh-autoreconf libfakeroot zlib1g zlib1g-dev automake
+liblzma-dev libzstd-dev` and ensure that tor has systemd support enabled:
+`./configure --enable-systemd`. Without this, on a recent Ubuntu, my tor service
+was forcefully restarted (SIGINT interrupt) by systemd every five minutes.
+
+If you want to install on your localsystem, run `make install`. For our case we
+just want the tor binary at `src/app/tor`.
+
+## Use tor in TB and at a relay
+
+Download and install a fresh Tor Browser (TB) from torproject.org. Make sure it
+works. From the command line, relative to the folder created when you extracted
+TB, run `./Browser/start-tor-browser --verbose` to get some basic log output.
+Note the version of tor, in my case, `Tor 0.4.0.5 (git-bf071e34aa26e096)` as
+part of TB 8.5.4. Shut down TB, copy the `tor` binary that you compiled earlier
+and replace `Browser/TorBrowser/Tor/tor`. Start TB from the command line again,
+you should see a different version, in my case `Tor 0.4.1.5
+(git-439ca48989ece545)`.
+
+The relay we run is also on linux, and `tor` is located at `/usr/bin/tor`. To
+view relevant logs since last boot `sudo journalctl -b /usr/bin/tor`, where we
+find `Tor 0.4.0.5 running on Linux`. Copy the locally compiled `tor` to the
+relay at a temporary location and then make sure it's ownership and access
+rights are identical to `/usr/bin/tor`. Next, shut down the running tor service
+with `sudo service tor stop`, wait for it to stop (typically 30s), copy our
+locally compiled tor to replace `/usr/bin/tor` then start the service again.
+Checking the logs we see `or 0.4.1.5 (git-439ca48989ece545)`.
+
+Repeatedly shutting down a relay is detrimental to the network and should be
+avoided. Sorry about that.
+
+We have one more step left before we move on the machine: configure TB to always
+use our middle relay. Edit `Browser/TorBrowser/Data/Tor/torrc` and set
+`MiddleNodes <fingerprint>`, where `<fingerprint>` is the fingerprint of the
+relay. Start TB, visit a website, and manually confirm that the middle is used
+by looking at the circuit display. 
+
+## Add a bare-bones APE padding machine
+
+Now the fun part. We have several resources at our disposal (mind that links
+might be broken in the future, just search for the headings):
+- The official [Circuit Padding Developer
+  Documentation](https://storm.torproject.org/shared/ChieH_sLU93313A2gopZYT3x2waJ41hz5Hn2uG1Uuh7).
+- Notes we made on the [implementation of the circuit padding
+  framework](https://github.com/pylls/padding-machines-for-tor/blob/master/notes/circuit-padding-framework.md).
+- The implementation of the current circuit padding machines in tor:
+  [circuitpadding.c](https://gitweb.torproject.org/tor.git/tree/src/core/or/circuitpadding_machines.c)
+  and
+  [circuitpadding_machines.h](https://gitweb.torproject.org/tor.git/tree/src/core/or/circuitpadding_machines.h).
+
+Please consult the above links for details. Moving forward, the focus is to
+describe what was done, not necessarily explaining all the details why. 
+
+Since we plan to make changes to tor, create a new branch `git checkout -b
+circuit-padding-ape-machine tor-0.4.1.5`. 
+
+We start with declaring two functions, one for the machine at the client and one
+at the relay, in `circuitpadding_machines.h`:
+
+```c
+void circpad_machine_relay_wf_ape(smartlist_t *machines_sl);
+void circpad_machine_client_wf_ape(smartlist_t *machines_sl);
+```
+
+The definitions go into `circuitpadding_machines.c`:
+
+```c
+/**************** Adaptive Padding Early (APE) machine ****************/
+
+/** 
+ * Create a relay-side padding machine based on the APE design. 
+ */
+void
+circpad_machine_relay_wf_ape(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+  = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_wf_ape";
+  relay_machine->is_origin_side = 0; // relay-side
+
+  // Pad to/from the middle relay, only when the circuit has streams
+  relay_machine->target_hopnum = 2;
+  relay_machine->conditions.min_hops = 2;
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_STREAMS;
+
+  // limits to help guard against excessive padding
+  relay_machine->allowed_padding_count = 1;
+  relay_machine->max_padding_percent = 1;
+
+  // one state to start with: START (-> END, never takes a slot in states)
+  circpad_machine_states_init(relay_machine, 1);
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_END;
+
+  // register the machine
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+  
+  log_info(LD_CIRC,
+           "Registered relay WF APE padding machine (%u)",
+           relay_machine->machine_num);
+}
+
+/** 
+ * Create a client-side padding machine based on the APE design. 
+ */
+void
+circpad_machine_client_wf_ape(smartlist_t *machines_sl)
+{
+    circpad_machine_spec_t *client_machine
+  = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_wf_ape";
+  client_machine->is_origin_side = 1; // client-side
+
+  /** Pad to/from the middle relay, only when the circuit has streams, and only
+  * for general purpose circuits (typical for web browsing)
+  */
+  client_machine->target_hopnum = 2;
+  client_machine->conditions.min_hops = 2;
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_STREAMS;
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_GENERAL);
+
+  // limits to help guard against excessive padding
+  client_machine->allowed_padding_count = 1;
+  client_machine->max_padding_percent = 1;
+
+  // one state to start with: START (-> END, never takes a slot in states)
+  circpad_machine_states_init(client_machine, 1);
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_END;
+
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+  log_info(LD_CIRC,
+           "Registered client WF APE padding machine (%u)",
+           client_machine->machine_num);
+}
+```
+
+We also have to modify `circpad_machines_init()` in `circuitpadding.c` to
+register our machines:
+
+```c
+  /* Register machines for the APE WF defense */
+  circpad_machine_client_wf_ape(origin_padding_machines);
+  circpad_machine_relay_wf_ape(relay_padding_machines);
+```
+
+We run `make` to get a new `tor` binary and copy it to our local TB. 
+
+## Run the machine
+
+To be able
+to view circuit info events in the console as we launch TB, we add `Log
+[circ]info notice stdout` to `torrc` of TB. 
+
+Running TB to visit example.com we first find in the log:
+
+```
+Aug 30 18:36:43.000 [info] circpad_machine_client_hide_intro_circuits(): Registered client intro point hiding padding machine (0)
+Aug 30 18:36:43.000 [info] circpad_machine_relay_hide_intro_circuits(): Registered relay intro circuit hiding padding machine (0)
+Aug 30 18:36:43.000 [info] circpad_machine_client_hide_rend_circuits(): Registered client rendezvous circuit hiding padding machine (1)
+Aug 30 18:36:43.000 [info] circpad_machine_relay_hide_rend_circuits(): Registered relay rendezvous circuit hiding padding machine (1)
+Aug 30 18:36:43.000 [info] circpad_machine_client_wf_ape(): Registered client WF APE padding machine (2)
+Aug 30 18:36:43.000 [info] circpad_machine_relay_wf_ape(): Registered relay WF APE padding machine (2)
+```
+
+All good, our machine is running. Looking further we find:
+
+```
+Aug 30 18:36:55.000 [info] circpad_setup_machine_on_circ(): Registering machine client_wf_ape to origin circ 2 (5)
+Aug 30 18:36:55.000 [info] circpad_node_supports_padding(): Checking padding: supported
+Aug 30 18:36:55.000 [info] circpad_negotiate_padding(): Negotiating padding on circuit 2 (5), command 2
+Aug 30 18:36:55.000 [info] circpad_machine_spec_transition(): Circuit 2 circpad machine 0 transitioning from 0 to 65535
+Aug 30 18:36:55.000 [info] circpad_machine_spec_transitioned_to_end(): Padding machine in end state on circuit 2 (5)
+Aug 30 18:36:55.000 [info] circpad_circuit_machineinfo_free_idx(): Freeing padding info idx 0 on circuit 2 (5)
+Aug 30 18:36:55.000 [info] circpad_handle_padding_negotiated(): Middle node did not accept our padding request on circuit 2 (5)
+```
+We see that our middle support padding (since we upgraded to tor-0.4.1.5), that
+we attempt to negotiate, our machine starts on the client, transitions to the
+end state, and is freed. The last line shows that the middle doesn't have a
+padding machine that can run. 
+
+Next, we follow the same steps as earlier and replace the modified `tor` at our
+middle relay. We don't update the logging there to avoid logging on the info
+level on the live network. Looking at the client log again we see that
+negotiation works as before except for the last line: it's missing, so the
+machine is running at the middle as well.  
+
+## Implementing the APE state machine
+
+Porting is fairly straightforward: define the states for all machines, add two
+more machines (for the receive portion of WTFP-PAD, beyond AP), and pick
+reasonable parameters for the distributions (I completely winged it now, as when
+implementing APE). The [circuit-padding-ape-machine
+branch](https://github.com/pylls/tor/tree/circuit-padding-ape-machine) contains
+the commits for the full machines with plenty of comments. 
+
+Some comments on the process:
+
+- `tor-0.4.1.5` does not support two machines on the same circuit, the following
+  fix has to be made: https://trac.torproject.org/projects/tor/ticket/31111 .
+  The good news is that everything else seems to work after the small change in
+  the fix. 
+- APE randomizes its distributions. Currently, this can only be done during
+  start of `tor`. This makes sense in the censorship circumvention setting
+  (`obfs4`), less so for WF defenses: further randomizing each circuit is likely
+  a PITA for attackers with few downsides.
+- it was annoying to figure out that the lack of systemd support in my compiled
+  tor caused systemd to interrupt (SIGINT) my tor process at the middle relay
+  every five minutes. Updated build steps above to hopefully save others the
+  pain.
+- there's for sure some bug on relays when sending padding cells too early (?).
+  It can happen with some probability with the APE implementation due to
+  `circpad_machine_relay_wf_ape_send()`. Will investigate next.
+- Moving the registration of machines from the definition of the machines to
+  `circpad_machines_init()` makes sense, as suggested in the circuit padding doc
+  draft.
+
+Remember that APE is just a proof-of-concept and we make zero claims about its
+ability to withstand WF attacks, in particular those based on deep learning.
diff --git a/doc/HACKING/CodeStructure.md b/doc/HACKING/CodeStructure.md
index 736d6cd484..d387018f9b 100644
--- a/doc/HACKING/CodeStructure.md
+++ b/doc/HACKING/CodeStructure.md
@@ -1,129 +1,123 @@
+# Code Structure
 
 TODO: revise this to talk about how things are, rather than how things
 have changed.
 
-TODO: Make this into good markdown.
-
-
-
-For quite a while now, the program "tor" has been built from source
-code in just two directories: src/common and src/or.
+For quite a while now, the program *tor* has been built from source
+code in just two directories: **src/common** and **src/or**.
 
 This has become more-or-less untenable, for a few reasons -- most
 notably of which is that it has led our code to become more
 spaghetti-ish than I can endorse with a clean conscience.
 
 So to fix that, we've gone and done a huge code movement in our git
-master branch, which will land in a release once Tor 0.3.5.1-alpha is
+master branch, which will land in a release once Tor `0.3.5.1-alpha` is
 out.
 
 Here's what we did:
 
-  * src/common has been turned into a set of static libraries.  These
-all live in the "src/lib/*" directories.  The dependencies between
+  * **src/common** has been turned into a set of static libraries.  These
+all live in the **src/lib/*** directories.  The dependencies between
 these libraries should have no cycles.  The libraries are:
 
-    arch -- Headers to handle architectural differences
-    cc -- headers to handle differences among compilers
-    compress -- wraps zlib, zstd, lzma
-    container -- high-level container types
-    crypt_ops -- Cryptographic operations. Planning to split this into
+    - **arch** -- Headers to handle architectural differences
+    - **cc** -- headers to handle differences among compilers
+    - **compress** -- wraps zlib, zstd, lzma
+    - **container** -- high-level container types
+    - **crypt_ops** -- Cryptographic operations. Planning to split this into
 a higher and lower level library
-    ctime -- Operations that need to run in constant-time. (Properly,
+    - **ctime** -- Operations that need to run in constant-time. (Properly,
 data-invariant time)
-    defs -- miscelaneous definitions needed throughout Tor.
-    encoding -- transforming one data type into another, and various
+    - **defs** -- miscelaneous definitions needed throughout Tor.
+    - **encoding** -- transforming one data type into another, and various
 data types into strings.
-    err -- lowest-level error handling, in cases where we can't use
+    - **err** -- lowest-level error handling, in cases where we can't use
 the logs because something that the logging system needs has broken.
-    evloop -- Generic event-loop handling logic
-    fdio -- Low-level IO wrapper functions for file descriptors.
-    fs -- Operations on the filesystem
-    intmath -- low-level integer math and misc bit-twiddling hacks
-    lock -- low-level locking code
-    log -- Tor's logging module.  This library sits roughly halfway up
+    - **evloop** -- Generic event-loop handling logic
+    - **fdio** -- Low-level IO wrapper functions for file descriptors.
+    - **fs** -- Operations on the filesystem
+    - **intmath** -- low-level integer math and misc bit-twiddling hacks
+    - **lock** -- low-level locking code
+    - **log** -- Tor's logging module.  This library sits roughly halfway up
 the library dependency diagram, since everything it depends on has to
 be carefully crafted to *not* log.
-    malloc -- Low-level wrappers for the platform memory allocation functions.
-    math -- Higher-level mathematical functions, and floating-point math
-    memarea -- An arena allocator
-    meminfo -- Functions for querying the current process's memory
+    - **malloc** -- Low-level wrappers for the platform memory allocation functions.
+    - **math** -- Higher-level mathematical functions, and floating-point math
+    - **memarea** -- An arena allocator
+    - **meminfo** -- Functions for querying the current process's memory
 status and resources
-    net -- Networking compatibility and convenience code
-    osinfo -- Querying information about the operating system
-    process -- Launching and querying the status of other processes
-    sandbox -- Backend for the linux seccomp2 sandbox
-    smartlist_core -- The lowest-level of the smartlist_t data type.
+    - **net** -- Networking compatibility and convenience code
+    - **osinfo** -- Querying information about the operating system
+    - **process** -- Launching and querying the status of other processes
+    - **sandbox** -- Backend for the linux seccomp2 sandbox
+    - **smartlist_core** -- The lowest-level of the smartlist_t data type.
 Separated from the rest of the containers library because the logging
 subsystem depends on it.
-    string -- Compatibility and convenience functions for manipulating
+    - **string** -- Compatibility and convenience functions for manipulating
 C strings.
-    term -- Terminal-related functions (currently limited to a getpass
+    - **term** -- Terminal-related functions (currently limited to a getpass
 function).
-    testsupport -- Macros for mocking, unit tests, etc.
-    thread -- Higher-level thread compatibility code
-    time -- Higher-level time management code, including format
+    - **testsupport** -- Macros for mocking, unit tests, etc.
+    - **thread** -- Higher-level thread compatibility code
+    - **time** -- Higher-level time management code, including format
 conversions and monotonic time
-    tls -- Our wrapper around our TLS library
-    trace -- Formerly src/trace -- a generic event tracing API
-    wallclock -- Low-level time code, used by the log module.
+    - **tls** -- Our wrapper around our TLS library
+    - **trace** -- Formerly src/trace -- a generic event tracing API
+    - **wallclock** -- Low-level time code, used by the log module.
 
-  * To ensure that the dependency graph in src/common remains under
-control, there is a tool that you can run called "make
-check-includes".  It verifies that each module in Tor only includes
+  * To ensure that the dependency graph in **src/common** remains under
+control, there is a tool that you can run called `make
+check-includes`.  It verifies that each module in Tor only includes
 the headers that it is permitted to include, using a per-directory
-".may_include" file.
+*.may_include* file.
 
-  * The src/or/or.h header has been split into numerous smaller
+  * The **src/or/or.h** header has been split into numerous smaller
 headers.  Notably, many important structures are now declared in a
-header called foo_st.h, where "foo" is the name of the structure.
+header called *foo_st.h*, where "foo" is the name of the structure.
 
-  * The src/or directory, which had most of Tor's code, had been split
+  * The **src/or** directory, which had most of Tor's code, had been split
 up into several directories.  This is still a work in progress:  This
 code has not itself been refactored, and its dependency graph is still
 a tangled web.  I hope we'll be working on that over the coming
 releases, but it will take a while to do.
 
-    The new top-level source directories are:
-
-     src/core -- Code necessary to actually perform or use onion routing.
-     src/feature -- Code used only by some onion routing
+    - The new top-level source directories are:
+        - **src/core** -- Code necessary to actually perform or use onion routing.
+        - **src/feature** -- Code used only by some onion routing
 configurations, or only for a special purpose.
-     src/app -- Top-level code to run, invoke, and configure the
+        - **src/app** -- Top-level code to run, invoke, and configure the
 lower-level code
 
-   The new second-level source directories are:
-     src/core/crypto -- High-level cryptographic protocols used in Tor
-     src/core/mainloop -- Tor's event loop, connection-handling, and
+    - The new second-level source directories are:
+        - **src/core/crypto** -- High-level cryptographic protocols used in Tor
+        - **src/core/mainloop** -- Tor's event loop, connection-handling, and
 traffic-routing code.
-     src/core/or -- Parts related to handling onion routing itself
-     src/core/proto -- support for encoding and decoding different
+        - **src/core/or** -- Parts related to handling onion routing itself
+        - **src/core/proto** -- support for encoding and decoding different
 wire protocols
-
-     src/feature/api -- Support for making Tor embeddable
-     src/feature/client -- Functionality which only Tor clients need
-     src/feature/control -- Controller implementation
-     src/feature/dirauth -- Directory authority
-     src/feature/dircache -- Directory cache
-     src/feature/dirclient -- Directory client
-     src/feature/dircommon -- Shared code between the other directory modules
-     src/feature/hibernate -- Hibernating when Tor is out of bandwidth
+        - **src/feature/api** -- Support for making Tor embeddable
+        - **src/feature/client** -- Functionality which only Tor clients need
+        - **src/feature/control** -- Controller implementation
+        - **src/feature/dirauth** -- Directory authority
+        - **src/feature/dircache** -- Directory cache
+        - **src/feature/dirclient** -- Directory client
+        - **src/feature/dircommon** -- Shared code between the other directory modules
+        - **src/feature/hibernate** -- Hibernating when Tor is out of bandwidth
 or shutting down
-     src/feature/hs -- v3 onion service implementation
-     src/feature/hs_common -- shared code between both onion service
+        - **src/feature/hs** -- v3 onion service implementation
+        - **src/feature/hs_common** -- shared code between both onion service
 implementations
-     src/feature/nodelist -- storing and accessing the list of relays on
+        - **src/feature/nodelist** -- storing and accessing the list of relays on
 the network.
-     src/feature/relay -- code that only relay servers and exit servers need.
-     src/feature/rend -- v2 onion service implementation
-     src/feature/stats -- statistics and history
-
-     src/app/config -- configuration and state for Tor
-     src/app/main -- Top-level functions to invoke the rest or Tor.
+        - **src/feature/relay** -- code that only relay servers and exit servers need.
+        - **src/feature/rend** -- v2 onion service implementation
+        - **src/feature/stats** -- statistics and history
+        - **src/app/config** -- configuration and state for Tor
+        - **src/app/main** -- Top-level functions to invoke the rest or Tor.
 
-  * The "tor" executable is now built in src/app/tor rather than src/or/tor.
+  * The `tor` executable is now built in **src/app/tor** rather than **src/or/tor**.
 
   * There are more static libraries than before that you need to build
 into your application if you want to embed Tor.  Rather than
-maintaining this list yourself, I recommend that you run "make
-show-libs" to have Tor emit a list of what you need to link.
+maintaining this list yourself, I recommend that you run `make
+show-libs` to have Tor emit a list of what you need to link.
diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md
index 4f229348e4..150acf1852 100644
--- a/doc/HACKING/CodingStandards.md
+++ b/doc/HACKING/CodingStandards.md
@@ -1,5 +1,4 @@
-Coding conventions for Tor
-==========================
+# Coding conventions for Tor
 
 tl;dr:
 
@@ -10,8 +9,7 @@ tl;dr:
    - Run `make distcheck` if you have made changes to build system components
    - Add a file in `changes` for your branch.
 
-Patch checklist
----------------
+## Patch checklist
 
 If possible, send your patch as one of these (in descending order of
 preference)
@@ -34,7 +32,7 @@ Did you remember...
 
 If you are submitting a major patch or new feature, or want to in the future...
 
-   - Set up Chutney and Stem, see HACKING/WritingTests.md
+   - Set up Chutney and Stem, see `doc/HACKING/WritingTests.md`
    - Run `make test-full` to test against all unit and integration tests.
 
 If you have changed build system components:
@@ -42,8 +40,7 @@ If you have changed build system components:
    - For example, if you have changed Makefiles, autoconf files, or anything
      else that affects the build system.
 
-License issues
-==============
+## License issues
 
 Tor is distributed under the license terms in the LICENSE -- in
 brief, the "3-clause BSD license".  If you send us code to
@@ -57,10 +54,7 @@ Some compatible licenses include:
   - 2-clause BSD
   - CC0 Public Domain Dedication
 
-
-
-How we use Git branches
-=======================
+## How we use Git branches
 
 Each main development series (like 0.2.1, 0.2.2, etc) has its main work
 applied to a single branch.  At most one series can be the development series
@@ -91,37 +85,71 @@ conflicts in the ChangeLog when it comes time to merge your branch into Tor.
 Best advice: don't try to keep an independent branch forked for more than 6
 months and expect it to merge cleanly. Try to merge pieces early and often.
 
-
-How we log changes
-==================
+## How we log changes
 
 When you do a commit that needs a ChangeLog entry, add a new file to
 the `changes` toplevel subdirectory.  It should have the format of a
 one-entry changelog section from the current ChangeLog file, as in
 
-- Major bugfixes:
+  o Major bugfixes (security):
     - Fix a potential buffer overflow. Fixes bug 99999; bugfix on
       0.3.1.4-beta.
+  o Minor features (performance):
+    - Make tor faster. Closes ticket 88888.
 
 To write a changes file, first categorize the change.  Some common categories
-are: Minor bugfixes, Major bugfixes, Minor features, Major features, Code
-simplifications and refactoring.  Then say what the change does.  If
-it's a bugfix, mention what bug it fixes and when the bug was
-introduced.  To find out which Git tag the change was introduced in,
-you can use `git describe --contains <sha1 of commit>`.
-
-If at all possible, try to create this file in the same commit where you are
-making the change.  Please give it a distinctive name that no other branch will
-use for the lifetime of your change. To verify the format of the changes file,
-you can use `make check-changes`.  This is run automatically as part of
-`make check` -- if it fails, we must fix it before we release.  These
-checks are implemented in `scripts/maint/lintChanges.py`.
+are:
+  o Minor bugfixes (subheading):
+  o Major bugfixes (subheading):
+  o Minor features (subheading):
+  o Major features (subheading):
+  o Code simplifications and refactoring:
+  o Testing:
+  o Documentation:
+
+The subheading is a particular area within Tor.  See the ChangeLog for
+examples.
+
+Then say what the change does.  If it's a bugfix, mention what bug it fixes
+and when the bug was introduced. To find out which Git tag the change was
+introduced in, you can use `git describe --contains <sha1 of commit>`.
+If you don't know the commit, you can search the git diffs (-S) for the first
+instance of the feature (--reverse).
+
+For example, for #30224, we wanted to know when the bridge-distribution-request
+feature was introduced into Tor:
+    $ git log -S bridge-distribution-request --reverse
+    commit ebab521525
+    Author: Roger Dingledine <arma@torproject.org>
+    Date:   Sun Nov 13 02:39:16 2016 -0500
+
+        Add new BridgeDistribution config option
+
+    $ git describe --contains ebab521525
+    tor-0.3.2.3-alpha~15^2~4
+
+If you need to know all the Tor versions that contain a commit, use:
+    $ git tag --contains 9f2efd02a1 | sort -V
+    tor-0.2.5.16
+    tor-0.2.8.17
+    tor-0.2.9.14
+    tor-0.2.9.15
+    ...
+    tor-0.3.0.13
+    tor-0.3.1.9
+    tor-0.3.1.10
+    ...
+
+If at all possible, try to create the changes file in the same commit where
+you are making the change.  Please give it a distinctive name that no other
+branch will use for the lifetime of your change. We usually use "ticketNNNNN"
+or "bugNNNNN", where NNNNN is the ticket number. To verify the format of the
+changes file, you can use `make check-changes`.  This is run automatically as
+part of `make check` -- if it fails, we must fix it as soon as possible, so
+that our CI passes.  These checks are implemented in
+`scripts/maint/lintChanges.py`.
 
 Changes file style guide:
-  * Changes files begin with "  o Header (subheading):".  The header
-    should usually be "Minor/Major bugfixes/features". The subheading is a
-    particular area within Tor.  See the ChangeLog for examples.
-
   * Make everything terse.
 
   * Write from the user's point of view: describe the user-visible changes
@@ -163,11 +191,11 @@ Why use changes files instead of entries in the ChangeLog?
    * Having every single commit touch the ChangeLog file tended to create
    zillions of merge conflicts.
 
-Whitespace and C conformance
-----------------------------
+## Whitespace and C conformance
 
-Invoke `make check-spaces` from time to time, so it can tell you about
-deviations from our C whitespace style.  Generally, we use:
+Tor's C code is written in accordance with the C99 standard. Invoke `make
+check-spaces` from time to time, so it can tell you about deviations from our C
+whitespace style.  Generally, we use:
 
    - Unix-style line endings
    - K&R-style indentation
@@ -183,6 +211,11 @@ deviations from our C whitespace style.  Generally, we use:
    - No space between a function name and an opening paren. `puts(x)`, not
      `puts (x)`.
    - Function declarations at the start of the line.
+   - Use `void foo(void)` to declare a function with no arguments.  Saying
+     `void foo()` is C++ syntax.
+   - Use `const` for new APIs.
+   - Variables should be initialized when declared, rather than declared at the
+     top of a scope.
 
 If you use an editor that has plugins for editorconfig.org, the file
 `.editorconfig` will help you to conform this coding style.
@@ -192,30 +225,55 @@ you're using gcc, you should invoke the configure script with the
 option `--enable-fatal-warnings`.  This will tell the compiler
 to make all warnings into errors.
 
-Functions to use; functions not to use
---------------------------------------
+## Functions to use; functions not to use
 
 We have some wrapper functions like `tor_malloc`, `tor_free`, `tor_strdup`, and
 `tor_gettimeofday;` use them instead of their generic equivalents.  (They
 always succeed or exit.)
 
+Specifically, Don't use `malloc`, `realloc`, `calloc`, `free`, or
+`strdup`. Use `tor_malloc`, `tor_realloc`, `tor_calloc`, `tor_free`, or
+`tor_strdup`.
+
+Don't use `tor_realloc(x, y\*z)`. Use `tor_reallocarray(x, y, z)` instead.;
+
 You can get a full list of the compatibility functions that Tor provides by
 looking through `src/lib/*/*.h`.  You can see the
 available containers in `src/lib/containers/*.h`.  You should probably
 familiarize yourself with these modules before you write too much code, or
 else you'll wind up reinventing the wheel.
 
-We don't use `strcat` or `strcpy` or `sprintf` of any of those notoriously broken
-old C functions.  Use `strlcat`, `strlcpy`, or `tor_snprintf/tor_asprintf` instead.
+We don't use `strcat` or `strcpy` or `sprintf` of any of those notoriously
+broken old C functions.  We also avoid `strncat` and `strncpy`. Use
+`strlcat`, `strlcpy`, or `tor_snprintf/tor_asprintf` instead.
 
 We don't call `memcmp()` directly.  Use `fast_memeq()`, `fast_memneq()`,
-`tor_memeq()`, or `tor_memneq()` for most purposes.
+`tor_memeq()`, or `tor_memneq()` for most purposes.  If you really need a
+tristate return value, use `tor_memcmp()` or `fast_memcmp()`.
+
+Don't call `assert()` directly. For hard asserts, use `tor_assert()`. For
+soft asserts, use `tor_assert_nonfatal()` or `BUG()`. If you need to print
+debug information in assert error message, consider using `tor_assertf()` and
+`tor_assertf_nonfatal()`.  If you are writing code that is too low-level to
+use the logging subsystem, use `raw_assert()`.
 
-Also see a longer list of functions to avoid in:
-https://people.torproject.org/~nickm/tor-auto/internal/this-not-that.html
+Don't use `toupper()` and `tolower()` functions. Use `TOR_TOUPPER` and
+`TOR_TOLOWER` macros instead. Similarly, use `TOR_ISALPHA`, `TOR_ISALNUM` et.
+al. instead of `isalpha()`, `isalnum()`, etc.
 
-What code can use what other code?
-----------------------------------
+When allocating new string to be added to a smartlist, use
+`smartlist_add_asprintf()` to do both at once.
+
+Avoid calling BSD socket functions directly. Use portable wrappers to work
+with sockets and socket addresses. Also, sockets should be of type
+`tor_socket_t`.
+
+Don't use any of these functions: they aren't portable. Use the
+version prefixed with `tor_` instead: strtok_r, memmem, memstr,
+asprintf, localtime_r, gmtime_r, inet_aton, inet_ntop, inet_pton,
+getpass, ntohll, htonll.  (This list is incomplete.)
+
+## What code can use what other code?
 
 We're trying to simplify Tor's structure over time.  In the long run, we want
 Tor to be structured as a set of modules with *no circular dependencies*.
@@ -232,8 +290,7 @@ included except those specifically permitted by the `.may_include` file.
 When editing one of these files, please make sure that you are not
 introducing any cycles into Tor's dependency graph.
 
-Floating point math is hard
----------------------------
+## Floating point math is hard
 
 Floating point arithmetic as typically implemented by computers is
 very counterintuitive.  Failure to adequately analyze floating point
@@ -279,7 +336,7 @@ General advice:
 
 For additional useful advice (and a little bit of background), see
 [What Every Programmer Should Know About Floating-Point
-Arithmetic](http://floating-point-gui.de/).
+Arithmetic](https://floating-point-gui.de/).
 
 A list of notable (and surprising) facts about floating point
 arithmetic is at [Floating-point
@@ -292,8 +349,7 @@ For more detailed (and math-intensive) background, see [What Every
 Computer Scientist Should Know About Floating-Point
 Arithmetic](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html).
 
-Other C conventions
--------------------
+## Other C conventions
 
 The `a ? b : c` trinary operator only goes inside other expressions;
 don't use it as a replacement for if. (You can ignore this inside macro
@@ -302,8 +358,15 @@ definitions when necessary.)
 Assignment operators shouldn't nest inside other expressions.  (You can
 ignore this inside macro definitions when necessary.)
 
-Functions not to write
-----------------------
+## Binary data and wire formats
+
+Use pointer to `char` when representing NUL-terminated string. To represent
+arbitrary binary data, use pointer to `uint8_t`. (Many older Tor APIs ignore
+this rule.)
+
+Refrain from attempting to encode integers by casting their pointers to byte
+arrays. Use something like `set_uint32()`/`get_uint32()` instead and don't
+forget about endianness.
 
 Try to never hand-write new code to parse or generate binary
 formats. Instead, use trunnel if at all possible.  See
@@ -314,9 +377,7 @@ for more information about trunnel.
 
 For information on adding new trunnel code to Tor, see src/trunnel/README
 
-
-Calling and naming conventions
-------------------------------
+## Calling and naming conventions
 
 Whenever possible, functions should return -1 on error and 0 on success.
 
@@ -339,17 +400,15 @@ probably time to create an enum.  If you find that you are passing three or
 more flags to a function, it's probably time to create a flags argument that
 takes a bitfield.
 
-What To Optimize
-----------------
+## What To Optimize
 
 Don't optimize anything if it's not in the critical path.  Right now, the
 critical path seems to be AES, logging, and the network itself.  Feel free to
 do your own profiling to determine otherwise.
 
-Log conventions
----------------
+## Log conventions
 
-`https://www.torproject.org/docs/faq#LogLevel`
+[FAQ - Log Levels](https://www.torproject.org/docs/faq#LogLevel)
 
 No error or warning messages should be expected during normal OR or OP
 operation.
@@ -363,8 +422,7 @@ end-users that they aren't expected to understand the message (perhaps
 with a string like "internal error"). Option (A) is to be preferred to
 option (B).
 
-Assertions In Tor
------------------
+## Assertions In Tor
 
 Assertions should be used for bug-detection only.  Don't use assertions to
 detect bad user inputs, network errors, resource exhaustion, or similar
@@ -383,8 +441,7 @@ macro, as in:
 	if (BUG(ptr == NULL))
 		return -1;
 
-Allocator conventions
----------------------
+## Allocator conventions
 
 By convention, any tor type with a name like `abc_t` should be allocated
 by a function named `abc_new()`.  This function should never return
@@ -422,9 +479,10 @@ to use it as a function callback), define it with a name like
       abc_free_(obj);
     }
 
+When deallocating, don't say e.g. `if (x) tor_free(x)`. The convention is to
+have deallocators do nothing when NULL pointer is passed.
 
-Doxygen comment conventions
----------------------------
+## Doxygen comment conventions
 
 Say what functions do as a series of one or more imperative sentences, as
 though you were telling somebody how to be the function.  In other words, DO
diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md
index fc562816db..36a0dcda2a 100644
--- a/doc/HACKING/CodingStandardsRust.md
+++ b/doc/HACKING/CodingStandardsRust.md
@@ -1,39 +1,36 @@
+# Rust Coding Standards
 
- Rust Coding Standards
-=======================
-
-You MUST follow the standards laid out in `.../doc/HACKING/CodingStandards.md`,
+You MUST follow the standards laid out in `doc/HACKING/CodingStandards.md`,
 where applicable.
 
- Module/Crate Declarations
----------------------------
+## Module/Crate Declarations
 
 Each Tor C module which is being rewritten MUST be in its own crate.
-See the structure of `.../src/rust` for examples.
+See the structure of `src/rust` for examples.
 
 In your crate, you MUST use `lib.rs` ONLY for pulling in external
 crates (e.g. `extern crate libc;`) and exporting public objects from
 other Rust modules (e.g. `pub use mymodule::foo;`).  For example, if
-you create a crate in `.../src/rust/yourcrate`, your Rust code should
-live in `.../src/rust/yourcrate/yourcode.rs` and the public interface
-to it should be exported in `.../src/rust/yourcrate/lib.rs`.
+you create a crate in `src/rust/yourcrate`, your Rust code should
+live in `src/rust/yourcrate/yourcode.rs` and the public interface
+to it should be exported in `src/rust/yourcrate/lib.rs`.
 
 If your code is to be called from Tor C code, you MUST define a safe
 `ffi.rs`.  See the "Safety" section further down for more details.
 
 For example, in a hypothetical `tor_addition` Rust module:
 
-In `.../src/rust/tor_addition/addition.rs`:
+In `src/rust/tor_addition/addition.rs`:
 
     pub fn get_sum(a: i32, b: i32) -> i32 {
         a + b
     }
 
-In `.../src/rust/tor_addition/lib.rs`:
+In `src/rust/tor_addition/lib.rs`:
 
     pub use addition::*;
 
-In `.../src/rust/tor_addition/ffi.rs`:
+In `src/rust/tor_addition/ffi.rs`:
 
     #[no_mangle]
     pub extern "C" fn tor_get_sum(a: c_int, b: c_int) -> c_int {
@@ -42,7 +39,7 @@ In `.../src/rust/tor_addition/ffi.rs`:
 
 If your Rust code must call out to parts of Tor's C code, you must
 declare the functions you are calling in the `external` crate, located
-at `.../src/rust/external`.
+at `src/rust/external`.
 
 <!-- XXX get better examples of how to declare these externs, when/how they -->
 <!-- XXX are unsafe, what they are expected to do —isis                     -->
@@ -54,8 +51,7 @@ If you have any external modules as dependencies (e.g. `extern crate
 libc;`), you MUST declare them in your crate's `lib.rs` and NOT in any
 other module.
 
- Dependencies and versions
----------------------------
+## Dependencies and versions
 
 In general, we use modules from only the Rust standard library
 whenever possible. We will review including external crates on a
@@ -81,8 +77,7 @@ Currently, Tor requires that you use the latest stable Rust version. At
 some point in the future, we will freeze on a given stable Rust version,
 to ensure backward compatibility with stable distributions that ship it.
 
- Updating/Adding Dependencies
-------------------------------
+## Updating/Adding Dependencies
 
 To add/remove/update dependencies, first add your dependencies,
 exactly specifying their versions, into the appropriate *crate-level*
@@ -101,14 +96,13 @@ Next, run `/scripts/maint/updateRustDependencies.sh`.  Then, go into
 `src/ext/rust` and commit the changes to the `tor-rust-dependencies`
 repo.
 
- Documentation
----------------
+## Documentation
 
 You MUST include `#![deny(missing_docs)]` in your crate.
 
 For function/method comments, you SHOULD include a one-sentence, "first person"
 description of function behaviour (see requirements for documentation as
-described in `.../src/HACKING/CodingStandards.md`), then an `# Inputs` section
+described in `src/HACKING/CodingStandards.md`), then an `# Inputs` section
 for inputs or initialisation values, a `# Returns` section for return
 values/types, a `# Warning` section containing warnings for unsafe behaviours or
 panics that could happen.  For publicly accessible
@@ -118,14 +112,12 @@ types/constants/objects/functions/methods, you SHOULD also include an
 You MUST document your module with _module docstring_ comments,
 i.e. `//!` at the beginning of each line.
 
- Style
--------
+## Style
 
 You SHOULD consider breaking up large literal numbers with `_` when it makes it
 more human readable to do so, e.g. `let x: u64 = 100_000_000_000`.
 
- Testing
----------
+## Testing
 
 All code MUST be unittested and integration tested.
 
@@ -134,7 +126,7 @@ describing how the function/object is expected to be used.
 
 Integration tests SHOULD go into a `tests/` directory inside your
 crate.  Unittests SHOULD go into their own module inside the module
-they are testing, e.g. in `.../src/rust/tor_addition/addition.rs` you
+they are testing, e.g. in `src/rust/tor_addition/addition.rs` you
 should put:
 
     #[cfg(test)]
@@ -148,8 +140,7 @@ should put:
         }
     }
 
- Benchmarking
---------------
+## Benchmarking
 
 The external `test` crate can be used for most benchmarking.  However, using
 this crate requires nightly Rust.  Since we may want to switch to a more
@@ -173,7 +164,7 @@ for basic benchmarks, is only used when running benchmarks via `cargo
 bench --features bench`.
 
 Finally, to write your benchmark code, in
-`.../src/rust/tor_addition/addition.rs` you SHOULD put:
+`src/rust/tor_addition/addition.rs` you SHOULD put:
 
     #[cfg(all(test, features = "bench"))]
     mod bench {
@@ -186,23 +177,20 @@ Finally, to write your benchmark code, in
         }
     }
 
- Fuzzing
----------
+## Fuzzing
 
 If you wish to fuzz parts of your code, please see the
-[`cargo fuzz`](https://github.com/rust-fuzz/cargo-fuzz) crate, which uses
+[cargo fuzz](https://github.com/rust-fuzz/cargo-fuzz) crate, which uses
 [libfuzzer-sys](https://github.com/rust-fuzz/libfuzzer-sys).
 
- Whitespace & Formatting
--------------------------
+## Whitespace & Formatting
 
 You MUST run `rustfmt` (https://github.com/rust-lang-nursery/rustfmt)
 on your code before your code will be merged.  You can install rustfmt
 by doing `cargo install rustfmt-nightly` and then run it with `cargo
 fmt`.
 
- Safety
---------
+## Safety
 
 You SHOULD read [the nomicon](https://doc.rust-lang.org/nomicon/) before writing
 Rust FFI code.  It is *highly advised* that you read and write normal Rust code
@@ -222,10 +210,10 @@ Here are some additional bits of advice and rules:
    > 
    > * Data races
    > * Dereferencing a null/dangling raw pointer
-   > * Reads of [undef](http://llvm.org/docs/LangRef.html#undefined-values)
+   > * Reads of [undef](https://llvm.org/docs/LangRef.html#undefined-values)
    >   (uninitialized) memory
    > * Breaking the
-   >   [pointer aliasing rules](http://llvm.org/docs/LangRef.html#pointer-aliasing-rules)
+   >   [pointer aliasing rules](https://llvm.org/docs/LangRef.html#pointer-aliasing-rules)
    >   with raw pointers (a subset of the rules used by C)
    > * `&mut T` and `&T` follow LLVM’s scoped noalias model, except if the `&T`
    >   contains an `UnsafeCell<U>`. Unsafe code must not violate these aliasing
@@ -256,7 +244,7 @@ Here are some additional bits of advice and rules:
    or 2) should fail (i.e. in a unittest).
 
    You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the
-   potential error with either `expect()` or the eel operator, `?`.
+   potential error with the eel operator, `?` or another non panicking way.
    For example, consider a function which parses a string into an integer:
 
         fn parse_port_number(config_string: &str) -> u16 {
@@ -264,12 +252,12 @@ Here are some additional bits of advice and rules:
         }
 
    There are numerous ways this can fail, and the `unwrap()` will cause the
-   whole program to byte the dust!  Instead, either you SHOULD use `expect()`
+   whole program to byte the dust!  Instead, either you SHOULD use `ok()`
    (or another equivalent function which will return an `Option` or a `Result`)
    and change the return type to be compatible:
 
         fn parse_port_number(config_string: &str) -> Option<u16> {
-            u16::from_str_radix(config_string, 10).expect("Couldn't parse port into a u16")
+            u16::from_str_radix(config_string, 10).ok()
         }
 
    or you SHOULD use `or()` (or another similar method):
diff --git a/doc/HACKING/Fuzzing.md b/doc/HACKING/Fuzzing.md
index 2039d6a4c0..487716bb6d 100644
--- a/doc/HACKING/Fuzzing.md
+++ b/doc/HACKING/Fuzzing.md
@@ -1,6 +1,6 @@
-= Fuzzing Tor
+# Fuzzing Tor
 
-== The simple version (no fuzzing, only tests)
+## The simple version (no fuzzing, only tests)
 
 Check out fuzzing-corpora, and set TOR_FUZZ_CORPORA to point to the place
 where you checked it out.
@@ -11,8 +11,7 @@ To run the fuzzing test cases in a deterministic fashion, use:
 This won't actually fuzz Tor!  It will just run all the fuzz binaries
 on our existing set of testcases for the fuzzer.
 
-
-== Different kinds of fuzzing
+## Different kinds of fuzzing
 
 Right now we support three different kinds of fuzzer.
 
@@ -26,7 +25,7 @@ have a reasonably recent clang and libfuzzer installed.  At that point, you
 just build with --enable-expensive-hardening and --enable-libfuzzer.  That
 will produce a set of binaries in src/test/fuzz/lf-fuzz-* .  These programs
 take as input a series of directories full of fuzzing examples.  For more
-information on libfuzzer, see http://llvm.org/docs/LibFuzzer.html
+information on libfuzzer, see https://llvm.org/docs/LibFuzzer.html
 
 Third, there's Google's OSS-Fuzz infrastructure, which expects to get all of
 its.  For more on this, see https://github.com/google/oss-fuzz and the
@@ -37,7 +36,7 @@ In all cases, you'll need some starting examples to give the fuzzer when it
 starts out.  There's a set in the "fuzzing-corpora" git repository.  Try
 setting TOR_FUZZ_CORPORA to point to a checkout of that repository
 
-== Writing Tor fuzzers
+## Writing Tor fuzzers
 
 A tor fuzzing harness should have:
 * a fuzz_init() function to set up any necessary global state.
@@ -51,8 +50,7 @@ But the fuzzing harness should crash if tor fails an assertion, triggers a
 bug, or accesses memory it shouldn't. This helps fuzzing frameworks detect
 "interesting" cases.
 
-
-== Guided Fuzzing with AFL
+## Guided Fuzzing with AFL
 
 There is no HTTPS, hash, or signature for American Fuzzy Lop's source code, so
 its integrity can't be verified. That said, you really shouldn't fuzz on a
@@ -74,7 +72,7 @@ and then not actually use it.
 
 Read afl/docs/notes_for_asan.txt for more details.
 
-  Download recidivm from http://jwilk.net/software/recidivm
+  Download recidivm from https://jwilk.net/software/recidivm
   Download the signature
   Check the signature
   tar xvzf recidivm*.tar.gz
@@ -101,7 +99,7 @@ macOS (OS X) requires slightly more preparation, including:
 * using afl-clang (or afl-clang-fast from the llvm directory)
 * disabling external crash reporting (AFL will guide you through this step)
 
-== Triaging Issues
+## Triaging Issues
 
 Crashes are usually interesting, particularly if using AFL_HARDEN=1 and --enable-expensive-hardening. Sometimes crashes are due to bugs in the harness code.
 
@@ -115,7 +113,7 @@ To see what fuzz-http is doing with a test case, call it like this:
 
 (Logging is disabled while fuzzing to increase fuzzing speed.)
 
-== Reporting Issues
+## Reporting Issues
 
 Please report any issues discovered using the process in Tor's security issue
 policy:
diff --git a/doc/HACKING/GettingStarted.md b/doc/HACKING/GettingStarted.md
index 0c42404634..633a7f0417 100644
--- a/doc/HACKING/GettingStarted.md
+++ b/doc/HACKING/GettingStarted.md
@@ -1,23 +1,19 @@
-
-Getting started in Tor development
-==================================
+# Getting started in Tor development
 
 Congratulations!  You've found this file, and you're reading it!  This
 means that you might be interested in getting started in developing Tor.
 
-(This guide is just about Tor itself--the small network program at the
+(_This guide is just about Tor itself--the small network program at the
 heart of the Tor network--and not about all the other programs in the
-whole Tor ecosystem.)
-
+whole Tor ecosystem._)
 
 If you are looking for a more bare-bones, less user-friendly information
-dump of important information, you might like reading the "torguts"
-documents linked to below. You should probably read it before you write
+dump of important information, you might like reading the
+[doxygen output](https://src-ref.docs.torproject.org/tor/index.html).
+You probably should skim some of the topic headings there before you write
 your first patch.
 
-
-Required background
--------------------
+## Required background
 
 First, I'm going to assume that you can build Tor from source, and that
 you know enough of the C language to read and write it.  (See the README
@@ -26,22 +22,20 @@ and any high-quality guide to C for information on programming.)
 
 I'm also going to assume that you know a little bit about how to use
 Git, or that you're able to follow one of the several excellent guides
-at http://git-scm.org to learn.
+at [git-scm](https://git-scm.org) to learn.
 
-Most Tor developers develop using some Unix-based system, such as Linux,
-BSD, or OSX.  It's okay to develop on Windows if you want, but you're
+Most Tor developers develop using some Unix-based system, such as GNU/Linux,
+BSD, or macOS.  It's okay to develop on Windows if you want, but you're
 going to have a more difficult time.
 
-
-Getting your first patch into Tor
----------------------------------
+## Getting your first patch into Tor
 
 Once you've reached this point, here's what you need to know.
 
   1. Get the source.
 
      We keep our source under version control in Git.  To get the latest
-     version, run
+     version, run:
 
          git clone https://git.torproject.org/git/tor
 
@@ -49,20 +43,18 @@ Once you've reached this point, here's what you need to know.
      going to fix a bug that appears in a stable version, check out the
      appropriate "maint" branch, as in:
 
-         git checkout maint-0.2.7
-
-  2. Find your way around the source
+         git checkout maint-0.4.3
 
-     Our overall code structure is explained in the "torguts" documents,
-     currently at
+  2. Find your way around the source.
 
-        git clone https://git.torproject.org/user/nickm/torguts.git
+     Our overall code structure is explained in our
+     [source documentation](https://src-ref.docs.torproject.org/tor/index.html).
 
      Find a part of the code that looks interesting to you, and start
      looking around it to see how it fits together!
 
      We do some unusual things in our codebase.  Our testing-related
-     practices and kludges are explained in doc/WritingTests.txt.
+     practices and kludges are explained in `doc/HACKING/WritingTests.md`.
 
      If you see something that doesn't make sense, we love to get
      questions!
@@ -74,10 +66,10 @@ Once you've reached this point, here's what you need to know.
 
      Many people have gotten started by looking for an area where they
      personally felt Tor was underperforming, and investigating ways to
-     fix it. If you're looking for ideas, you can head to our bug
-     tracker at trac.torproject.org and look for tickets that have
-     received the "easy" tag: these are ones that developers think would
-     be pretty simple for a new person to work on.  For a bigger
+     fix it. If you're looking for ideas, you can head to
+     [trac](https://trac.torproject.org) our bug tracking tool and look for
+     tickets that have received the "easy" tag: these are ones that developers
+     think would be pretty simple for a new person to work on.  For a bigger
      challenge, you might want to look for tickets with the "lorax"
      keyword: these are tickets that the developers think might be a
      good idea to build, but which we have no time to work on any time
@@ -96,7 +88,7 @@ Once you've reached this point, here's what you need to know.
 
   4. Meet the developers!
 
-     We discuss stuff on the tor-dev mailing list and on the #tor-dev
+     We discuss stuff on the tor-dev mailing list and on the `#tor-dev`
      IRC channel on OFTC.  We're generally friendly and approachable,
      and we like to talk about how Tor fits together.  If we have ideas
      about how something should be implemented, we'll be happy to share
@@ -113,8 +105,8 @@ Once you've reached this point, here's what you need to know.
      protocols, there needs to be a written design proposal before it
      can be merged. (We use this process to manage changes in the
      protocols.)  To write one, see the instructions at
-     https://gitweb.torproject.org/torspec.git/tree/proposals/001-process.txt
-     .  If you'd like help writing a proposal, just ask!  We're happy to
+     [the Tor proposal process](https://gitweb.torproject.org/torspec.git/plain/proposals/001-process.txt).
+     If you'd like help writing a proposal, just ask!  We're happy to
      help out with good ideas.
 
      You might also like to look around the rest of that directory, to
@@ -125,7 +117,7 @@ Once you've reached this point, here's what you need to know.
      As you write your code, you'll probably want it to fit in with the
      standards of the rest of the Tor codebase so it will be easy for us
      to review and merge.  You can learn our coding standards in
-     doc/HACKING.
+     `doc/HACKING` directory.
 
      If your patch is large and/or is divided into multiple logical
      components, remember to divide it into a series of Git commits.  A
@@ -137,16 +129,16 @@ Once you've reached this point, here's what you need to know.
      ensure that it runs correctly.  Also, all code should actually be
      _run_ by somebody, to make sure it works.
 
-     See doc/WritingTests.txt for more information on how we test things
+     See `doc/HACKING/WritingTests.md` for more information on how we test things
      in Tor.  If you'd like any help writing tests, just ask!  We're
      glad to help out.
 
   8. Submitting your patch
 
      We review patches through tickets on our bugtracker at
-     trac.torproject.org.  You can either upload your patches there, or
+    [trac](https://trac.torproject.org). You can either upload your patches there, or
      put them at a public git repository somewhere we can fetch them
-     (like github or bitbucket) and then paste a link on the appropriate
+     (like gitlab, github or bitbucket) and then paste a link on the appropriate
      trac ticket.
 
      Once your patches are available, write a short explanation of what
@@ -163,17 +155,17 @@ Once you've reached this point, here's what you need to know.
 
      When your patch is reviewed, one of these things will happen:
 
-       * The reviewer will say "looks good to me" and your
+       * The reviewer will say "_looks good to me_" and your
          patch will get merged right into Tor.  [Assuming we're not
          in the middle of a code-freeze window.  If the codebase is
          frozen, your patch will go into the next release series.]
 
-       * OR the reviewer will say "looks good, just needs some small
-         changes!"  And then the reviewer will make those changes,
+       * OR the reviewer will say "_looks good, just needs some small
+         changes!_"  And then the reviewer will make those changes,
          and merge the modified patch into Tor.
 
-       * OR the reviewer will say "Here are some questions and
-         comments," followed by a bunch of stuff that the reviewer
+       * OR the reviewer will say "_Here are some questions and
+         comments,_" followed by a bunch of stuff that the reviewer
          thinks should change in your code, or questions that the
          reviewer has.
 
diff --git a/doc/HACKING/GettingStartedRust.md b/doc/HACKING/GettingStartedRust.md
index aa29c097da..af80018f4e 100644
--- a/doc/HACKING/GettingStartedRust.md
+++ b/doc/HACKING/GettingStartedRust.md
@@ -1,12 +1,9 @@
+# Hacking on Rust in Tor
 
- Hacking on Rust in Tor
-========================
-
- Getting Started
------------------
+## Getting Started
 
 Please read or review our documentation on Rust coding standards
-(`.../doc/HACKING/CodingStandardsRust.md`) before doing anything.
+(`doc/HACKING/CodingStandardsRust.md`) before doing anything.
 
 Please also read
 [the Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). We
@@ -23,8 +20,7 @@ Please be patient with the other people who are working on getting more
 Rust code into Tor, because they are graciously donating their free time
 to contribute to this effort.
 
- Resources for learning Rust
------------------------------
+## Resources for learning Rust
 
 **Beginning resources**
 
@@ -47,10 +43,9 @@ is
 [The Little Book of Rust Macros](https://danielkeep.github.io/tlborm/book/index.html).
 
 For learning more about FFI and Rust, see Jake Goulding's
-[Rust FFI Omnibus](http://jakegoulding.com/rust-ffi-omnibus/).
+[Rust FFI Omnibus](https://jakegoulding.com/rust-ffi-omnibus/).
 
- Compiling Tor with Rust enabled
----------------------------------
+## Compiling Tor with Rust enabled
 
 You will need to run the `configure` script with the `--enable-rust`
 flag to explicitly build with Rust. Additionally, you will need to
@@ -79,7 +74,7 @@ you are in the top level of the repository) configure tor with:
 
     TOR_RUST_DEPENDENCIES='path_to_dependencies_directory' ./configure --enable-rust
 
-(Note that TOR_RUST_DEPENDENCIES must be the full path to the directory; it
+(Note that `TOR_RUST_DEPENDENCIES` must be the full path to the directory; it
 cannot be relative.)
 
 Assuming you used the above `git submodule` commands and you're in the
@@ -87,9 +82,7 @@ topmost directory of the repository, this would be:
 
     TOR_RUST_DEPENDENCIES=`pwd`/src/ext/rust/crates ./configure --enable-rust
 
-
- Identifying which modules to rewrite
-======================================
+## Identifying which modules to rewrite
 
 The places in the Tor codebase that are good candidates for porting to
 Rust are:
@@ -117,12 +110,11 @@ interconnected your target module is.
 The output will tell you each module name, along with a set of every module that
 the module calls.  Modules which call fewer other modules are better targets.
 
- Writing your Rust module
-==========================
+## Writing your Rust module
 
 Strive to change the C API as little as possible.
 
-We are currently targetting Rust stable. (See CodingStandardsRust.md for more
+We are currently targetting Rust stable. (See `CodingStandardsRust.md` for more
 details.)
 
 It is on our TODO list to try to cultivate good
@@ -134,19 +126,17 @@ If parts of your Rust code needs to stay in sync with C code (such as
 handling enums across the FFI boundary), annonotate these places in a
 comment structured as follows:
 
-  /// C_RUST_COUPLED: <path_to_file> `<name_of_c_object>`
+  `/// C_RUST_COUPLED: <path_to_file> <name_of_c_object>`
 
-Where <name_of_c_object> can be an enum, struct, constant, etc.  Then,
+Where `<name_of_c_object>` can be an enum, struct, constant, etc.  Then,
 do the same in the C code, to note that rust will need to be changed
 when the C does.
 
-
- Adding your Rust module to Tor's build system
------------------------------------------------
+## Adding your Rust module to Tor's build system
 
 0. Your translation of the C module should live in its own crate(s)
-   in the `.../tor/src/rust/` directory.
-1. Add your crate to `.../tor/src/rust/Cargo.toml`, in the
+   in the `src/rust/` directory.
+1. Add your crate to `src/rust/Cargo.toml`, in the
    `[workspace.members]` section.
 2. Add your crate's files to src/rust/include.am
 
@@ -156,12 +146,11 @@ dependency of other Rust modules):
    `src/rust/tor_util/Cargo.toml` and include it in
    `src/rust/tor_rust/lib.rs`
 
- How to test your Rust code
-----------------------------
+## How to test your Rust code
 
 Everything should be tested full stop.  Even non-public functionality.
 
-Be sure to edit `.../tor/src/test/test_rust.sh` to add the name of your
+Be sure to edit `src/test/test_rust.sh` to add the name of your
 crate to the `crates` variable! This will ensure that `cargo test` is
 run on your crate.
 
@@ -177,7 +166,6 @@ Tor's integration tests should also pass:
 
     make test-stem
 
- Submitting a patch
-=====================
+## Submitting a patch
 
-Please follow the instructions in `.../doc/HACKING/GettingStarted.md`.
+Please follow the instructions in `doc/HACKING/GettingStarted.md`.
diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md
index d499238526..15bd153318 100644
--- a/doc/HACKING/HelpfulTools.md
+++ b/doc/HACKING/HelpfulTools.md
@@ -1,11 +1,10 @@
-Useful tools
-============
+# Useful tools
 
 These aren't strictly necessary for hacking on Tor, but they can help track
 down bugs.
 
-Travis/Appveyor CI
-------------------
+## Travis/Appveyor CI
+
 It's CI.
 
 Looks like this:
@@ -29,8 +28,7 @@ for your fork to build commits outside of PRs too:
 Builds should show up on the web at travis-ci.com and on IRC at #tor-ci on
 OFTC. If they don't, ask #tor-dev (also on OFTC).
 
-Jenkins
--------
+## Jenkins
 
 It's CI/builders. Looks like this: https://jenkins.torproject.org
 
@@ -43,8 +41,7 @@ Builds Linux and Windows cross-compilation. Runs Linux tests.
 Builds should show up on the web at jenkins.torproject.org and on IRC at
 #tor-bots on OFTC. If they don't, ask #tor-dev (also on OFTC).
 
-Valgrind
---------
+## Valgrind
 
     valgrind --leak-check=yes --error-limit=no --show-reachable=yes src/app/tor
 
@@ -52,16 +49,14 @@ Valgrind
 pass `--undef-value-errors=no` to valgrind, or rebuild your openssl
 with `-DPURIFY`.)
 
-Coverity
---------
+## Coverity
 
 Nick regularly runs the coverity static analyzer on the Tor codebase.
 
 The preprocessor define `__COVERITY__` is used to work around instances
 where coverity picks up behavior that we wish to permit.
 
-clang Static Analyzer
----------------------
+## clang Static Analyzer
 
 The clang static analyzer can be run on the Tor codebase using Xcode (WIP)
 or a command-line build.
@@ -69,8 +64,7 @@ or a command-line build.
 The preprocessor define `__clang_analyzer__` is used to work around instances
 where clang picks up behavior that we wish to permit.
 
-clang Runtime Sanitizers
-------------------------
+## clang Runtime Sanitizers
 
 To build the Tor codebase with the clang Address and Undefined Behavior
 sanitizers, see the file `contrib/clang/sanitize_blacklist.txt`.
@@ -78,8 +72,7 @@ sanitizers, see the file `contrib/clang/sanitize_blacklist.txt`.
 Preprocessor workarounds for instances where clang picks up behavior that
 we wish to permit are also documented in the blacklist file.
 
-Running lcov for unit test coverage
------------------------------------
+## Running lcov for unit test coverage
 
 Lcov is a utility that generates pretty HTML reports of test code coverage.
 To generate such a report:
@@ -96,8 +89,7 @@ output directory, use `make coverage-html HTML_COVER_DIR=./funky_new_cov_dir`.
 Coverage diffs using lcov are not currently implemented, but are being
 investigated (as of July 2014).
 
-Running the unit tests
-----------------------
+## Running the unit tests
 
 To quickly run all the tests distributed with Tor:
 
@@ -123,8 +115,7 @@ working connection to the internet:
 
     make test-full-online
 
-Running gcov for unit test coverage
------------------------------------
+## Running gcov for unit test coverage
 
     ./configure --enable-coverage
     make
@@ -164,8 +155,7 @@ lines?
 If you run ./scripts/test/cov-exclude, it marks excluded unreached
 lines with 'x', and excluded reached lines with '!!!'.
 
-Running integration tests
--------------------------
+## Running integration tests
 
 We have the beginnings of a set of scripts to run integration tests using
 Chutney. To try them, set CHUTNEY_PATH to your chutney source directory, and
@@ -174,14 +164,12 @@ run `make test-network`.
 We also have scripts to run integration tests using Stem.  To try them, set
 `STEM_SOURCE_DIR` to your Stem source directory, and run `test-stem`.
 
-Profiling Tor
--------------
+## Profiling Tor
 
 Ongoing notes about Tor profiling can be found at
 https://pad.riseup.net/p/profiling-tor
 
-Profiling Tor with oprofile
----------------------------
+## Profiling Tor with oprofile
 
 The oprofile tool runs (on Linux only!) to tell you what functions Tor is
 spending its CPU time in, so we can identify performance bottlenecks.
@@ -206,8 +194,7 @@ Here are some basic instructions
    * `opreport -l that_dir/*`
  - Profit
 
-Profiling Tor with perf
------------------------
+## Profiling Tor with perf
 
 This works with a running Tor, and requires root.
 
@@ -236,8 +223,7 @@ This works with a running Tor, and requires root.
     report -g > <FILENAME>.out`. Then you can compress that and put it somewhere
     public.
 
-Profiling Tor with gperftools aka Google-performance-tools
-----------------------------------------------------------
+## Profiling Tor with gperftools aka Google-performance-tools
 
 This should work on nearly any unixy system. It doesn't seem to be compatible
 with RunAsDaemon though.
@@ -251,23 +237,21 @@ Now you can run Tor with profiling enabled, and use the pprof utility to look at
 performance! See the gperftools manual for more info, but basically:
 
 2. Run `env CPUPROFILE=/tmp/profile src/app/tor -f <path/torrc>`. The profile file
-   is not written to until Tor finishes execuction.
+   is not written to until Tor finishes execution.
 
-3. Run `pprof src/app/tor /tm/profile` to start the REPL.
+3. Run `pprof src/app/tor /tmp/profile` to start the REPL.
 
-Generating and analyzing a callgraph
-------------------------------------
+## Generating and analyzing a callgraph
 
 0. Build Tor on linux or mac, ideally with -O0 or -fno-inline.
 
-1. Clone 'https://gitweb.torproject.org/user/nickm/calltool.git/' .
+1. Clone 'https://git.torproject.org/user/nickm/calltool.git/' .
    Follow the README in that repository.
 
 Note that currently the callgraph generator can't detect calls that pass
 through function pointers.
 
-Getting emacs to edit Tor source properly
------------------------------------------
+## Getting emacs to edit Tor source properly
 
 Nick likes to put the following snippet in his .emacs file:
 
@@ -315,9 +299,31 @@ If you use emacs for editing Tor and nothing else, you could always just say:
 There is probably a better way to do this.  No, we are probably not going
 to clutter the files with emacs stuff.
 
+## Building a tag file (code index)
+
+Many functions in tor use `MOCK_IMPL` wrappers for unit tests. Your
+tag-building program must be told how to handle this syntax.
+
+If you're using emacs, you can generate an emacs-compatible tag file using
+`make tags`. This will run your system's `etags`. Tor's build system assumes
+that you're using the emacs-specific version of `etags` (bundled under the
+`xemacs21-bin` package on Debian). This is incompatible with other versions of
+`etags` such as the version provided by Exuberant Ctags.
+
+If you're using vim or emacs, you can also use Universal Ctags to build a tag
+file using the syntax:
+
+    ctags -R -D 'MOCK_IMPL(r,h,a)=r h a' .
+
+If you're using an older version of Universal Ctags, you can use the following
+instead:
 
-Doxygen
--------
+    ctags -R --mline-regex-c='/MOCK_IMPL\([^,]+,\W*([a-zA-Z0-9_]+)\W*,/\1/f/{mgroup=1}' .
+
+A vim-compatible tag file will be generated by default. If you use emacs, add
+the `-e` flag to generate an emacs-compatible tag file.
+
+## Doxygen
 
 We use the 'doxygen' utility to generate documentation from our
 source code. Here's how to use it:
@@ -371,3 +377,17 @@ source code. Here's how to use it:
 
   6. See the Doxygen manual for more information; this summary just
      scratches the surface.
+
+## Style and best-practices checking
+
+We use scripts to check for various problems in the formatting and style
+of our source code.  The "check-spaces" test detects a bunch of violations
+of our coding style on the local level.  The "check-best-practices" test
+looks for violations of some of our complexity guidelines.
+
+You can tell the tool about exceptions to the complexity guidelines via its
+exceptions file (scripts/maint/practracker/exceptions.txt).  But before you
+do this, consider whether you shouldn't fix the underlying problem.  Maybe
+that file really _is_ too big.  Maybe that function really _is_ doing too
+much.  (On the other hand, for stable release series, it is sometimes better
+to leave things unrefactored.)
diff --git a/doc/HACKING/HowToReview.md b/doc/HACKING/HowToReview.md
index 2d1f3d1c9e..7815e76632 100644
--- a/doc/HACKING/HowToReview.md
+++ b/doc/HACKING/HowToReview.md
@@ -1,5 +1,4 @@
-How to review a patch
-=====================
+# How to review a patch
 
 Some folks have said that they'd like to review patches more often, but they
 don't know how.
@@ -9,9 +8,7 @@ So, here are a bunch of things to check for when reviewing a patch!
 Note that if you can't do every one of these, that doesn't mean you can't do
 a good review!  Just make it clear what you checked for and what you didn't.
 
-
-Top-level smell-checks
-----------------------
+## Top-level smell-checks
 
 (Difficulty: easy)
 
@@ -37,10 +34,9 @@ Top-level smell-checks
 - If this changes anything in the code, is there a "changes" file?
 
 
-Let's look at the code!
------------------------
+## Let's look at the code!
 
-- Does the code conform to CodingStandards.txt?
+- Does the code conform to `CodingStandards.md`?
 
 - Does the code leak memory?
 
@@ -60,18 +56,16 @@ Let's look at the code!
 - Is there duplicated code that could be turned into a function?
 
 
-Let's look at the documentation!
---------------------------------
+## Let's look at the documentation!
 
-- Does the documentation confirm to CodingStandards.txt?
+- Does the documentation conform to CodingStandards.txt?
 
 - Does it make sense?
 
 - Can you predict what the function will do from its documentation?
 
 
-Let's think about security!
----------------------------
+## Let's think about security!
 
 - If there are any arrays, buffers, are you 100% sure that they cannot
   overflow?
diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md
new file mode 100644
index 0000000000..4d5a7f6b76
--- /dev/null
+++ b/doc/HACKING/Maintaining.md
@@ -0,0 +1,113 @@
+# Maintaining Tor
+
+This document details the duties and processes on maintaining the Tor code
+base.
+
+The first section describes who is the current Tor maintainer and what are the
+responsibilities. Tor has one main single maintainer but does have many
+committers and subsystem maintainers.
+
+The second third section describes how the **alpha and master** branches are
+maintained and by whom.
+
+Finally, the last section describes how the **stable** branches are maintained
+and by whom.
+
+This document does not cover how Tor is released, please see
+[ReleasingTor.md](ReleasingTor.md) for that information.
+
+## Tor Maintainer
+
+The current maintainer is Nick Mathewson <nickm@torproject.org>.
+
+The maintainer takes final decisions in terms of engineering, architecture and
+protocol design. Releasing Tor falls under their responsibility.
+
+## Alpha and Master Branches
+
+The Tor repository always has at all times a **master** branch which contains
+the upstream ongoing development.
+
+It may also contain a branch for a released feature freezed version which is
+called the **alpha** branch. The git tag and version number is always
+postfixed with `-alpha[-dev]`. For example: `tor-0.3.5.0-alpha-dev` or
+`tor-0.3.5.3-alpha`.
+
+Tor is separated into subsystems and some of those are maintained by other
+developers than the main maintainer. Those people have commit access to the
+code base but only commit (in most cases) into the subsystem they maintain.
+
+Upstream merges are restricted to the alpha and master branches. Subsystem
+maintainers should never push a patch into a stable branch which is the
+responsibility of the [stable branch maintainer](#stable-branches).
+
+### Who
+
+In alphabetical order, the following people have upstream commit access and
+maintain the following subsystems:
+
+- David Goulet <dgoulet@torproject.org>
+  * Onion Service (including Shared Random).  
+    ***keywords:*** *[tor-hs]*
+  * Channels, Circuitmux, Connection, Scheduler.  
+    ***keywords:*** *[tor-chan, tor-cmux, tor-sched, tor-conn]*
+  * Cell Logic (Handling/Parsing).  
+    ***keywords:*** *[tor-cell]*
+  * Threading backend.
+    ***keywords:*** *[tor-thread]*  
+
+- George Kadianakis <asn@torproject.org>
+  * Onion Service (including Shared Random).  
+    ***keywords:*** *[tor-hs]*
+  * Guard.  
+    ***keywords:*** *[tor-guard]*
+  * Pluggable Transport (excluding Bridge networking).  
+    ***keywords:*** *[tor-pt]*
+
+### Tasks
+
+These are the tasks of a subsystem maintainer:
+
+1. Regularly go over `merge_ready` tickets relevant to the related subsystem
+   and for the current alpha or development (master branch) Milestone.
+
+2. A subsystem maintainer is expected to contribute to any design changes
+   (including proposals) or large patch set about the subsystem.
+
+3. Leave their ego at the door. Mistakes will be made but they have to be
+   taking care of seriously. Learn and move on quickly.
+
+### Merging Policy
+
+These are few important items to follow when merging code upstream:
+
+1. To merge code upstream, the patch must have passed our CI (currently
+   github.com/torproject), have a corresponding ticket and reviewed by
+   **at least** one person that is not the original coder.
+
+   Example A: If Alice writes a patch then Bob, a Tor network team member,
+   reviews it and flags it `merge_ready`. Then, the maintainer is required
+   to look at the patch and makes a decision.
+
+   Example B: If the maintainer writes a patch then Bob, a Tor network
+   team member, reviews it and flags it `merge_ready`, then the maintainer
+   can merge the code upstream.
+
+2. Maintainer makes sure the commit message should describe what was fixed
+   and, if it applies, how was it fixed. It should also always refer to
+   the ticket number.
+
+3. Trivial patches such as comment change, documentation, syntax issues or
+   typos can be merged without a ticket or reviewers.
+
+4. Tor uses the "merge forward" method, that is, if a patch applies to the
+   alpha branch, it has to be merged there first and then merged forward
+   into master.
+
+5. Maintainer should always consult with the network team about any doubts,
+   mis-understandings or unknowns of a patch. Final word will always go to the
+   main Tor maintainer.
+
+## Stable Branches
+
+(Currently being drafted and reviewed by the network team.)
diff --git a/doc/HACKING/Module.md b/doc/HACKING/Module.md
index 9cf36090b4..f8a9773d47 100644
--- a/doc/HACKING/Module.md
+++ b/doc/HACKING/Module.md
@@ -1,22 +1,33 @@
-# Modules in Tor #
+# Modules in Tor
 
 This document describes the build system and coding standards when writing a
 module in Tor.
 
-## What is a module? ##
+## What is a module?
 
 In the context of the tor code base, a module is a subsystem that we can
 selectively enable or disable, at `configure` time.
 
-Currently, there is only one module:
+Currently, tor has these modules:
 
+  - Relay subsystem (relay)
+  - Directory cache system (dircache).
   - Directory Authority subsystem (dirauth)
 
-It is located in its own directory in `src/feature/dirauth/`. To disable it,
-one need to pass `--disable-module-dirauth` at configure time. All modules
-are currently enabled by default.
+The dirauth code is located in its own directory in `src/feature/dirauth/`.
 
-## Build System ##
+The relay code is located in a directory named `src/*/*relay`, which is
+being progressively refactored and disabled.
+
+The dircache code is located in `src/*/*dircache`.  Right now, it is
+disabled if and only if the relay module is disabled.  (We are treating
+them as separate modules because they are logically independent, not
+because you would actually want to run one without the other.)
+
+To disable a module, pass `--disable-module-{dirauth,relay}` at configure
+time. All modules are currently enabled by default.
+
+## Build System
 
 The changes to the build system are pretty straightforward.
 
@@ -24,7 +35,7 @@ The changes to the build system are pretty straightforward.
    contains a list (white-space separated) of the module in tor. Add yours to
    the list.
 
-2. Use the `AC_ARG_ENABLE([module-dirauth]` template for your new module. We
+2. Use the `AC_ARG_ENABLE([module-relay]` template for your new module. We
    use the "disable module" approach instead of enabling them one by one. So,
    by default, tor will build all the modules.
 
@@ -32,7 +43,7 @@ The changes to the build system are pretty straightforward.
    the C code to conditionally compile things for your module. And the
    `BUILD_MODULE_<name>` is also defined for automake files (e.g: include.am).
 
-3. In the `src/core/include.am` file, locate the `MODULE_DIRAUTH_SOURCES`
+3. In the `src/core/include.am` file, locate the `MODULE_RELAY_SOURCES`
    value.  You need to create your own `_SOURCES` variable for your module
    and then conditionally add the it to `LIBTOR_A_SOURCES` if you should
    build the module.
@@ -40,18 +51,14 @@ The changes to the build system are pretty straightforward.
    It is then **very** important to add your SOURCES variable to
    `src_or_libtor_testing_a_SOURCES` so the tests can build it.
 
-4. Do the same for header files, locate `ORHEADERS +=` which always add all
-   headers of all modules so the symbol can be found for the module entry
-   points.
-
 Finally, your module will automatically be included in the
-`TOR_MODULES_ALL_ENABLED` variable which is used to build the unit tests. They
-always build everything in order to tests everything.
+`TOR_MODULES_ALL_ENABLED` variable which is used to build the unit tests.
+They always build everything in order to test everything.
 
-## Coding ##
+## Coding
 
-As mentioned above, a module must be isolated in its own directory (name of
-the module) in `src/feature/`.
+As mentioned above, a module should be isolated in its own directories,
+suffixed with the name of the module, in `src/*/`.
 
 There are couples of "rules" you want to follow:
 
diff --git a/doc/HACKING/README.1st.md b/doc/HACKING/README.1st.md
index 8299fe634a..2278a61d6c 100644
--- a/doc/HACKING/README.1st.md
+++ b/doc/HACKING/README.1st.md
@@ -1,17 +1,18 @@
+# README.1st
 
-In this directory
------------------
+## In this directory
 
 This directory has helpful information about what you need to know to
 hack on Tor!
 
-First, read `GettingStarted.md` to learn how to get a start in Tor
-development.
+First, read `GettingStarted.md` and `GettingStartedRust.md`
+to learn how to get a start in Tor development.
 
-If you've decided to write a patch, `CodingStandards.txt` will give
-you a bunch of information about how we structure our code.
+If you've decided to write a patch, `CodingStandards.md` and
+`CodingStandardsRust.md` will give you a bunch of information
+about how we structure our code.
 
-It's important to get code right!  Reading `WritingTests.md` will
+It's important to get the code right!  Reading `WritingTests.md` will
 tell you how to write and run tests in the Tor codebase.
 
 There are a bunch of other programs we use to help maintain and
@@ -21,42 +22,26 @@ with Tor.
 If it's your job to put out Tor releases, see `ReleasingTor.md` so
 that you don't miss any steps!
 
-
------------------------
+## Additional Information
 
 For full information on how Tor is supposed to work, look at the files in
-`https://gitweb.torproject.org/torspec.git/tree`.
+[Tor specification](https://gitweb.torproject.org/torspec.git/tree).
 
 For an explanation of how to change Tor's design to work differently, look at
-`https://gitweb.torproject.org/torspec.git/blob_plain/HEAD:/proposals/001-process.txt`.
+[the Tor proposal process](https://gitweb.torproject.org/torspec.git/plain/proposals/001-process.txt).
 
 For the latest version of the code, get a copy of git, and
 
     git clone https://git.torproject.org/git/tor
 
+## Stay in touch
+
 We talk about Tor on the `tor-talk` mailing list.  Design proposals and
 discussion belong on the `tor-dev` mailing list.  We hang around on
-irc.oftc.net, with general discussion happening on #tor and development
+irc.oftc.net, with general discussion happening on `#tor` and development
 happening on `#tor-dev`.
 
 The other files in this `HACKING` directory may also be useful as you
 get started working with Tor.
 
 Happy hacking!
-
-
------------------------
-
-XXXXX also describe
-
-doc/HACKING/WritingTests.md
-
-torguts.git
-
-torspec.git
-
-The design paper
-
-freehaven.net/anonbib
-
-XXXX describe these and add links.
diff --git a/doc/HACKING/ReleaseSeriesLifecycle.md b/doc/HACKING/ReleaseSeriesLifecycle.md
new file mode 100644
index 0000000000..e4068ed806
--- /dev/null
+++ b/doc/HACKING/ReleaseSeriesLifecycle.md
@@ -0,0 +1,113 @@
+# Release Series Lifecycle
+
+
+## End Of Life On An Old Release Series
+
+Here are the steps that the maintainer should take when an old Tor release
+series reaches End of Life.
+
+Note that they are _only_ for an entire series that has reached its planned
+EOL: they do not apply to security-related deprecations of individual
+patch versions.
+
+
+### 1. Preliminaries
+
+1. A few months before End of Life:
+   Write a deprecation announcement.
+   Send the announcement out with every new release announcement.
+
+2. A month before End of Life:
+   Send the announcement to tor-announce, tor-talk, tor-relays, and the
+   packagers.
+
+
+### 2. On The Day
+
+1. Open tickets to remove the release from:
+   - the jenkins builds
+   - tor's Travis CI cron jobs
+   - chutney's Travis CI tests
+   - sbws' Travis CI tests
+   - stem's Travis CI tests (but see
+     https://github.com/torproject/stem/issues/51)
+   - tor's scripts/git/gist-list-tor-branches.sh script
+
+2. Close the milestone in Trac. To do this, go to Trac, log in,
+   select "Admin" near the top of the screen, then select "Milestones" from
+   the menu on the left.  Click on the milestone for this version, and
+   select the "Completed" checkbox. By convention, we select the date as
+   the End of Life date.
+
+3. Replace NNN-backport with NNN-unreached-backport in all open trac tickets.
+
+4. If there are any remaining tickets in the milestone:
+     - merge_ready tickets are for backports:
+       - if there are no supported releases for the backport, close the ticket
+       - if there is an earlier (LTS) release for the backport, move the ticket
+         to that release
+     - other tickets should be closed (if we won't fix them) or moved to a
+       supported release (if we will fix them)
+
+5. Mail the end of life announcement to tor-announce, the packagers list,
+   and tor-relays. The current list of packagers is in ReleasingTor.md.
+
+6. Ask at least two of weasel/arma/Sebastian to remove the old version
+   number from their approved versions list.
+
+7. Update the CoreTorReleases wiki page.
+
+8. Open a ticket (if there is not one already) for authorities to
+    start rejecting relays that are running that release series.
+    This ticket should be targeted for at least a month or two
+    after the series is officially EOL, unless there is an important
+    reason to un-list relays early.
+
+9. (LTS end-of-life only) Open a ticket (if appropriate) for updates to the
+    set of required and recommended subprotocol versions.  (For the process
+    here, see proposal 303.)
+
+10. (LTS end-of-life only) Open a ticket to remove no-longer-needed
+    consensus methods. (For the process here, see proposal 290.)
+
+11. (All EOL) Open a ticket to grep for obsolete series names (e.g., "0.2.9"
+    and "029") in tor, chutney, sbws, fallback-scripts, and so on. These
+    should be updated or removed.
+
+12. Finally, make sure this document is up to date with our latest
+   process.
+
+## Starting A New Release Series
+
+Here are the steps that the maintainer should take to start new maint and
+release branches for a stable release.
+
+Note that they are _only_ for an entire series, when it first becomes stable:
+they do not apply to security-related patch release versions.
+
+(Ideally, do this immediately after a release.)
+
+1. Start a new maint-x.y.z branch based on master, and a new
+   release-x.y.z branch based on master. They should have the same
+   starting point.
+
+   Push both of these branches to the master git repository.
+
+2. In master, change the version to "0.x.y.0-alpha-dev". Run the
+   update_versions.py script, and commit this version bump.
+
+3. Tag the version bump with "tor-0.x.y.0-alpha-dev". Push the tag
+   and master.
+
+4. Open tickets for connecting the new branches to various other
+   places.  See section 2 above for a list of affected locations.
+
+5. Stop running practracker on maintainence and release branches:
+   * Remove "check-best-practices" from the check-local Makefile
+     target in the maint-x.y.z branch only.
+   * Delete the file scripts/maint/practracker/.enable_practracker_in_hooks
+     in the maint-x.y.z branch only.
+   * Merge to release-x.y.z, but do not forward-port to master.
+
+6. Finally, make sure this document is up to date with our latest
+   process.
diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md
index 55a40fc89b..2464d8afb4 100644
--- a/doc/HACKING/ReleasingTor.md
+++ b/doc/HACKING/ReleasingTor.md
@@ -1,11 +1,9 @@
-
-Putting out a new release
--------------------------
+# Putting out a new release
 
 Here are the steps that the maintainer should take when putting out a
 new Tor release:
 
-=== 0. Preliminaries
+## 0. Preliminaries
 
 1. Get at least two of weasel/arma/Sebastian to put the new
    version number in their approved versions list.  Give them a few
@@ -18,36 +16,40 @@ new Tor release:
    date of a TB that contains it.  See note below in "commit, upload,
    announce".
 
-=== I. Make sure it works
-
-1. Use it for a while, as a client, as a relay, as a hidden service,
-   and as a directory authority. See if it has any obvious bugs, and
-   resolve those.
+## I. Make sure it works
 
-   As applicable, merge the `maint-X` branch into the `release-X` branch.
-   But you've been doing that all along, right?
+1. Make sure that CI passes: have a look at Travis
+   (https://travis-ci.org/torproject/tor/branches), Appveyor
+   (https://ci.appveyor.com/project/torproject/tor/history), and
+   Jenkins (https://jenkins.torproject.org/view/tor/).
+   Make sure you're looking at the right branches.
 
-2. Are all of the jenkins builders happy?  See jenkins.torproject.org.
+   If there are any unexplained failures, try to fix them or figure them
+   out.
 
-   What about the bsd buildbots?
-         See http://buildbot.pixelminers.net/builders/
+2. Verify that there are no big outstanding issues.  You might find such
+   issues --
 
-   What about Coverity Scan?
+    * On Trac
 
-   What about clang scan-build?
+    * On coverity scan
 
-   Does 'make distcheck' complain?
+    * On OSS-Fuzz
 
-   How about 'make test-stem' and 'make test-network' and
-   `make test-network-full`?
+3. Run checks that aren't covered above, including:
 
-       - Are all those tests still happy with --enable-expensive-hardening ?
+    * clang scan-build.  (See the script in ./scripts/test/scan_build.sh)
 
-   Any memory leaks?
+    * make test-network and make test-network-all (with
+      --enable-fragile-hardening)
 
+    * Running Tor yourself and making sure that it actually works for you.
 
-=== II. Write a changelog
+    * Running Tor under valgrind.  (Our 'fragile hardening' doesn't cover
+      libevent and openssl, so using valgrind will sometimes find extra
+      memory leaks.)
 
+## II. Write a changelog
 
 1a. (Alpha release variant)
 
@@ -55,11 +57,14 @@ new Tor release:
    of them and reordering to focus on what users and funders would find
    interesting and understandable.
 
-   To do this, first run `./scripts/maint/lintChanges.py changes/*` and
-   fix as many warnings as you can.  Then run `./scripts/maint/sortChanges.py
-   changes/* > changelog.in` to combine headings and sort the entries.
-   After that, it's time to hand-edit and fix the issues that lintChanges
-   can't find:
+   To do this, run
+      `./scripts/maint/sortChanges.py changes/* > changelog.in`
+   to combine headings and sort the entries.  Copy the changelog.in file
+   into the ChangeLog.  Run 'format_changelog.py' (see below) to clean
+   up the line breaks.
+
+   After that, it's time to hand-edit and fix the issues that
+   lintChanges can't find:
 
    1. Within each section, sort by "version it's a bugfix on", else by
       numerical ticket order.
@@ -68,8 +73,6 @@ new Tor release:
 
       Make stuff very terse
 
-      Make sure each section name ends with a colon
-
       Describe the user-visible problem right away
 
       Mention relevant config options by name.  If they're rare or unusual,
@@ -79,7 +82,9 @@ new Tor release:
 
       Present and imperative tense: not past.
 
-      'Relays', not 'servers' or 'nodes' or 'Tor relays'.
+      "Relays", not "servers" or "nodes" or "Tor relays".
+
+      "Onion services", not "hidden services".
 
       "Stop FOOing", not "Fix a bug where we would FOO".
 
@@ -100,12 +105,14 @@ new Tor release:
 
    For stable releases that backport things from later, we try to compose
    their releases, we try to make sure that we keep the changelog entries
-   identical to their original versions, with a 'backport from 0.x.y.z'
+   identical to their original versions, with a "backport from 0.x.y.z"
    note added to each section.  So in this case, once you have the items
    from the changes files copied together, don't use them to build a new
    changelog: instead, look up the corrected versions that were merged
    into ChangeLog in the master branch, and use those.
 
+   Add "backport from X.Y.Z" in the section header for these entries.
+
 2. Compose a short release blurb to highlight the user-facing
    changes. Insert said release blurb into the ChangeLog stanza. If it's
    a stable release, add it to the ReleaseNotes file too. If we're adding
@@ -127,52 +134,65 @@ new Tor release:
    to start sorting and condensing entries.  (Generally, we don't edit the
    text of existing entries, though.)
 
-
-=== III. Making the source release.
+## III. Making the source release.
 
 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run
-   `perl scripts/maint/updateVersions.pl` to update version numbers in other
+   `make update-versions` to update version numbers in other
    places, and commit.  Then merge `maint-0.?.x` into `release-0.?.x`.
 
-   (NOTE: To bump the version number, edit `configure.ac`, and then run
-   either `make`, or `perl scripts/maint/updateVersions.pl`, depending on
-   your version.)
-
    When you merge the maint branch forward to the next maint branch, or into
    master, merge it with "-s ours" to avoid a needless version bump.
 
 2. Make distcheck, put the tarball up in somewhere (how about your
-   homedir on your homedir on people.torproject.org?) , and tell `#tor`
-   about it. Wait a while to see if anybody has problems building it.
-   (Though jenkins is usually pretty good about catching these things.)
+   homedir on your homedir on people.torproject.org?) , and tell `#tor-dev`
+   about it.
 
-=== IV. Commit, upload, announce
+   If you want, wait until at least one person has built it
+   successfully.  (We used to say "wait for others to test it", but our
+   CI has successfully caught these kinds of errors for the last several
+   years.)
+
+
+3. Make sure that the new version is recommended in the latest consensus.
+   (Otherwise, users will get confused when it complains to them
+   about its status.)
+
+   If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see
+   item 0.1 at the start of this document.
+
+## IV. Commit, upload, announce
 
 1. Sign the tarball, then sign and push the git tag:
 
         gpg -ba <the_tarball>
-        git tag -u <keyid> tor-0.3.x.y-status
-        git push origin tag tor-0.3.x.y-status
+        git tag -s tor-0.4.x.y-<status>
+        git push origin tag tor-0.4.x.y-<status>
 
-   (You must do this before you update the website: it relies on finding
-   the version by tag.)
+   (You must do this before you update the website: the website scripts
+   rely on finding the version by tag.)
+
+   (If your default PGP key is not the one you want to sign with, then say
+   "-u <keyid>" instead of "-s".)
 
 2. scp the tarball and its sig to the dist website, i.e.
-   `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want
-   it to go live, you run "static-update-component dist.torproject.org"
-   on dist-master.
+   `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run
+   "static-update-component dist.torproject.org" on dist-master.
 
-   In the webwml.git repository, `include/versions.wmi` and `Makefile`
-   to note the new version.
+   In the webwml.git repository, `include/versions.wmi` and `Makefile`.
+   In the project/web/tpo.git repository, update `databags/versions.ini`
+   to note the new version.  Push these changes to master.
 
    (NOTE: Due to #17805, there can only be one stable version listed at
    once.  Nonetheless, do not call your version "alpha" if it is stable,
    or people will get confused.)
 
+   (NOTE: It will take a while for the website update scripts to update
+   the website.)
+
 3. Email the packagers (cc'ing tor-team) that a new tarball is up.
    The current list of packagers is:
 
-       - {weasel,gk,mikeperry} at torproject dot org
+       - {weasel,sysrqb,mikeperry} at torproject dot org
        - {blueness} at gentoo dot org
        - {paul} at invizbox dot io
        - {vincent} at invizbox dot com
@@ -186,37 +206,46 @@ new Tor release:
 
    Also, email tor-packagers@lists.torproject.org.
 
+   Mention where to download the tarball (https://dist.torproject.org).
+
+   Include a link to the changelog.
+
 4. Add the version number to Trac.  To do this, go to Trac, log in,
     select "Admin" near the top of the screen, then select "Versions" from
     the menu on the left.  At the right, there will be an "Add version"
     box.  By convention, we enter the version in the form "Tor:
-    0.2.2.23-alpha" (or whatever the version is), and we select the date as
+    0.4.0.1-alpha" (or whatever the version is), and we select the date as
     the date in the ChangeLog.
 
-5. Double-check: did the version get recommended in the consensus yet?  Is
-   the website updated?  If not, don't announce until they have the
-   up-to-date versions, or people will get confused.
+5. Wait for the download page to be updated. (If you don't do this before you
+   announce, people will be confused.)
 
 6. Mail the release blurb and ChangeLog to tor-talk (development release) or
    tor-announce (stable).
 
    Post the changelog on the blog as well. You can generate a
-   blog-formatted version of the changelog with the -B option to
-   format-changelog.
+   blog-formatted version of the changelog with
+      `./scripts/maint/format_changelog.py --B`
 
    When you post, include an estimate of when the next TorBrowser
    releases will come out that include this Tor release.  This will
    usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it
    can vary.
 
+   For templates to use when announcing, see:
+       https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/AnnouncementTemplates
 
-=== V. Aftermath and cleanup
+## V. Aftermath and cleanup
 
 1. If it's a stable release, bump the version number in the
     `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours`
     merge to avoid taking that change into master.
 
-2. Forward-port the ChangeLog (and ReleaseNotes if appropriate).
+2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that
+   builds the release every week. (It's ok to skip the weekly build if the
+   branch was updated in the last 24 hours.)
 
-3. Keep an eye on the blog post, to moderate comments and answer questions.
+3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the
+   master branch.
 
+4. Keep an eye on the blog post, to moderate comments and answer questions.
diff --git a/doc/HACKING/Tracing.md b/doc/HACKING/Tracing.md
index 24fa761310..e1e97abe6d 100644
--- a/doc/HACKING/Tracing.md
+++ b/doc/HACKING/Tracing.md
@@ -1,16 +1,16 @@
-# Tracing #
+# Tracing
 
 This document describes how the event tracing subsystem works in tor so
 developers can add events to the code base but also hook them to an event
 tracing framework.
 
-## Basics ###
+## Basics
 
 Event tracing is separated in two concepts, trace events and a tracer. The
 tracing subsystem can be found in `src/trace`. The `events.h` header file is
 the main file that maps the different tracers to trace events.
 
-### Events ###
+### Events
 
 A trace event is basically a function from which we can pass any data that
 we want to collect. In addition, we specify a context for the event such as
@@ -39,7 +39,7 @@ How `argc` is collected or used has nothing to do with the instrumentation
 the trace events and collection framework (tracer) are decoupled. You _can_
 have trace events without a tracer.
 
-### Tracer ###
+### Tracer
 
 In `src/trace/events.h`, we map the `tor_trace()` function to the right
 tracer. A tracer support is only enabled at compile time. For instance, the
@@ -47,7 +47,7 @@ file `src/trace/debug.h` contains the mapping of the generic tracing function
 `tor_trace()` to the `log_debug()` function. More specialized function can be
 mapped depending on the tracepoint.
 
-## Build System ##
+## Build System
 
 This section describes how it is integrated into the build system of tor.
 
@@ -66,7 +66,7 @@ configure option:
 
 	--enable-tracing-debug
 
-## Instrument Tor ##
+## Instrument Tor
 
 This is pretty easy. Let's say you want to add a trace event in
 `src/feature/rend/rendcache.c`, you only have to add this include statement:
diff --git a/doc/HACKING/WritingTests.md b/doc/HACKING/WritingTests.md
index 05de8e0be8..d212020525 100644
--- a/doc/HACKING/WritingTests.md
+++ b/doc/HACKING/WritingTests.md
@@ -1,6 +1,4 @@
-
-Writing tests for Tor: an incomplete guide
-==========================================
+# Writing tests for Tor: an incomplete guide
 
 Tor uses a variety of testing frameworks and methodologies to try to
 keep from introducing bugs.  The major ones are:
@@ -19,8 +17,7 @@ keep from introducing bugs.  The major ones are:
 
    5. The Shadow network simulator.
 
-How to run these tests
-----------------------
+## How to run these tests
 
 ### The easy version
 
@@ -64,7 +61,7 @@ The former are those that should finish in a few seconds; the latter tend to
 take more time, and may include CPU-intensive operations, deliberate delays,
 and stuff like that.
 
-### Finding test coverage
+## Finding test coverage
 
 Test coverage is a measurement of which lines your tests actually visit.
 
@@ -112,7 +109,7 @@ To count new or modified uncovered lines in D2, you can run:
 
     ./scripts/test/cov-diff ${D1} ${D2}" | grep '^+ *\#' | wc -l
 
-### Marking lines as unreachable by tests
+## Marking lines as unreachable by tests
 
 You can mark a specific line as unreachable by using the special
 string LCOV_EXCL_LINE.  You can mark a range of lines as unreachable
@@ -126,9 +123,7 @@ unreached lines with 'x', and excluded reached lines with '!!!'.
 Note: you should never do this unless the line is meant to 100%
 unreachable by actual code.
 
-
-What kinds of test should I write?
-----------------------------------
+## What kinds of test should I write?
 
 Integration testing and unit testing are complementary: it's probably a
 good idea to make sure that your code is hit by both if you can.
@@ -143,8 +138,7 @@ If your code adds new externally visible functionality to Tor, it would
 be great to have a test for that functionality.  That's where
 integration tests more usually come in.
 
-Unit and regression tests: Does this function do what it's supposed to?
------------------------------------------------------------------------
+## Unit and regression tests: Does this function do what it's supposed to?
 
 Most of Tor's unit tests are made using the "tinytest" testing framework.
 You can see a guide to using it in the tinytest manual at
@@ -165,7 +159,7 @@ If you have created a new test file, you will need to:
 
 I use the term "unit test" and "regression tests" very sloppily here.
 
-### A simple example
+## A simple example
 
 Here's an example of a test function for a simple function in util.c:
 
@@ -207,7 +201,7 @@ Finally, remember that by convention, all `*_free()` functions that
 Tor defines are defined to accept NULL harmlessly.  Thus, you don't
 need to say `if (contents)` in the cleanup block.
 
-### Exposing static functions for testing
+## Exposing static functions for testing
 
 Sometimes you need to test a function, but you don't want to expose
 it outside its usual module.
@@ -228,7 +222,7 @@ For example, `crypto_curve25519.h` contains:
 The `crypto_curve25519.c` file and the `test_crypto.c` file both define
 `CRYPTO_CURVE25519_PRIVATE`, so they can see this declaration.
 
-### STOP!  Does this test really test?
+## STOP!  Does this test really test?
 
 When writing tests, it's not enough to just generate coverage on all the
 lines of the code that you're testing:  It's important to make sure that
@@ -269,8 +263,7 @@ it's supposed to do, and fail otherwise.  Try to design your tests so
 that they check for the code's intended and documented functionality
 as much as possible.
 
-
-### Mock functions for testing in isolation
+## Mock functions for testing in isolation
 
 Often we want to test that a function works right, but the function to
 be tested depends on other functions whose behavior is hard to observe,
@@ -311,7 +304,7 @@ And later, you can restore the original function with:
 For more information, see the definitions of this mocking logic in
 `testsupport.h`.
 
-### Okay but what should my tests actually do?
+## Okay but what should my tests actually do?
 
 We talk above about "test coverage" -- making sure that your tests visit
 every line of code, or every branch of code.  But visiting the code isn't
@@ -382,8 +375,7 @@ Based on the implementation, we now see three more edge cases to test:
    * Removing an element from the end of the list
    * Removing an element from a position other than the end of the list.
 
-
-### What should my tests NOT do?
+## What should my tests NOT do?
 
 Tests shouldn't require a network connection.
 
@@ -401,8 +393,7 @@ When possible, tests should not be over-fit to the implementation.  That is,
 the test should verify that the documented behavior is implemented, but
 should not break if other permissible behavior is later implemented.
 
-
-### Advanced techniques: Namespaces
+## Advanced techniques: Namespaces
 
 Sometimes, when you're doing a lot of mocking at once, it's convenient to
 isolate your identifiers within a single namespace.  If this were C++, we'd
@@ -414,9 +405,7 @@ them, you define `NS_MODULE` to a prefix to be used for your identifiers, and
 then use other macros in place of identifier names.  See `src/test/test.h` for
 more documentation.
 
-
-Integration tests: Calling Tor from the outside
------------------------------------------------
+## Integration tests: Calling Tor from the outside
 
 Some tests need to invoke Tor from the outside, and shouldn't run from the
 same process as the Tor test program.  Reasons for doing this might include:
@@ -436,8 +425,7 @@ wrapped, add a new shell script to `TESTS`, and the new program to
 makefile (eg `${PYTHON}` for a python interpreter), then make sure that the
 makefile exports them.
 
-Writing integration tests with Stem
------------------------------------
+## Writing integration tests with Stem
 
 The 'stem' library includes extensive tests for the Tor controller protocol.
 You can run stem tests from tor with `make test-stem`, or see
@@ -483,8 +471,7 @@ you notice any strange behaviour that seems totally unreasonable.
 Check out the `test_exit_policy()` function in abovementioned file to see the
 final implementation for this test.
 
-System testing with Chutney
----------------------------
+## System testing with Chutney
 
 The 'chutney' program configures and launches a set of Tor relays,
 authorities, and clients on your local host.  It has a `test network`
diff --git a/doc/HACKING/android/Simpleperf.md b/doc/HACKING/android/Simpleperf.md
index 25f39a3d23..c7e63a7c86 100644
--- a/doc/HACKING/android/Simpleperf.md
+++ b/doc/HACKING/android/Simpleperf.md
@@ -95,4 +95,3 @@ was spend on the call.
   Start Tor the normal way via Orbot and collect the logs from your computer using
 
       $ adb logcat
-
diff --git a/doc/asciidoc-helper.sh b/doc/asciidoc-helper.sh
index a3ef53f884..765850a125 100755
--- a/doc/asciidoc-helper.sh
+++ b/doc/asciidoc-helper.sh
@@ -19,7 +19,7 @@ if [ "$1" = "html" ]; then
     base=${output%%.html.in}
 
     if [ "$2" != none ]; then
-      TZ=UTC "$2" -d manpage -o $output $input;
+      TZ=UTC "$2" -d manpage -o "$output" "$input";
     else
       echo "==================================";
       echo;
@@ -44,8 +44,8 @@ elif [ "$1" = "man" ]; then
       echo "==================================";
       exit 1;
     fi
-    if "$2" -f manpage $input; then
-      mv $base.1 $output;
+    if "$2" -f manpage "$input"; then
+      mv "$base.1" "$output";
     else
       cat<<EOF
 ==================================
diff --git a/doc/building-tor-msvc.txt b/doc/building-tor-msvc.txt
index 3d3eced8af..dbc644d172 100644
--- a/doc/building-tor-msvc.txt
+++ b/doc/building-tor-msvc.txt
@@ -11,15 +11,15 @@ Requirements:
 -------------

 

  * Visual Studio 2010

-    http://go.microsoft.com/fwlink/?LinkId=323467

+    https://go.microsoft.com/fwlink/?LinkId=323467

  * CMake 2.8.12.2

-    http://www.cmake.org/download/

+    https://www.cmake.org/download/

  * Perl 5.16

-    http://www.activestate.com/activeperl/downloads

+    https://www.activestate.com/activeperl/downloads

  * Latest stable OpenSSL tarball

     https://www.openssl.org/source/

  * Latest stable zlib tarball

-    http://zlib.net/

+    https://zlib.net/

  * Latest stable libevent Libevent tarball

     https://github.com/libevent/libevent/releases

 

diff --git a/doc/contrib/tor-rpm-creation.txt b/doc/contrib/tor-rpm-creation.txt
index 9c4e05764e..f4090aa874 100644
--- a/doc/contrib/tor-rpm-creation.txt
+++ b/doc/contrib/tor-rpm-creation.txt
@@ -4,7 +4,7 @@ The process used to create the official rpms is as follows:
 
 You'll need to install libevent headers, usually located in package named
 libevent-devel. Alternatively, you could download latest libevent from
-http://libevent.org/ but that shouldn't be necessary.
+https://libevent.org/ but that shouldn't be necessary.
 
 Download and Extract the latest tor source code from
 https://www.torproject.org/download
diff --git a/doc/include.am b/doc/include.am
index 0a123aae11..8651f845eb 100644
--- a/doc/include.am
+++ b/doc/include.am
@@ -15,23 +15,39 @@
 all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify doc/tor-print-ed-signing-cert
 
 if USE_ASCIIDOC
-nodist_man1_MANS = $(all_mans:=.1)
-doc_DATA = $(all_mans:=.html)
+txt_in = $(all_mans:=.1.txt)
+
+if BUILD_HTML_DOCS
 html_in = $(all_mans:=.html.in)
+doc_DATA = $(all_mans:=.html)
+else
+html_in =
+doc_DATA =
+endif
+
+if BUILD_MANPAGE
+nodist_man1_MANS = $(all_mans:=.1)
 man_in = $(all_mans:=.1.in)
-txt_in = $(all_mans:=.1.txt)
 else
+nodist_man1_MANS =
+man_in =
+endif
+
+else
+
 html_in =
+doc_DATA =
 man_in =
 txt_in =
 nodist_man1_MANS =
-doc_DATA =
+
 endif
 
 EXTRA_DIST+= doc/asciidoc-helper.sh			\
 	     $(html_in) $(man_in) $(txt_in)			\
 	     doc/state-contents.txt				\
 	     doc/torrc_format.txt 				\
+	     doc/tor-doxygen.css 				\
 	     doc/TUNING						\
 	     doc/HACKING/README.1st.md				\
 	     doc/HACKING/CodingStandards.md 			\
diff --git a/doc/tor-doxygen.css b/doc/tor-doxygen.css
new file mode 100644
index 0000000000..97cd1886db
--- /dev/null
+++ b/doc/tor-doxygen.css
@@ -0,0 +1,10 @@
+
+p.definition {
+    font-size: small;
+    padding-left: 1.5em;
+}
+
+p.reference {
+    font-size: small;
+    padding-left: 1.5em;
+}
diff --git a/doc/tor-gencert.1.txt b/doc/tor-gencert.1.txt
index 6bba548b87..26f68b29c0 100644
--- a/doc/tor-gencert.1.txt
+++ b/doc/tor-gencert.1.txt
@@ -1,7 +1,7 @@
 // Copyright (c) The Tor Project, Inc.
 // See LICENSE for licensing information
 // This is an asciidoc file used to generate the manpage/html reference.
-// Learn asciidoc on http://www.methods.co.nz/asciidoc/userguide.html
+// Learn asciidoc on https://www.methods.co.nz/asciidoc/userguide.html
 :man source:   Tor
 :man manual:   Tor Manual
 tor-gencert(1)
diff --git a/doc/tor-print-ed-signing-cert.1.txt b/doc/tor-print-ed-signing-cert.1.txt
index 1a3109df95..71c8b67ec4 100644
--- a/doc/tor-print-ed-signing-cert.1.txt
+++ b/doc/tor-print-ed-signing-cert.1.txt
@@ -1,7 +1,7 @@
 // Copyright (c) The Tor Project, Inc.
 // See LICENSE for licensing information
 // This is an asciidoc file used to generate the manpage/html reference.
-// Learn asciidoc on http://www.methods.co.nz/asciidoc/userguide.html
+// Learn asciidoc on https://www.methods.co.nz/asciidoc/userguide.html
 :man source:   Tor
 :man manual:   Tor Manual
 tor-print-ed-signing-cert(1)
@@ -21,6 +21,12 @@ DESCRIPTION
 **tor-print-ed-signing-cert** is utility program for Tor relay operators to
 check expiration date of ed25519 signing certificate.
 
+Expiration date is printed in three formats:
+
+* Human-readable timestamp, localized to current timezone.
+* RFC 1123 timestamp (at GMT timezone).
+* Unix time value.
+
 SEE ALSO
 --------
 **tor**(1) +
diff --git a/doc/tor-resolve.1.txt b/doc/tor-resolve.1.txt
index f1f8f77a42..17a77e482f 100644
--- a/doc/tor-resolve.1.txt
+++ b/doc/tor-resolve.1.txt
@@ -1,7 +1,7 @@
 // Copyright (c) The Tor Project, Inc.
 // See LICENSE for licensing information
 // This is an asciidoc file used to generate the manpage/html reference.
-// Learn asciidoc on http://www.methods.co.nz/asciidoc/userguide.html
+// Learn asciidoc on https://www.methods.co.nz/asciidoc/userguide.html
 :man source:   Tor
 :man manual:   Tor Manual
 tor-resolve(1)
diff --git a/doc/tor.1.txt b/doc/tor.1.txt
index 21b482802e..3dc8c7a034 100644
--- a/doc/tor.1.txt
+++ b/doc/tor.1.txt
@@ -1,162 +1,200 @@
 // Copyright (c) The Tor Project, Inc.
 // See LICENSE for licensing information
 // This is an asciidoc file used to generate the manpage/html reference.
-// Learn asciidoc on http://www.methods.co.nz/asciidoc/userguide.html
+// Learn asciidoc on https://www.methods.co.nz/asciidoc/userguide.html
 :man source:   Tor
 :man manual:   Tor Manual
-TOR(1)
-======
+// compat-mode tells Asciidoctor tools to process this as legacy AsciiDoc
+:compat-mode:
+// attribute to make it easier to write names containing double underscores
+:dbl_: __
+= TOR(1)
+:toc:
+
+== NAME
 
-NAME
-----
 tor - The second-generation onion router
 
+== SYNOPSIS
 
-SYNOPSIS
---------
 **tor** [__OPTION__ __value__]...
 
-DESCRIPTION
------------
-Tor is a connection-oriented anonymizing communication
-service. Users choose a source-routed path through a set of nodes, and
-negotiate a "virtual circuit" through the network, in which each node
-knows its predecessor and successor, but no others. Traffic flowing down
-the circuit is unwrapped by a symmetric key at each node, which reveals
-the downstream node. +
-
-Basically, Tor provides a distributed network of servers or relays ("onion routers").
-Users bounce their TCP streams -- web traffic, ftp, ssh, etc. -- around the
-network, and recipients, observers, and even the relays themselves have
-difficulty tracking the source of the stream.
-
-By default, **tor** will act as a client only.  To help the network
-by providing bandwidth as a relay, change the **ORPort** configuration
-option -- see below.  Please also consult the documentation on the Tor
-Project's website.
-
-COMMAND-LINE OPTIONS
---------------------
-[[opt-h]] **-h**, **-help**::
+== DESCRIPTION
+
+Tor is a connection-oriented anonymizing communication service. Users
+choose a source-routed path through a set of nodes, and negotiate a
+"virtual circuit" through the network. Each node in a virtual circuit
+knows its predecessor and successor nodes, but no other nodes. Traffic
+flowing down the circuit is unwrapped by a symmetric key at each node,
+which reveals the downstream node. +
+
+Basically, Tor provides a distributed network of servers or relays
+("onion routers").  Users bounce their TCP streams, including web
+traffic, ftp, ssh, etc., around the network, so that recipients,
+observers, and even the relays themselves have difficulty tracking the
+source of the stream.
+
+[NOTE]
+By default, **tor** acts as a client only.  To help the network by
+providing bandwidth as a relay, change the **ORPort** configuration
+option as mentioned below.  Please also consult the documentation on
+the Tor Project's website.
+
+== COMMAND-LINE OPTIONS
+
+Tor has a powerful command-line interface. This section lists optional
+arguments you can specify at the command line using the **`tor`**
+command.
+
+Configuration options can be specified on the command line in the
+format **`--`**_OptionName_ _OptionValue_, on the command line in the
+format _OptionName_ _OptionValue_, or in a configuration file.  For
+instance, you can tell Tor to start listening for SOCKS connections on
+port 9999 by passing either **`--SocksPort 9999`** or **`SocksPort
+9999`** on the command line, or by specifying **`SocksPort 9999`** in
+the configuration file.  On the command line, quote option values that
+contain spaces. For instance, if you want Tor to log all debugging
+messages to **`debug.log`**, you must specify **`--Log "debug file
+debug.log"`**.
+
+NOTE: Configuration options on the command line override those in
+configuration files.  See **<<conf-format,THE CONFIGURATION FILE
+FORMAT>>** for more information.
+
+The following options in this section are only recognized on the
+**`tor`** command line, not in a configuration file.
+
+[[opt-h]] **`-h`**, **`--help`**::
     Display a short help message and exit.
 
-[[opt-f]] **-f** __FILE__::
+[[opt-f]] **`-f`** __FILE__::
     Specify a new configuration file to contain further Tor configuration
-    options OR pass *-* to make Tor read its configuration from standard
-    input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not
-    found)
+    options, or pass *-* to make Tor read its configuration from standard
+    input. (Default: **`@CONFDIR@/torrc`**, or **`$HOME/.torrc`** if
+    that file is not found)
 
-[[opt-allow-missing-torrc]] **--allow-missing-torrc**::
-    Do not require that configuration file specified by **-f** exist if
-    default torrc can be accessed.
+[[opt-allow-missing-torrc]] **`--allow-missing-torrc`**::
+    Allow the configuration file specified by **`-f`** to be missing,
+    if the defaults-torrc file (see below) is accessible.
 
-[[opt-defaults-torrc]] **--defaults-torrc** __FILE__::
+[[opt-defaults-torrc]] **`--defaults-torrc`** __FILE__::
     Specify a file in which to find default values for Tor options.  The
     contents of this file are overridden by those in the regular
     configuration file, and by those on the command line. (Default:
-    @CONFDIR@/torrc-defaults.)
+    **`@CONFDIR@/torrc-defaults`**.)
 
-[[opt-ignore-missing-torrc]] **--ignore-missing-torrc**::
-    Specifies that Tor should treat a missing torrc file as though it
+[[opt-ignore-missing-torrc]] **`--ignore-missing-torrc`**::
+    Specify that Tor should treat a missing torrc file as though it
     were empty. Ordinarily, Tor does this for missing default torrc files,
     but not for those specified on the command line.
 
-[[opt-hash-password]] **--hash-password** __PASSWORD__::
-    Generates a hashed password for control port access.
+[[opt-hash-password]] **`--hash-password`** __PASSWORD__::
+    Generate a hashed password for control port access.
 
-[[opt-list-fingerprint]] **--list-fingerprint**::
+[[opt-list-fingerprint]] **`--list-fingerprint`**::
     Generate your keys and output your nickname and fingerprint.
 
-[[opt-verify-config]] **--verify-config**::
-    Verify the configuration file is valid.
+[[opt-verify-config]] **`--verify-config`**::
+    Verify whether the configuration file is valid.
+
+[[opt-dump-config]] **`--dump-config`** **`short`**|**`full`**|**`non-builtin`**::
+    Write a complete list of Tor's configured options to standard output.
+    When the `short` flag is selected, only write the options that
+    are different from their default values.  When `non-builtin` is selected,
+    write options that are not zero or the empty string.
+    When `full` is selected, write every option.
 
-[[opt-serviceinstall]] **--service install** [**--options** __command-line options__]::
+[[opt-serviceinstall]] **`--service install`** [**`--options`** __command-line options__]::
     Install an instance of Tor as a Windows service, with the provided
     command-line options. Current instructions can be found at
     https://www.torproject.org/docs/faq#NTService
 
-[[opt-service]] **--service** **remove**|**start**|**stop**::
+[[opt-service]] **`--service`** **`remove`**|**`start`**|**`stop`**::
     Remove, start, or stop a configured Tor Windows service.
 
-[[opt-nt-service]] **--nt-service**::
+[[opt-nt-service]] **`--nt-service`**::
     Used internally to implement a Windows service.
 
-[[opt-list-torrc-options]] **--list-torrc-options**::
+[[opt-list-torrc-options]] **`--list-torrc-options`**::
     List all valid options.
 
-[[opt-list-deprecated-options]] **--list-deprecated-options**::
+[[opt-list-deprecated-options]] **`--list-deprecated-options`**::
     List all valid options that are scheduled to become obsolete in a
     future version. (This is a warning, not a promise.)
 
-[[opt-list-modules]] **--list-modules**::
-    For each optional module, list whether or not it has been compiled
-    into Tor.  (Any module not listed is not optional in this version of Tor.)
-
-[[opt-version]] **--version**::
-    Display Tor version and exit.
-
-[[opt-quiet]] **--quiet**|**--hush**::
-    Override the default console log.  By default, Tor starts out logging
-    messages at level "notice" and higher to the console.  It stops doing so
-    after it parses its configuration, if the configuration tells it to log
-    anywhere else.  You can override this behavior with the **--hush** option,
-    which tells Tor to only send warnings and errors to the console, or with
-    the **--quiet** option, which tells Tor not to log to the console at all.
-
-[[opt-keygen]] **--keygen** [**--newpass**]::
-   Running "tor --keygen" creates a new ed25519 master identity key for a
-   relay, or only a fresh temporary signing key and certificate, if you
-   already have a master key.  Optionally you can encrypt the master identity
-   key with a passphrase: Tor will ask you for one. If you don't want to
-   encrypt the master key, just don't enter any passphrase when asked. +
- +
-   The **--newpass** option should be used with --keygen only when you need
-   to add, change, or remove a passphrase on an existing ed25519 master
-   identity key. You will be prompted for the old passphase (if any),
-   and the new passphrase (if any). +
- +
-   When generating a master key, you will probably want to use
-   **--DataDirectory** to control where the keys
-   and certificates will be stored, and **--SigningKeyLifetime** to
-   control their lifetimes.  Their behavior is as documented in the
-   server options section below.  (You must have write access to the specified
-   DataDirectory.) +
- +
-   To use the generated files, you must copy them to the DataDirectory/keys
-   directory of your Tor daemon, and make sure that they are owned by the
-   user actually running the Tor daemon on your system.
-
-**--passphrase-fd** __FILEDES__::
-    Filedescriptor to read the passphrase from.  Note that unlike with the
+[[opt-list-modules]] **`--list-modules`**::
+    List whether each optional module has been compiled into Tor.
+    (Any module not listed is not optional in this version of Tor.)
+
+[[opt-version]] **`--version`**::
+    Display Tor version and exit. The output is a single line of the format
+    "Tor version [version number]."  (The version number format
+    is as specified in version-spec.txt.)
+
+[[opt-quiet]] **`--quiet`**|**`--hush`**::
+    Override the default console logging behavior.  By default, Tor
+    starts out logging messages at level "notice" and higher to the
+    console.  It stops doing so after it parses its configuration, if
+    the configuration tells it to log anywhere else.  These options
+    override the default console logging behavior.  Use the
+    **`--hush`** option if you want Tor to log only warnings and
+    errors to the console, or use the **`--quiet`** option if you want
+    Tor not to log to the console at all.
+
+[[opt-keygen]] **`--keygen`** [**`--newpass`**]::
+    Running **`tor --keygen`** creates a new ed25519 master identity key
+    for a relay, or only a fresh temporary signing key and
+    certificate, if you already have a master key.  Optionally, you
+    can encrypt the master identity key with a passphrase.  When Tor
+    asks you for a passphrase and you don't want to encrypt the master
+    key, just don't enter any passphrase when asked. +
+     +
+    Use the **`--newpass`** option with **`--keygen`** only when you
+    need to add, change, or remove a passphrase on an existing ed25519
+    master identity key. You will be prompted for the old passphase
+    (if any), and the new passphrase (if any).
++
+[NOTE]
+    When generating a master key, you may want to use
+    **`--DataDirectory`** to control where the keys and certificates
+    will be stored, and **`--SigningKeyLifetime`** to control their
+    lifetimes.  See <<server-options,SERVER OPTIONS>> to learn more about the
+    behavior of these options.  You must have write access to the
+    specified DataDirectory.
++
+[normal]
+    To use the generated files, you must copy them to the
+    __DataDirectory__/**`keys`** directory of your Tor daemon, and
+    make sure that they are owned by the user actually running the Tor
+    daemon on your system.
+
+**`--passphrase-fd`** __FILEDES__::
+    File descriptor to read the passphrase from.  Note that unlike with the
     tor-gencert program, the entire file contents are read and used as
     the passphrase, including any trailing newlines.
-    Default: read from the terminal.
+    If the file descriptor is not specified, the passphrase is read
+    from the terminal by default.
 
-[[opt-key-expiration]] **--key-expiration** [**purpose**]::
-    The **purpose** specifies which type of key certificate to determine
-    the expiration of.  The only currently recognised **purpose** is
+[[opt-key-expiration]] **`--key-expiration`** [__purpose__]::
+    The __purpose__ specifies which type of key certificate to determine
+    the expiration of.  The only currently recognised __purpose__ is
     "sign". +
- +
-    Running "tor --key-expiration sign" will attempt to find your signing
-    key certificate and will output, both in the logs as well as to stdout,
-    the signing key certificate's expiration time in ISO-8601 format.
-    For example, the output sent to stdout will be of the form:
-    "signing-cert-expiry: 2017-07-25 08:30:15 UTC"
-
-Other options can be specified on the command-line in the format "--option
-value", in the format "option value", or in a configuration file.  For
-instance, you can tell Tor to start listening for SOCKS connections on port
-9999 by passing --SocksPort 9999 or SocksPort 9999 to it on the command line,
-or by putting "SocksPort 9999" in the configuration file.  You will need to
-quote options with spaces in them: if you want Tor to log all debugging
-messages to debug.log, you will probably need to say --Log 'debug file
-debug.log'.
-
-Options on the command line override those in configuration files. See the
-next section for more information.
-
-THE CONFIGURATION FILE FORMAT
------------------------------
+     +
+    Running **`tor --key-expiration sign`** will attempt to find your
+    signing key certificate and will output, both in the logs as well
+    as to stdout, the signing key certificate's expiration time in
+    ISO-8601 format.  For example, the output sent to stdout will be
+    of the form: "signing-cert-expiry: 2017-07-25 08:30:15 UTC"
+
+[[opt-dbg]] **--dbg-**...::
+    Tor may support other options beginning with the string "dbg". These
+    are intended for use by developers to debug and test Tor. They are
+    not supported or guaranteed to be stable, and you should probably
+    not use them.
+
+
+[[conf-format]]
+== THE CONFIGURATION FILE FORMAT
 
 All configuration options in a configuration are written on a single line by
 default.  They take the form of an option name and a value, or an option name
@@ -173,6 +211,8 @@ file will be parsed as if they were written where the %include option is. If
 the path is a folder, all files on that folder will be parsed following lexical
 order. Files starting with a dot are ignored. Files on subfolders are ignored.
 The %include option can be used recursively.
+New configuration files or directories cannot be added to already running Tor
+instance if **Sandbox** is enabled.
 
 By default, an option on the command line overrides an option found in the
 configuration file, and an option in a configuration file overrides one in
@@ -194,8 +234,48 @@ command line that you want no SocksPorts at all.  To do that, prefix the
 option name with a forward slash (/).  You can use the plus sign (+) and the
 forward slash (/) in the configuration file and on the command line.
 
-GENERAL OPTIONS
----------------
+== GENERAL OPTIONS
+
+// These options are in alphabetical order, with exceptions as noted.
+// Please keep them that way!
+
+[[AccelDir]] **AccelDir** __DIR__::
+    Specify this option if using dynamic hardware acceleration and the engine
+    implementation library resides somewhere other than the OpenSSL default.
+    Can not be changed while tor is running.
+
+[[AccelName]] **AccelName** __NAME__::
+    When using OpenSSL hardware crypto acceleration attempt to load the dynamic
+    engine of this name. This must be used for any dynamic hardware engine.
+    Names can be verified with the openssl engine command. Can not be changed
+    while tor is running. +
+     +
+    If the engine name is prefixed with a "!", then Tor will exit if the
+    engine cannot be loaded.
+
+[[AlternateBridgeAuthority]] **AlternateBridgeAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __ fingerprint__::
+[[AlternateDirAuthority]] **AlternateDirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__::
+    These options behave as DirAuthority, but they replace fewer of the
+    default directory authorities. Using
+    AlternateDirAuthority replaces the default Tor directory authorities, but
+    leaves the default bridge authorities in
+    place.  Similarly,
+    AlternateBridgeAuthority replaces the default bridge authority,
+    but leaves the directory authorities alone.
+
+[[AndroidIdentityTag]] **AndroidIdentityTag** __tag__::
+    When logging to Android's logging subsystem, adds a tag to the log identity
+    such that log entries are marked with "Tor-__tag__". Can not be changed while
+    tor is running. (Default: none)
+
+[[AvoidDiskWrites]] **AvoidDiskWrites** **0**|**1**::
+    If non-zero, try to write to disk less frequently than we would otherwise.
+    This is useful when running on flash memory or other media that support
+    only a limited number of writes. (Default: 0)
+
+[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    Limit the maximum token bucket size (also known as the burst) to the given
+    number of bytes in each direction. (Default: 1 GByte)
 
 [[BandwidthRate]] **BandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     A token bucket limits the average incoming bandwidth usage on this node
@@ -205,55 +285,45 @@ GENERAL OPTIONS
     relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) -- but of
     course, more is better; we recommend at least 250 KBytes (2 mbits) if
     possible.  (Default: 1 GByte) +
- +
+     +
     Note that this option, and other bandwidth-limiting options, apply to TCP
     data only: They do not count TCP headers or DNS traffic. +
- +
+     +
+    Tor uses powers of two, not powers of ten, so 1 GByte is
+    1024*1024*1024 bytes as opposed to 1 billion bytes. +
+     +
     With this option, and in other options that take arguments in bytes,
     KBytes, and so on, other formats are also supported. Notably, "KBytes" can
     also be written as "kilobytes" or "kb"; "MBytes" can be written as
     "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth.
+    Case doesn't matter.
     Tor also accepts "byte" and "bit" in the singular.
     The prefixes "tera" and "T" are also recognized.
     If no units are given, we default to bytes.
     To avoid confusion, we recommend writing "bytes" or "bits" explicitly,
     since it's easy to forget that "B" means bytes, not bits.
 
-[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    Limit the maximum token bucket size (also known as the burst) to the given
-    number of bytes in each direction. (Default: 1 GByte)
-
-[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    If set, we will not advertise more than this amount of bandwidth for our
-    BandwidthRate. Server operators who want to reduce the number of clients
-    who ask to build circuits through them (since this is proportional to
-    advertised bandwidth rate) can thus reduce the CPU demands on their server
-    without impacting network performance.
-
-[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    If not 0, a separate token bucket limits the average incoming bandwidth
-    usage for \_relayed traffic_ on this node to the specified number of bytes
-    per second, and the average outgoing bandwidth usage to that same value.
-    Relayed traffic currently is calculated to include answers to directory
-    requests, but that may change in future versions. They do not include directory
-    fetches by the relay (from authority or other relays), because that is considered
-    "client" activity.  (Default: 0)
-
-[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    If not 0, limit the maximum token bucket size (also known as the burst) for
-    \_relayed traffic_ to the given number of bytes in each direction.
-    They do not include directory fetches by the relay (from authority
-    or other relays), because that is considered "client" activity. (Default: 0)
+[[CacheDirectory]] **CacheDirectory** __DIR__::
+    Store cached directory data in DIR. Can not be changed while tor is
+    running.
+    (Default: uses the value of DataDirectory.)
 
-[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    If this option is set manually, or via the "perconnbwrate" consensus
-    field, Tor will use it for separate rate limiting for each connection
-    from a non-relay. (Default: 0)
+[[CacheDirectoryGroupReadable]] **CacheDirectoryGroupReadable** **0**|**1**|**auto**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    CacheDirectory. If the option is set to 1, make the CacheDirectory readable
+    by the default GID. If the option is "auto", then we use the
+    setting for DataDirectoryGroupReadable when the CacheDirectory is the
+    same as the DataDirectory, and 0 otherwise. (Default: auto)
 
-[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    If this option is set manually, or via the "perconnbwburst" consensus
-    field, Tor will use it for separate rate limiting for each connection
-    from a non-relay. (Default: 0)
+[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM__::
+    If this value is set, we override the default algorithm for choosing which
+    circuit's cell to deliver or relay next. It is delivered first to the
+    circuit that has the lowest weighted cell count, where cells are weighted
+    exponentially according to this value (in seconds). If the value is -1, it
+    is taken from the consensus if possible else it will fallback to the
+    default value of 30. Minimum: 1, Maximum: 2147483647. This can be defined
+    as a float value. This is an advanced option; you generally shouldn't have
+    to mess with it. (Default: -1)
 
 [[ClientTransportPlugin]] **ClientTransportPlugin** __transport__ socks4|socks5 __IP__:__PORT__::
 **ClientTransportPlugin** __transport__ exec __path-to-binary__ [options]::
@@ -262,70 +332,29 @@ GENERAL OPTIONS
     (IPv4 addresses should written as-is; IPv6 addresses should be wrapped in
     square brackets.) It's the
     duty of that proxy to properly forward the traffic to the bridge. +
- +
+     +
     In its second form, when set along with a corresponding Bridge line, the Tor
     client launches the pluggable transport proxy executable in
     __path-to-binary__ using __options__ as its command-line options, and
     forwards its traffic to it. It's the duty of that proxy to properly forward
-    the traffic to the bridge.
-
-[[ServerTransportPlugin]] **ServerTransportPlugin** __transport__ exec __path-to-binary__ [options]::
-    The Tor relay launches the pluggable transport proxy in __path-to-binary__
-    using __options__ as its command-line options, and expects to receive
-    proxied client traffic from it.
-
-[[ServerTransportListenAddr]] **ServerTransportListenAddr** __transport__ __IP__:__PORT__::
-    When this option is set, Tor will suggest __IP__:__PORT__ as the
-    listening address of any pluggable transport proxy that tries to
-    launch __transport__. (IPv4 addresses should written as-is; IPv6
-    addresses should be wrapped in square brackets.)
-
-[[ServerTransportOptions]] **ServerTransportOptions** __transport__ __k=v__ __k=v__ ...::
-    When this option is set, Tor will pass the __k=v__ parameters to
-    any pluggable transport proxy that tries to launch __transport__. +
-    (Example: ServerTransportOptions obfs45 shared-secret=bridgepasswd cache=/var/lib/tor/cache)
-
-[[ExtORPort]] **ExtORPort** \['address':]__port__|**auto**::
-    Open this port to listen for Extended ORPort connections from your
-    pluggable transports.
-
-[[ExtORPortCookieAuthFile]] **ExtORPortCookieAuthFile** __Path__::
-    If set, this option overrides the default location and file name
-    for the Extended ORPort's cookie file -- the cookie file is needed
-    for pluggable transports to communicate through the Extended ORPort.
-
-[[ExtORPortCookieAuthFileGroupReadable]] **ExtORPortCookieAuthFileGroupReadable** **0**|**1**::
-    If this option is set to 0, don't allow the filesystem group to read the
-    Extended OR Port cookie file. If the option is set to 1, make the cookie
-    file readable by the default GID. [Making the file readable by other
-    groups is not yet implemented; let us know if you need this for some
-    reason.] (Default: 0)
+    the traffic to the bridge. (Default: none)
 
 [[ConnLimit]] **ConnLimit** __NUM__::
     The minimum number of file descriptors that must be available to the Tor
     process before it will start. Tor will ask the OS for as many file
     descriptors as the OS will allow (you can find this by "ulimit -H -n").
     If this number is less than ConnLimit, then Tor will refuse to start. +
- +
+     +
     Tor relays need thousands of sockets, to connect to every other relay.
     If you are running a private bridge, you can reduce the number of sockets
     that Tor uses. For example, to limit Tor to 500 sockets, run
     "ulimit -n 500" in a shell. Then start tor in the same shell, with
     **ConnLimit 500**. You may also need to set **DisableOOSCheck 0**. +
- +
+     +
     Unless you have severely limited sockets, you probably don't need to
     adjust **ConnLimit** itself. It has no effect on Windows, since that
     platform lacks getrlimit(). (Default: 1000)
 
-[[DisableNetwork]] **DisableNetwork** **0**|**1**::
-    When this option is set, we don't listen for or accept any connections
-    other than controller connections, and we close (and don't reattempt)
-    any outbound
-    connections.  Controllers sometimes use this option to avoid using
-    the network until Tor is fully configured.  Tor will make still certain
-    network-related calls (like DNS lookups) as a part of its configuration
-    process, even if DisableNetwork is set. (Default: 0)
-
 [[ConstrainedSockets]] **ConstrainedSockets** **0**|**1**::
     If set, Tor will tell the kernel to attempt to shrink the buffers for all
     sockets to the size specified in **ConstrainedSockSize**. This is useful for
@@ -333,15 +362,15 @@ GENERAL OPTIONS
     be limited. If you're on a virtual server, and you encounter the "Error
     creating network socket: No buffer space available" message, you are
     likely experiencing this problem. +
- +
+     +
     The preferred solution is to have the admin increase the buffer pool for
     the host itself via /proc/sys/net/ipv4/tcp_mem or equivalent facility;
     this configuration option is a second-resort. +
- +
+     +
     The DirPort option should also not be used if TCP buffers are scarce. The
     cached directory requests consume additional sockets which exacerbates
     the problem. +
- +
+     +
     You should **not** enable this feature unless you encounter the "no buffer
     space available" issue. Reducing the TCP buffers affects window size for
     the TCP stream and will reduce throughput in proportion to round trip
@@ -352,7 +381,7 @@ GENERAL OPTIONS
     all sockets will be set to this limit. Must be a value between 2048 and
     262144, in 1024 byte increments. Default of 8192 is recommended.
 
-[[ControlPort]] **ControlPort** __PORT__|**unix:**__path__|**auto** [__flags__]::
+[[ControlPort]] **ControlPort** ['address'**:**]{empty}__port__|**unix:**__path__|**auto** [__flags__]::
     If set, Tor will accept connections on this port and allow those
     connections to control the Tor process using the Tor Control Protocol
     (described in control-spec.txt in
@@ -363,10 +392,11 @@ GENERAL OPTIONS
     methods means either method is sufficient to authenticate to Tor.) This
     option is required for many Tor controllers; most use the value of 9051.
     If a unix domain socket is used, you may quote the path using standard
-    C escape sequences.
+    C escape sequences. You can specify this directive multiple times, to
+    bind to multiple address/port pairs.
     Set it to "auto" to have Tor pick a port for you. (Default: 0) +
- +
-    Recognized flags are...
+     +
+    Recognized flags are:
     **GroupWritable**;;
         Unix domain sockets only: makes the socket get created as
         group-writable.
@@ -377,6 +407,16 @@ GENERAL OPTIONS
         Unix domain sockets only: Do not insist that the directory
         that holds the socket be read-restricted.
 
+[[ControlPortFileGroupReadable]] **ControlPortFileGroupReadable** **0**|**1**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    control port file. If the option is set to 1, make the control port
+    file readable by the default GID. (Default: 0)
+
+[[ControlPortWriteToFile]] **ControlPortWriteToFile** __Path__::
+    If set, Tor writes the address and port of any control port it opens to
+    this address.  Usable by controllers to learn the actual control port
+    when ControlPort is set to "auto".
+
 [[ControlSocket]] **ControlSocket** __Path__::
     Like ControlPort, but listens on a Unix domain socket, rather than a TCP
     socket. '0' disables ControlSocket. (Unix and Unix-like systems only.)
@@ -387,13 +427,6 @@ GENERAL OPTIONS
     write unix sockets (e.g. ControlSocket). If the option is set to 1, make
     the control socket readable and writable by the default GID. (Default: 0)
 
-[[HashedControlPassword]] **HashedControlPassword** __hashed_password__::
-    Allow connections on the control port if they present
-    the password whose one-way hash is __hashed_password__. You
-    can compute the hash of a password by running "tor --hash-password
-    __password__". You can provide several acceptable passwords by using more
-    than one HashedControlPassword line.
-
 [[CookieAuthentication]] **CookieAuthentication** **0**|**1**::
     If this option is set to 1, allow connections on the control port
     when the connecting process knows the contents of a file named
@@ -403,7 +436,7 @@ GENERAL OPTIONS
 
 [[CookieAuthFile]] **CookieAuthFile** __Path__::
     If set, this option overrides the default location and file name
-    for Tor's cookie file. (See CookieAuthentication above.)
+    for Tor's cookie file. (See <<CookieAuthentication,CookieAuthentication>>.)
 
 [[CookieAuthFileGroupReadable]] **CookieAuthFileGroupReadable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read the
@@ -411,15 +444,11 @@ GENERAL OPTIONS
     the default GID. [Making the file readable by other groups is not yet
     implemented; let us know if you need this for some reason.] (Default: 0)
 
-[[ControlPortWriteToFile]] **ControlPortWriteToFile** __Path__::
-    If set, Tor writes the address and port of any control port it opens to
-    this address.  Usable by controllers to learn the actual control port
-    when ControlPort is set to "auto".
-
-[[ControlPortFileGroupReadable]] **ControlPortFileGroupReadable** **0**|**1**::
-    If this option is set to 0, don't allow the filesystem group to read the
-    control port file. If the option is set to 1, make the control port
-    file readable by the default GID. (Default: 0)
+[[CountPrivateBandwidth]] **CountPrivateBandwidth** **0**|**1**::
+    If this option is set, then Tor's rate-limiting applies not only to
+    remote connections, but also to connections to private addresses like
+    127.0.0.1 or 10.0.0.1.  This is mostly useful for debugging
+    rate-limiting.  (Default: 0)
 
 [[DataDirectory]] **DataDirectory** __DIR__::
     Store working data in DIR. Can not be changed while tor is running.
@@ -432,59 +461,31 @@ GENERAL OPTIONS
     DataDirectory. If the option is set to 1, make the DataDirectory readable
     by the default GID. (Default: 0)
 
-[[CacheDirectory]] **CacheDirectory** __DIR__::
-    Store cached directory data in DIR. Can not be changed while tor is
-    running.
-    (Default: uses the value of DataDirectory.)
-
-[[CacheDirectoryGroupReadable]] **CacheDirectoryGroupReadable** **0**|**1**|**auto**::
-    If this option is set to 0, don't allow the filesystem group to read the
-    CacheDirectory. If the option is set to 1, make the CacheDirectory readable
-    by the default GID. If the option is "auto", then we use the
-    setting for DataDirectoryGroupReadable when the CacheDirectory is the
-    same as the DataDirectory, and 0 otherwise. (Default: auto)
-
-[[FallbackDir]] **FallbackDir** __ipv4address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]::
-    When we're unable to connect to any directory cache for directory info
-    (usually because we don't know about any yet) we try a directory authority.
-    Clients also simultaneously try a FallbackDir, to avoid hangs on client
-    startup if a directory authority is down. Clients retry FallbackDirs more
-    often than directory authorities, to reduce the load on the directory
-    authorities.
-    By default, the directory authorities are also FallbackDirs. Specifying a
-    FallbackDir replaces Tor's default hard-coded FallbackDirs (if any).
-    (See the **DirAuthority** entry for an explanation of each flag.)
-
-[[UseDefaultFallbackDirs]] **UseDefaultFallbackDirs** **0**|**1**::
-    Use Tor's default hard-coded FallbackDirs (if any). (When a
-    FallbackDir line is present, it replaces the hard-coded FallbackDirs,
-    regardless of the value of UseDefaultFallbackDirs.) (Default: 1)
-
-[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__::
+[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__dirport__ __fingerprint__::
     Use a nonstandard authoritative directory server at the provided address
     and port, with the specified key fingerprint. This option can be repeated
     many times, for multiple authoritative directory servers. Flags are
     separated by spaces, and determine what kind of an authority this directory
     is. By default, an authority is not authoritative for any directory style
-    or version unless an appropriate flag is given.
+    or version unless an appropriate flag is given. +
+     +
     Tor will use this authority as a bridge authoritative directory if the
-    "bridge" flag is set. If a flag "orport=**port**" is given, Tor will use the
-    given port when opening encrypted tunnels to the dirserver. If a flag
-    "weight=**num**" is given, then the directory server is chosen randomly
-    with probability proportional to that weight (default 1.0). If a
+    "bridge" flag is set. If a flag "orport=**orport**" is given, Tor will
+    use the given port when opening encrypted tunnels to the dirserver. If a
+    flag "weight=**num**" is given, then the directory server is chosen
+    randomly with probability proportional to that weight (default 1.0). If a
     flag "v3ident=**fp**" is given, the dirserver is a v3 directory authority
     whose v3 long-term signing key has the fingerprint **fp**. Lastly,
     if an "ipv6=**[**__ipv6address__**]**:__orport__" flag is present, then
-    the directory
-    authority is listening for IPv6 connections on the indicated IPv6 address
-    and OR Port. +
- +
+    the directory authority is listening for IPv6 connections on the
+    indicated IPv6 address and OR Port. +
+     +
     Tor will contact the authority at __ipv4address__ to
-    download directory documents. The provided __port__ value is a dirport;
-    clients ignore this in favor of the specified "orport=" value.  If an
-    IPv6 ORPort is supplied, Tor will
-    also download directory documents at the IPv6 ORPort. +
- +
+    download directory documents. Clients always use the ORPort. Relays
+    usually use the DirPort, but will use the ORPort in some circumstances.
+    If an IPv6 ORPort is supplied, clients will also download directory
+    documents at the IPv6 ORPort, if they are configured to use IPv6. +
+     +
     If no **DirAuthority** line is given, Tor will use the default directory
     authorities. NOTE: this option is intended for setting up a private Tor
     network with its own directory authorities. If you use it, you will be
@@ -498,17 +499,6 @@ GENERAL OPTIONS
     should be 1.0 or less. The default is less than 1, to reduce load on
     authorities. (Default: 0.1)
 
-[[AlternateDirAuthority]] **AlternateDirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__ +
-
-[[AlternateBridgeAuthority]] **AlternateBridgeAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __ fingerprint__::
-    These options behave as DirAuthority, but they replace fewer of the
-    default directory authorities. Using
-    AlternateDirAuthority replaces the default Tor directory authorities, but
-    leaves the default bridge authorities in
-    place.  Similarly,
-    AlternateBridgeAuthority replaces the default bridge authority,
-    but leaves the directory authorities alone.
-
 [[DisableAllSwap]] **DisableAllSwap** **0**|**1**::
     If set to 1, Tor will attempt to lock all current and future memory pages,
     so that memory cannot be paged out. Windows, OS X and Solaris are currently
@@ -533,6 +523,55 @@ GENERAL OPTIONS
    this to 0 for the duration of your debugging. Normal users should leave it
    on. Disabling this option while Tor is running is prohibited. (Default: 1)
 
+[[DisableNetwork]] **DisableNetwork** **0**|**1**::
+    When this option is set, we don't listen for or accept any connections
+    other than controller connections, and we close (and don't reattempt)
+    any outbound
+    connections.  Controllers sometimes use this option to avoid using
+    the network until Tor is fully configured.  Tor will make still certain
+    network-related calls (like DNS lookups) as a part of its configuration
+    process, even if DisableNetwork is set. (Default: 0)
+
+[[ExtendByEd25519ID]] **ExtendByEd25519ID** **0**|**1**|**auto**::
+    If this option is set to 1, we always try to include a relay's Ed25519 ID
+    when telling the preceding relay in a circuit to extend to it.
+    If this option is set to 0, we never include Ed25519 IDs when extending
+    circuits.  If the option is set to "auto", we obey a
+    parameter in the consensus document. (Default: auto)
+
+[[ExtORPort]] **ExtORPort** ['address'**:**]{empty}__port__|**auto**::
+    Open this port to listen for Extended ORPort connections from your
+    pluggable transports. +
+    (Default: **DataDirectory**/extended_orport_auth_cookie)
+
+[[ExtORPortCookieAuthFile]] **ExtORPortCookieAuthFile** __Path__::
+    If set, this option overrides the default location and file name
+    for the Extended ORPort's cookie file -- the cookie file is needed
+    for pluggable transports to communicate through the Extended ORPort.
+
+[[ExtORPortCookieAuthFileGroupReadable]] **ExtORPortCookieAuthFileGroupReadable** **0**|**1**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    Extended OR Port cookie file. If the option is set to 1, make the cookie
+    file readable by the default GID. [Making the file readable by other
+    groups is not yet implemented; let us know if you need this for some
+    reason.] (Default: 0)
+
+[[FallbackDir]] **FallbackDir** __ipv4address__:__dirport__ orport=__orport__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]::
+    When tor is unable to connect to any directory cache for directory info
+    (usually because it doesn't know about any yet) it tries a hard-coded
+    directory. Relays try one directory authority at a time. Clients try
+    multiple directory authorities and FallbackDirs, to avoid hangs on
+    startup if a hard-coded directory is down. Clients wait for a few seconds
+    between each attempt, and retry FallbackDirs more often than directory
+    authorities, to reduce the load on the directory authorities.  +
+     +
+    FallbackDirs should be stable relays with stable IP addresses, ports,
+    and identity keys. They must have a DirPort. +
+     +
+    By default, the directory authorities are also FallbackDirs. Specifying a
+    FallbackDir replaces Tor's default hard-coded FallbackDirs (if any).
+    (See <<DirAuthority,DirAuthority>> for an explanation of each flag.)
+
 [[FetchDirInfoEarly]] **FetchDirInfoEarly** **0**|**1**::
     If set to 1, Tor will always fetch directory information like other
     directory caches, even if you don't meet the normal criteria for fetching
@@ -572,6 +611,17 @@ GENERAL OPTIONS
     flavour of consensues and descriptors that is fetched and used for
     building circuits. (Default: 0)
 
+[[HardwareAccel]] **HardwareAccel** **0**|**1**::
+    If non-zero, try to use built-in (static) crypto hardware acceleration when
+    available. Can not be changed while tor is running. (Default: 0)
+
+[[HashedControlPassword]] **HashedControlPassword** __hashed_password__::
+    Allow connections on the control port if they present
+    the password whose one-way hash is __hashed_password__. You
+    can compute the hash of a password by running "tor --hash-password
+    __password__". You can provide several acceptable passwords by using more
+    than one HashedControlPassword line.
+
 [[HTTPProxy]] **HTTPProxy** __host__[:__port__]::
     Tor will make all its directory requests through this host:port (or host:80
     if port is not specified), rather than connecting directly to any directory
@@ -597,51 +647,19 @@ GENERAL OPTIONS
     proxy authentication that Tor supports; feel free to submit a patch if you
     want it to support others.
 
-[[Sandbox]] **Sandbox** **0**|**1**::
-    If set to 1, Tor will run securely through the use of a syscall sandbox.
-    Otherwise the sandbox will be disabled. The option is currently an
-    experimental feature. It only works on Linux-based operating systems,
-    and only when Tor has been built with the libseccomp library. This option
-    can not be changed while tor is running.
- +
-    When the Sandbox is 1, the following options can not be changed when tor
-    is running:
-    Address
-    ConnLimit
-    CookieAuthFile
-    DirPortFrontPage
-    ExtORPortCookieAuthFile
-    Logs
-    ServerDNSResolvConfFile
-    Tor must remain in client or server mode (some changes to ClientOnly and
-    ORPort are not allowed).
-    ClientOnionAuthDir and any files in it won't reload on HUP signal.
-    (Default: 0)
-
-[[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]::
-    Tor will make all OR connections through the SOCKS 4 proxy at host:port
-    (or host:1080 if port is not specified).
-
-[[Socks5Proxy]] **Socks5Proxy** __host__[:__port__]::
-    Tor will make all OR connections through the SOCKS 5 proxy at host:port
-    (or host:1080 if port is not specified).
-
-[[Socks5ProxyUsername]] **Socks5ProxyUsername** __username__ +
-
-[[Socks5ProxyPassword]] **Socks5ProxyPassword** __password__::
-    If defined, authenticate to the SOCKS 5 server using username and password
-    in accordance to RFC 1929. Both username and password must be between 1 and
-    255 characters.
-
-[[UnixSocksGroupWritable]] **UnixSocksGroupWritable** **0**|**1**::
-    If this option is set to 0, don't allow the filesystem group to read and
-    write unix sockets (e.g. SocksPort unix:). If the option is set to 1, make
-    the Unix socket readable and writable by the default GID. (Default: 0)
-
 [[KeepalivePeriod]] **KeepalivePeriod** __NUM__::
     To keep firewalls from expiring connections, send a padding keepalive cell
     every NUM seconds on open connections that are in use. (Default: 5 minutes)
 
+[[KeepBindCapabilities]] **KeepBindCapabilities** **0**|**1**|**auto**::
+    On Linux, when we are started as root and we switch our identity using
+    the **User** option, the **KeepBindCapabilities** option tells us whether to
+    try to retain our ability to bind to low ports.  If this value is 1, we
+    try to keep the capability; if it is 0 we do not; and if it is **auto**,
+    we keep the capability only if we are configured to listen on a low port.
+    Can not be changed while tor is running.
+    (Default: auto.)
+
 [[Log]] **Log** __minSeverity__[-__maxSeverity__] **stderr**|**stdout**|**syslog**::
     Send all messages between __minSeverity__ and __maxSeverity__ to the standard
     output stream, the standard error stream, or to the system log. (The
@@ -649,7 +667,16 @@ GENERAL OPTIONS
     debug, info, notice, warn, and err. We advise using "notice" in most cases,
     since anything more verbose may provide sensitive information to an
     attacker who obtains the logs. If only one severity level is given, all
-    messages of that level or higher will be sent to the listed destination.
+    messages of that level or higher will be sent to the listed destination. +
+     +
+    Some low-level logs may be sent from signal handlers, so their destination
+    logs must be signal-safe. These low-level logs include backtraces,
+    logging function errors, and errors in code called by logging functions.
+    Signal-safe logs are always sent to stderr or stdout. They are also sent to
+    a limited number of log files that are configured to log messages at error
+    severity from the bug or general domains. They are never sent as syslogs,
+    android logs, control port log events, or to any API-based log
+    destinations.
 
 [[Log2]] **Log** __minSeverity__[-__maxSeverity__] **file** __FILENAME__::
     As above, but send log messages to the listed filename. The
@@ -667,15 +694,16 @@ GENERAL OPTIONS
     list of logging domains.  You can prefix a domain with $$~$$ to indicate
     negation, and use * to indicate "all domains".  If you specify a severity
     range without a list of domains, it matches all domains. +
- +
+     +
     This is an advanced feature which is most useful for debugging one or two
     of Tor's subsystems at a time. +
- +
+     +
     The currently recognized domains are: general, crypto, net, config, fs,
     protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge,
-    acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos.
+    acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos,
+    process, pt, btrack, and mesg.
     Domain names are case-insensitive. +
- +
+     +
     For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends
     to stdout: all handshake messages of any severity, all info-and-higher
     messages from domains other than networking and memory management, and all
@@ -686,12 +714,33 @@ GENERAL OPTIONS
     message currently has at least one domain; most currently have exactly
     one.  This doesn't affect controller log messages. (Default: 0)
 
+[[LogTimeGranularity]] **LogTimeGranularity** __NUM__::
+    Set the resolution of timestamps in Tor's logs to NUM milliseconds.
+    NUM must be positive and either a divisor or a multiple of 1 second.
+    Note that this option only controls the granularity written by Tor to
+    a file or console log.  Tor does not (for example) "batch up" log
+    messages to affect times logged by a controller, times attached to
+    syslog messages, or the mtime fields on log files.  (Default: 1 second)
+
+[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If set, we will not advertise more than this amount of bandwidth for our
+    BandwidthRate. Server operators who want to reduce the number of clients
+    who ask to build circuits through them (since this is proportional to
+    advertised bandwidth rate) can thus reduce the CPU demands on their server
+    without impacting network performance.
+
 [[MaxUnparseableDescSizeToLog]] **MaxUnparseableDescSizeToLog** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**::
     Unparseable descriptors (e.g. for votes, consensuses, routers) are logged
     in separate files by hash, up to the specified size in total.  Note that
     only files logged during the lifetime of this Tor process count toward the
     total; this is intended to be used to debug problems without opening live
-    servers to resource exhaustion attacks. (Default: 10 MB)
+    servers to resource exhaustion attacks. (Default: 10 MBytes)
+
+[[NoExec]] **NoExec** **0**|**1**::
+    If this option is set to 1, then Tor will never launch another
+    executable, regardless of the settings of ClientTransportPlugin
+    or ServerTransportPlugin.  Once this option has been set to 1,
+    it cannot be set back to 0 without restarting Tor. (Default: 0)
 
 [[OutboundBindAddress]] **OutboundBindAddress** __IP__::
     Make all outbound connections originate from the IP address specified. This
@@ -702,6 +751,15 @@ GENERAL OPTIONS
     This setting will be ignored for connections to the loopback addresses
     (127.0.0.0/8 and ::1), and is not used for DNS requests as well.
 
+[[OutboundBindAddressExit]] **OutboundBindAddressExit** __IP__::
+    Make all outbound exit connections originate from the IP address
+    specified. This option overrides **OutboundBindAddress** for the
+    same IP version. This option may be used twice, once with an IPv4
+    address and once with an IPv6 address.
+    IPv6 addresses should be wrapped in square brackets.
+    This setting will be ignored
+    for connections to the loopback addresses (127.0.0.0/8 and ::1).
+
 [[OutboundBindAddressOR]] **OutboundBindAddressOR** __IP__::
     Make all outbound non-exit (relay and other) connections
     originate from the IP address specified. This option overrides
@@ -711,14 +769,20 @@ GENERAL OPTIONS
     This setting will be ignored for connections to the loopback
     addresses (127.0.0.0/8 and ::1).
 
-[[OutboundBindAddressExit]] **OutboundBindAddressExit** __IP__::
-    Make all outbound exit connections originate from the IP address
-    specified. This option overrides **OutboundBindAddress** for the
-    same IP version. This option may be used twice, once with an IPv4
-    address and once with an IPv6 address.
-    IPv6 addresses should be wrapped in square brackets.
-    This setting will be ignored
-    for connections to the loopback addresses (127.0.0.0/8 and ::1).
+[[OwningControllerProcess]] **{dbl_}OwningControllerProcess** __PID__::
+    Make Tor instance periodically check for presence of a controller process
+    with given PID and terminate itself if this process is no longer alive.
+    Polling interval is 15 seconds.
+
+[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If this option is set manually, or via the "perconnbwburst" consensus
+    field, Tor will use it for separate rate limiting for each connection
+    from a non-relay. (Default: 0)
+
+[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If this option is set manually, or via the "perconnbwrate" consensus
+    field, Tor will use it for separate rate limiting for each connection
+    from a non-relay. (Default: 0)
 
 [[PidFile]] **PidFile** __FILE__::
     On startup, write our PID to FILE. On clean shutdown, remove
@@ -729,40 +793,38 @@ GENERAL OPTIONS
     following the Tor specification. Otherwise, they are logged with severity
     \'info'. (Default: 0)
 
+[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If not 0, limit the maximum token bucket size (also known as the burst) for
+    \_relayed traffic_ to the given number of bytes in each direction.
+    They do not include directory fetches by the relay (from authority
+    or other relays), because that is considered "client" activity. (Default: 0)
+
+[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If not 0, a separate token bucket limits the average incoming bandwidth
+    usage for \_relayed traffic_ on this node to the specified number of bytes
+    per second, and the average outgoing bandwidth usage to that same value.
+    Relayed traffic currently is calculated to include answers to directory
+    requests, but that may change in future versions. They do not include directory
+    fetches by the relay (from authority or other relays), because that is considered
+    "client" activity.  (Default: 0)
+
+[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
+    Tells an authority, or other node tracking node reliability and history,
+    that fine-grained information about nodes can be discarded when it hasn't
+    changed for a given amount of time.  (Default: 24 hours)
+
 [[RunAsDaemon]] **RunAsDaemon** **0**|**1**::
     If 1, Tor forks and daemonizes to the background. This option has no effect
     on Windows; instead you should use the --service command-line option.
     Can not be changed while tor is running.
     (Default: 0)
 
-[[LogTimeGranularity]] **LogTimeGranularity** __NUM__::
-    Set the resolution of timestamps in Tor's logs to NUM milliseconds.
-    NUM must be positive and either a divisor or a multiple of 1 second.
-    Note that this option only controls the granularity written by Tor to
-    a file or console log.  Tor does not (for example) "batch up" log
-    messages to affect times logged by a controller, times attached to
-    syslog messages, or the mtime fields on log files.  (Default: 1 second)
-
-[[TruncateLogFile]] **TruncateLogFile** **0**|**1**::
-    If 1, Tor will overwrite logs at startup and in response to a HUP signal,
-    instead of appending to them. (Default: 0)
-
-[[SyslogIdentityTag]] **SyslogIdentityTag** __tag__::
-    When logging to syslog, adds a tag to the syslog identity such that
-    log entries are marked with "Tor-__tag__". Can not be changed while tor is
-    running. (Default: none)
-
-[[AndroidIdentityTag]] **AndroidIdentityTag** __tag__::
-    When logging to Android's logging subsystem, adds a tag to the log identity
-    such that log entries are marked with "Tor-__tag__". Can not be changed while
-    tor is running. (Default: none)
-
 [[SafeLogging]] **SafeLogging** **0**|**1**|**relay**::
     Tor can scrub potentially sensitive strings from log messages (e.g.
     addresses) by replacing them with the string [scrubbed]. This way logs can
     still be useful, but they don't leave behind personally identifying
     information about what sites a user might have visited. +
- +
+     +
     If this option is set to 0, Tor will not perform any scrubbing, if it is
     set to 1, all potentially sensitive strings are replaced. If it is set to
     relay, all log messages generated when acting as a relay are sanitized, but
@@ -770,67 +832,36 @@ GENERAL OPTIONS
     Note: Tor may not heed this option when logging at log levels below Notice.
     (Default: 1)
 
-[[User]] **User** __Username__::
-    On startup, setuid to this user and setgid to their primary group.
-    Can not be changed while tor is running.
-
-[[KeepBindCapabilities]] **KeepBindCapabilities** **0**|**1**|**auto**::
-    On Linux, when we are started as root and we switch our identity using
-    the **User** option, the **KeepBindCapabilities** option tells us whether to
-    try to retain our ability to bind to low ports.  If this value is 1, we
-    try to keep the capability; if it is 0 we do not; and if it is **auto**,
-    we keep the capability only if we are configured to listen on a low port.
-    Can not be changed while tor is running.
-    (Default: auto.)
-
-[[HardwareAccel]] **HardwareAccel** **0**|**1**::
-    If non-zero, try to use built-in (static) crypto hardware acceleration when
-    available. Can not be changed while tor is running. (Default: 0)
-
-[[AccelName]] **AccelName** __NAME__::
-    When using OpenSSL hardware crypto acceleration attempt to load the dynamic
-    engine of this name. This must be used for any dynamic hardware engine.
-    Names can be verified with the openssl engine command. Can not be changed
-    while tor is running.
-
-[[AccelDir]] **AccelDir** __DIR__::
-    Specify this option if using dynamic hardware acceleration and the engine
-    implementation library resides somewhere other than the OpenSSL default.
-    Can not be changed while tor is running.
-
-[[AvoidDiskWrites]] **AvoidDiskWrites** **0**|**1**::
-    If non-zero, try to write to disk less frequently than we would otherwise.
-    This is useful when running on flash memory or other media that support
-    only a limited number of writes. (Default: 0)
-
-[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM__::
-    If this value is set, we override the default algorithm for choosing which
-    circuit's cell to deliver or relay next. It is delivered first to the
-    circuit that has the lowest weighted cell count, where cells are weighted
-    exponentially according to this value (in seconds). If the value is -1, it
-    is taken from the consensus if possible else it will fallback to the
-    default value of 30. Minimum: 1, Maximum: 2147483647. This can be defined
-    as a float value. This is an advanced option; you generally shouldn't have
-    to mess with it. (Default: -1)
-
-[[CountPrivateBandwidth]] **CountPrivateBandwidth** **0**|**1**::
-    If this option is set, then Tor's rate-limiting applies not only to
-    remote connections, but also to connections to private addresses like
-    127.0.0.1 or 10.0.0.1.  This is mostly useful for debugging
-    rate-limiting.  (Default: 0)
-
-[[ExtendByEd25519ID]] **ExtendByEd25519ID** **0**|**1**|**auto**::
-    If this option is set to 1, we always try to include a relay's Ed25519 ID
-    when telling the proceeding relay in a circuit to extend to it.
-    If this option is set to 0, we never include Ed25519 IDs when extending
-    circuits.  If the option is set to "default", we obey a
-    parameter in the consensus document. (Default: auto)
-
-[[NoExec]] **NoExec** **0**|**1**::
-    If this option is set to 1, then Tor will never launch another
-    executable, regardless of the settings of ClientTransportPlugin
-    or ServerTransportPlugin.  Once this option has been set to 1,
-    it cannot be set back to 0 without restarting Tor. (Default: 0)
+[[Sandbox]] **Sandbox** **0**|**1**::
+    If set to 1, Tor will run securely through the use of a syscall sandbox.
+    Otherwise the sandbox will be disabled. The option is currently an
+    experimental feature. It only works on Linux-based operating systems,
+    and only when Tor has been built with the libseccomp library. This option
+    can not be changed while tor is running. +
+     +
+    When the **Sandbox** is 1, the following options can not be changed when tor
+    is running:
+    **Address**,
+    **ConnLimit**,
+    **CookieAuthFile**,
+    **DirPortFrontPage**,
+    **ExtORPortCookieAuthFile**,
+    **Logs**,
+    **ServerDNSResolvConfFile**,
+    **ClientOnionAuthDir** (and any files in it won't reload on HUP signal). +
+     +
+    Launching new Onion Services through the control port is not supported
+    with current syscall sandboxing implementation. +
+     +
+    Tor must remain in client or server mode (some changes to **ClientOnly**
+    and **ORPort** are not allowed). Currently, if **Sandbox** is 1,
+    **ControlPort** command "GETINFO address" will not work. +
+     +
+    When using %include in the tor configuration files, reloading the tor
+    configuration is not supported after adding new configuration files or
+    directories. +
+     +
+    (Default: 0)
 
 [[Schedulers]] **Schedulers** **KIST**|**KISTLite**|**Vanilla**::
     Specify the scheduler type that tor should use. The scheduler is
@@ -839,44 +870,137 @@ GENERAL OPTIONS
     unavailable, the second one is tried and so on. It is possible to change
     these values at runtime. This option mostly effects relays, and most
     operators should leave it set to its default value.
-    (Default: KIST,KISTLite,Vanilla)
- +
+    (Default: KIST,KISTLite,Vanilla) +
+     +
     The possible scheduler types are:
- +
+     +
     **KIST**: Kernel-Informed Socket Transport. Tor will use TCP information
     from the kernel to make informed decisions regarding how much data to send
     and when to send it. KIST also handles traffic in batches (see
-    KISTSchedRunInterval) in order to improve traffic prioritization decisions.
+    <<KISTSchedRunInterval,KISTSchedRunInterval>>) in order to improve traffic prioritization decisions.
     As implemented, KIST will only work on Linux kernel version 2.6.39 or
-    higher.
- +
+    higher. +
+     +
     **KISTLite**: Same as KIST but without kernel support. Tor will use all
     the same mechanics as with KIST, including the batching, but its decisions
     regarding how much data to send will not be as good. KISTLite will work on
     all kernels and operating systems, and the majority of the benefits of KIST
-    are still realized with KISTLite.
- +
+    are still realized with KISTLite. +
+     +
     **Vanilla**: The scheduler that Tor used before KIST was implemented. It
     sends as much data as possible, as soon as possible. Vanilla will work on
     all kernels and operating systems.
 
+// Out of order because it logically belongs near the Schedulers option
 [[KISTSchedRunInterval]] **KISTSchedRunInterval** __NUM__ **msec**::
     If KIST or KISTLite is used in the Schedulers option, this controls at which
     interval the scheduler tick is. If the value is 0 msec, the value is taken
     from the consensus if possible else it will fallback to the default 10
     msec. Maximum possible value is 100 msec. (Default: 0 msec)
 
+// Out of order because it logically belongs near the Schedulers option
 [[KISTSockBufSizeFactor]] **KISTSockBufSizeFactor** __NUM__::
     If KIST is used in Schedulers, this is a multiplier of the per-socket
     limit calculation of the KIST algorithm. (Default: 1.0)
 
-CLIENT OPTIONS
---------------
+
+[[ServerTransportListenAddr]] **ServerTransportListenAddr** __transport__ __IP__:__PORT__::
+    When this option is set, Tor will suggest __IP__:__PORT__ as the
+    listening address of any pluggable transport proxy that tries to
+    launch __transport__. (IPv4 addresses should written as-is; IPv6
+    addresses should be wrapped in square brackets.) (Default: none)
+
+[[ServerTransportOptions]] **ServerTransportOptions** __transport__ __k=v__ __k=v__ ...::
+    When this option is set, Tor will pass the __k=v__ parameters to
+    any pluggable transport proxy that tries to launch __transport__. +
+    (Example: ServerTransportOptions obfs45 shared-secret=bridgepasswd cache=/var/lib/tor/cache) (Default: none)
+
+[[ServerTransportPlugin]] **ServerTransportPlugin** __transport__ exec __path-to-binary__ [options]::
+    The Tor relay launches the pluggable transport proxy in __path-to-binary__
+    using __options__ as its command-line options, and expects to receive
+    proxied client traffic from it. (Default: none)
+
+[[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]::
+    Tor will make all OR connections through the SOCKS 4 proxy at host:port
+    (or host:1080 if port is not specified).
+
+[[Socks5Proxy]] **Socks5Proxy** __host__[:__port__]::
+    Tor will make all OR connections through the SOCKS 5 proxy at host:port
+    (or host:1080 if port is not specified).
+
+// Out of order because Username logically precedes Password
+[[Socks5ProxyUsername]] **Socks5ProxyUsername** __username__ +
+
+[[Socks5ProxyPassword]] **Socks5ProxyPassword** __password__::
+    If defined, authenticate to the SOCKS 5 server using username and password
+    in accordance to RFC 1929. Both username and password must be between 1 and
+    255 characters.
+
+[[SyslogIdentityTag]] **SyslogIdentityTag** __tag__::
+    When logging to syslog, adds a tag to the syslog identity such that
+    log entries are marked with "Tor-__tag__". Can not be changed while tor is
+    running. (Default: none)
+
+[[TCPProxy]] **TCPProxy** __protocol__ __host__:__port__::
+	Tor will use the given protocol to make all its OR (SSL) connections through
+	a TCP proxy on host:port, rather than connecting directly to servers. You may
+	want to set **FascistFirewall** to restrict the set of ports you might try to
+	connect to, if your proxy only allows connecting to certain ports. There is no
+	equivalent option for directory connections, because all Tor client versions
+	that support this option download directory documents via OR connections. +
++
+	The only protocol supported right now 'haproxy'. This option is only for
+	clients. (Default: none) +
++
+	The HAProxy version 1 proxy protocol is described in detail at
+	https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt +
++
+	Both source IP address and source port will be set to zero.
+
+[[TruncateLogFile]] **TruncateLogFile** **0**|**1**::
+    If 1, Tor will overwrite logs at startup and in response to a HUP signal,
+    instead of appending to them. (Default: 0)
+
+[[UnixSocksGroupWritable]] **UnixSocksGroupWritable** **0**|**1**::
+    If this option is set to 0, don't allow the filesystem group to read and
+    write unix sockets (e.g. SocksPort unix:). If the option is set to 1, make
+    the Unix socket readable and writable by the default GID. (Default: 0)
+
+[[UseDefaultFallbackDirs]] **UseDefaultFallbackDirs** **0**|**1**::
+    Use Tor's default hard-coded FallbackDirs (if any). (When a
+    FallbackDir line is present, it replaces the hard-coded FallbackDirs,
+    regardless of the value of UseDefaultFallbackDirs.) (Default: 1)
+
+[[User]] **User** __Username__::
+    On startup, setuid to this user and setgid to their primary group.
+    Can not be changed while tor is running.
+
+== CLIENT OPTIONS
+
+// These options are in alphabetical order, with exceptions as noted.
+// Please keep them that way!
 
 The following options are useful only for clients (that is, if
 **SocksPort**, **HTTPTunnelPort**, **TransPort**, **DNSPort**, or
 **NATDPort** is non-zero):
 
+[[AllowNonRFC953Hostnames]] **AllowNonRFC953Hostnames** **0**|**1**::
+    When this option is disabled, Tor blocks hostnames containing illegal
+    characters (like @ and :) rather than sending them to an exit node to be
+    resolved. This helps trap accidental attempts to resolve URLs and so on.
+    (Default: 0)
+
+[[AutomapHostsOnResolve]] **AutomapHostsOnResolve** **0**|**1**::
+    When this option is enabled, and we get a request to resolve an address
+    that ends with one of the suffixes in **AutomapHostsSuffixes**, we map an
+    unused virtual address to that address, and return the new virtual address.
+    This is handy for making ".onion" addresses work with applications that
+    resolve an address and then connect to it. (Default: 0)
+
+[[AutomapHostsSuffixes]] **AutomapHostsSuffixes** __SUFFIX__,__SUFFIX__,__...__::
+    A comma-separated list of suffixes to use with **AutomapHostsOnResolve**.
+    The "." suffix is equivalent to "all addresses." (Default: .exit,.onion).
+
 [[Bridge]] **Bridge** [__transport__] __IP__:__ORPort__ [__fingerprint__]::
     When set along with UseBridges, instructs Tor to use the relay at
     "IP:ORPort" as a "bridge" relaying into the Tor network. If "fingerprint"
@@ -884,45 +1008,79 @@ The following options are useful only for clients (that is, if
     the relay running at that location has the right fingerprint. We also use
     fingerprint to look up the bridge descriptor at the bridge authority, if
     it's provided and if UpdateBridgesFromAuthority is set too.  +
- +
+     +
     If "transport" is provided, it must match a ClientTransportPlugin line. We
     then use that pluggable transport's proxy to transfer data to the bridge,
     rather than connecting to the bridge directly. Some transports use a
     transport-specific method to work out the remote address to connect to.
     These transports typically ignore the "IP:ORPort" specified in the bridge
     line.  +
- +
+     +
     Tor passes any "key=val" settings to the pluggable transport proxy as
     per-connection arguments when connecting to the bridge.  Consult
     the documentation of the pluggable transport for details of what
     arguments it supports.
 
-[[LearnCircuitBuildTimeout]] **LearnCircuitBuildTimeout** **0**|**1**::
-    If 0, CircuitBuildTimeout adaptive learning is disabled. (Default: 1)
+[[CircuitPadding]] **CircuitPadding** **0**|**1**::
+    If set to 0, Tor will not pad client circuits with additional cover
+    traffic. Only clients may set this option. This option should be offered
+    via the UI to mobile users for use where bandwidth may be expensive. If
+    set to 1, padding will be negotiated as per the consensus and relay
+    support (unlike ConnectionPadding, CircuitPadding cannot be force-enabled).
+    (Default: 1)
 
-[[CircuitBuildTimeout]] **CircuitBuildTimeout** __NUM__::
+// Out of order because it logically belongs after CircuitPadding
+[[ReducedCircuitPadding]] **ReducedCircuitPadding** **0**|**1**::
+    If set to 1, Tor will only use circuit padding algorithms that have low
+    overhead. Only clients may set this option. This option should be offered
+    via the UI to mobile users for use where bandwidth may be expensive.
+    (Default: 0)
 
-    Try for at most NUM seconds when building circuits. If the circuit isn't
-    open in that time, give up on it. If LearnCircuitBuildTimeout is 1, this
-    value serves as the initial value to use before a timeout is learned. If
-    LearnCircuitBuildTimeout is 0, this value is the only value used.
-    (Default: 60 seconds)
+[[ClientBootstrapConsensusAuthorityDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from authorities
+    if they are bootstrapping (that is, they don't have a usable, reasonably
+    live consensus). Only used by clients fetching from a list of fallback
+    directory mirrors. This schedule is advanced by (potentially concurrent)
+    connection attempts, unlike other schedules, which are advanced by
+    connection failures. (Default: 6)
 
-[[CircuitsAvailableTimeout]] **CircuitsAvailableTimeout** __NUM__::
-    Tor will attempt to keep at least one open, unused circuit available for
-    this amount of time. This option governs how long idle circuits are kept
-    open, as well as the amount of time Tor will keep a circuit open to each
-    of the recently used ports. This way when the Tor client is entirely
-    idle, it can expire all of its circuits, and then expire its TLS
-    connections. Note that the actual timeout value is uniformly randomized
-    from the specified value to twice that amount. (Default: 30 minutes;
-    Max: 24 hours)
+[[ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from authorities
+    if they are bootstrapping (that is, they don't have a usable, reasonably
+    live consensus). Only used by clients which don't have or won't fetch
+    from a list of fallback directory mirrors. This schedule is advanced by
+    (potentially concurrent) connection attempts, unlike other schedules,
+    which are advanced by connection failures. (Default: 0)
 
-[[CircuitStreamTimeout]] **CircuitStreamTimeout** __NUM__::
-    If non-zero, this option overrides our internal timeout schedule for how
-    many seconds until we detach a stream from a circuit and try a new circuit.
-    If your network is particularly slow, you might want to set this to a
-    number like 60. (Default: 0)
+[[ClientBootstrapConsensusFallbackDownloadInitialDelay]] **ClientBootstrapConsensusFallbackDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from fallback
+    directory mirrors if they are bootstrapping (that is, they don't have a
+    usable, reasonably live consensus). Only used by clients fetching from a
+    list of fallback directory mirrors. This schedule is advanced by
+    (potentially concurrent) connection attempts, unlike other schedules,
+    which are advanced by connection failures. (Default: 0)
+
+[[ClientBootstrapConsensusMaxInProgressTries]] **ClientBootstrapConsensusMaxInProgressTries** __NUM__::
+    Try this many simultaneous connections to download a consensus before
+    waiting for one to complete, timeout, or error out. (Default: 3)
+
+[[ClientDNSRejectInternalAddresses]] **ClientDNSRejectInternalAddresses** **0**|**1**::
+    If true, Tor does not believe any anonymously retrieved DNS answer that
+    tells it that an address resolves to an internal address (like 127.0.0.1 or
+    192.168.0.1). This option prevents certain browser-based attacks; it
+    is not allowed to be set on the default network. (Default: 1)
+
+[[ClientOnionAuthDir]] **ClientOnionAuthDir** __path__::
+    Path to the directory containing v3 hidden service authorization files.
+    Each file is for a single onion address, and the files MUST have the suffix
+    ".auth_private" (i.e. "bob_onion.auth_private"). The content format MUST be:
+     +
+      <onion-address>:descriptor:x25519:<base32-encoded-privkey>
+     +
+    The <onion-address> MUST NOT have the ".onion" suffix. The
+    <base32-encoded-privkey> is the base32 representation of the raw key bytes
+    only (32 bytes for x25519). See Appendix G in the rend-spec-v3.txt file of
+    https://spec.torproject.org/[torspec] for more information.
 
 [[ClientOnly]] **ClientOnly** **0**|**1**::
     If set to 1, Tor will not run as a relay or serve
@@ -933,6 +1091,43 @@ The following options are useful only for clients (that is, if
     and fast enough. The current behavior is simply that Tor is a client
     unless ORPort, ExtORPort, or DirPort are configured.) (Default: 0)
 
+[[ClientPreferIPv6DirPort]] **ClientPreferIPv6DirPort** **0**|**1**|**auto**::
+    If this option is set to 1, Tor prefers a directory port with an IPv6
+    address over one with IPv4, for direct connections, if a given directory
+    server has both. (Tor also prefers an IPv6 DirPort if IPv4Client is set to
+    0.) If this option is set to auto, clients prefer IPv4. Other things may
+    influence the choice. This option breaks a tie to the favor of IPv6.
+    (Default: auto)  (DEPRECATED: This option has had no effect for some
+    time.)
+
+[[ClientPreferIPv6ORPort]] **ClientPreferIPv6ORPort** **0**|**1**|**auto**::
+    If this option is set to 1, Tor prefers an OR port with an IPv6
+    address over one with IPv4 if a given entry node has both. (Tor also
+    prefers an IPv6 ORPort if IPv4Client is set to 0.) If this option is set
+    to auto, Tor bridge clients prefer the configured bridge address, and
+    other clients prefer IPv4. Other things may influence the choice. This
+    option breaks a tie to the favor of IPv6. (Default: auto)
+
+[[ClientRejectInternalAddresses]] **ClientRejectInternalAddresses** **0**|**1**::
+    If true, Tor does not try to fulfill requests to connect to an internal
+    address (like 127.0.0.1 or 192.168.0.1) __unless an exit node is
+    specifically requested__ (for example, via a .exit hostname, or a
+    controller request).  If true, multicast DNS hostnames for machines on the
+    local network (of the form *.local) are also rejected.  (Default: 1)
+
+[[ClientUseIPv4]] **ClientUseIPv4** **0**|**1**::
+    If this option is set to 0, Tor will avoid connecting to directory servers
+    and entry nodes over IPv4. Note that clients with an IPv4
+    address in a **Bridge**, proxy, or pluggable transport line will try
+    connecting over IPv4 even if **ClientUseIPv4** is set to 0. (Default: 1)
+
+[[ClientUseIPv6]] **ClientUseIPv6** **0**|**1**::
+    If this option is set to 1, Tor might connect to directory servers or
+    entry nodes over IPv6. For IPv6 only hosts, you need to also set
+    **ClientUseIPv4** to 0 to disable IPv4. Note that clients configured with
+    an IPv6 address in a **Bridge**, proxy, or pluggable transportline will
+    try connecting over IPv6 even if **ClientUseIPv6** is set to 0. (Default: 0)
+
 [[ConnectionPadding]] **ConnectionPadding** **0**|**1**|**auto**::
     This option governs Tor's use of padding to defend against some forms of
     traffic analysis. If it is set to 'auto', Tor will send padding only
@@ -943,105 +1138,36 @@ The following options are useful only for clients (that is, if
     for use where bandwidth may be expensive.
     (Default: auto)
 
+// Out of order because it logically belongs after ConnectionPadding
 [[ReducedConnectionPadding]] **ReducedConnectionPadding** **0**|**1**::
     If set to 1, Tor will not not hold OR connections open for very long,
     and will send less padding on these connections. Only clients may set
     this option. This option should be offered via the UI to mobile users
     for use where bandwidth may be expensive. (Default: 0)
 
-[[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__::
-    A list of identity fingerprints, country codes, and address
-    patterns of nodes to avoid when building a circuit. Country codes are
-    2-letter ISO3166 codes, and must
-    be wrapped in braces; fingerprints may be preceded by a dollar sign.
-    (Example:
-    ExcludeNodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
- +
-    By default, this option is treated as a preference that Tor is allowed
-    to override in order to keep working.
-    For example, if you try to connect to a hidden service,
-    but you have excluded all of the hidden service's introduction points,
-    Tor will connect to one of them anyway.  If you do not want this
-    behavior, set the StrictNodes option (documented below).  +
- +
-    Note also that if you are a relay, this (and the other node selection
-    options below) only affects your own circuits that Tor builds for you.
-    Clients can still build circuits through you to any node.  Controllers
-    can tell Tor to build circuits through any node. +
- +
-    Country codes are case-insensitive. The code "\{??}" refers to nodes whose
-    country can't be identified. No country code, including \{??}, works if
-    no GeoIPFile can be loaded. See also the GeoIPExcludeUnknown option below.
-
-
-[[ExcludeExitNodes]] **ExcludeExitNodes** __node__,__node__,__...__::
-    A list of identity fingerprints, country codes, and address
-    patterns of nodes to never use when picking an exit node---that is, a
-    node that delivers traffic for you *outside* the Tor network.   Note that any
-    node listed in ExcludeNodes is automatically considered to be part of this
-    list too.  See
-    the **ExcludeNodes** option for more information on how to specify
-    nodes. See also the caveats on the "ExitNodes" option below.
-
-[[GeoIPExcludeUnknown]] **GeoIPExcludeUnknown** **0**|**1**|**auto**::
-    If this option is set to 'auto', then whenever any country code is set in
-    ExcludeNodes or ExcludeExitNodes, all nodes with unknown country (\{??} and
-    possibly \{A1}) are treated as excluded as well. If this option is set to
-    '1', then all unknown countries are treated as excluded in ExcludeNodes
-    and ExcludeExitNodes.  This option has no effect when a GeoIP file isn't
-    configured or can't be found.  (Default: auto)
-
-[[ExitNodes]] **ExitNodes** __node__,__node__,__...__::
-    A list of identity fingerprints, country codes, and address
-    patterns of nodes to use as exit node---that is, a
-    node that delivers traffic for you *outside* the Tor network.  See
-    the **ExcludeNodes** option for more information on how to specify nodes. +
- +
-    Note that if you list too few nodes here, or if you exclude too many exit
-    nodes with ExcludeExitNodes, you can degrade functionality.  For example,
-    if none of the exits you list allows traffic on port 80 or 443, you won't
-    be able to browse the web. +
- +
-    Note also that not every circuit is used to deliver traffic *outside* of
-    the Tor network.  It is normal to see non-exit circuits (such as those
-    used to connect to hidden services, those that do directory fetches,
-    those used for relay reachability self-tests, and so on) that end
-    at a non-exit node.  To
-    keep a node from being used entirely, see ExcludeNodes and StrictNodes. +
- +
-    The ExcludeNodes option overrides this option: any node listed in both
-    ExitNodes and ExcludeNodes is treated as excluded. +
- +
-    The .exit address notation, if enabled via MapAddress, overrides
-    this option.
+[[DNSPort]] **DNSPort** ['address'**:**]{empty}__port__|**auto** [_isolation flags_]::
+    If non-zero, open this port to listen for UDP DNS requests, and resolve
+    them anonymously.  This port only handles A, AAAA, and PTR requests---it
+    doesn't handle arbitrary DNS request types. Set the port to "auto" to
+    have Tor pick a port for
+    you. This directive can be specified multiple times to bind to multiple
+    addresses/ports. See <<SocksPort,SocksPort>> for an explanation of isolation
+    flags. (Default: 0)
 
-[[EntryNodes]] **EntryNodes** __node__,__node__,__...__::
-    A list of identity fingerprints and country codes of nodes
-    to use for the first hop in your normal circuits.
-    Normal circuits include all
-    circuits except for direct connections to directory servers.  The Bridge
-    option overrides this option; if you have configured bridges and
-    UseBridges is 1, the Bridges are used as your entry nodes. +
- +
-    The ExcludeNodes option overrides this option: any node listed in both
-    EntryNodes and ExcludeNodes is treated as excluded. See
-    the **ExcludeNodes** option for more information on how to specify nodes.
+[[DownloadExtraInfo]] **DownloadExtraInfo** **0**|**1**::
+    If true, Tor downloads and caches "extra-info" documents. These documents
+    contain information about servers other than the information in their
+    regular server descriptors. Tor does not use this information for anything
+    itself; to save bandwidth, leave this option turned off. (Default: 0)
 
-[[StrictNodes]] **StrictNodes** **0**|**1**::
-    If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option
-    as a requirement to follow for all the circuits you generate, even if
-    doing so will break functionality for you (StrictNodes applies to neither
-    ExcludeExitNodes nor to ExitNodes).  If StrictNodes is set to 0, Tor will
-    still try to avoid nodes in the ExcludeNodes list, but it will err on the
-    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells Tor
-    that it is okay to use an excluded node when it is *necessary* to perform
-    relay reachability self-tests, connect to a hidden service, provide a
-    hidden service to a client, fulfill a .exit request, upload directory
-    information, or download directory information.  (Default: 0)
+[[EnforceDistinctSubnets]] **EnforceDistinctSubnets** **0**|**1**::
+    If 1, Tor will not put two servers whose IP addresses are "too close" on
+    the same circuit. Currently, two addresses are "too close" if they lie in
+    the same /16 range. (Default: 1)
 
 [[FascistFirewall]] **FascistFirewall** **0**|**1**::
     If 1, Tor will only create outgoing connections to ORs running on ports
-    that your firewall allows (defaults to 80 and 443; see **FirewallPorts**).
+    that your firewall allows (defaults to 80 and 443; see <<FirewallPorts,FirewallPorts>>).
     This will allow you to run Tor as a client behind a firewall with
     restrictive policies, but will not allow you to run as a server behind such
     a firewall. If you prefer more fine-grained control, use
@@ -1052,38 +1178,8 @@ The following options are useful only for clients (that is, if
     **FascistFirewall** is set. This option is deprecated; use ReachableAddresses
     instead. (Default: 80, 443)
 
-[[ReachableAddresses]] **ReachableAddresses** __IP__[/__MASK__][:__PORT__]...::
-    A comma-separated list of IP addresses and ports that your firewall allows
-    you to connect to. The format is as for the addresses in ExitPolicy, except
-    that "accept" is understood unless "reject" is explicitly provided. For
-    example, \'ReachableAddresses 99.0.0.0/8, reject 18.0.0.0/8:80, accept
-    \*:80' means that your firewall allows connections to everything inside net
-    99, rejects port 80 connections to net 18, and accepts connections to port
-    80 otherwise. (Default: \'accept \*:*'.)
-
-[[ReachableDirAddresses]] **ReachableDirAddresses** __IP__[/__MASK__][:__PORT__]...::
-    Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
-    these restrictions when fetching directory information, using standard HTTP
-    GET requests. If not set explicitly then the value of
-    **ReachableAddresses** is used. If **HTTPProxy** is set then these
-    connections will go through that proxy.   (DEPRECATED: This option has
-    had no effect for some time.)
-
-[[ReachableORAddresses]] **ReachableORAddresses** __IP__[/__MASK__][:__PORT__]...::
-    Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
-    these restrictions when connecting to Onion Routers, using TLS/SSL. If not
-    set explicitly then the value of **ReachableAddresses** is used. If
-    **HTTPSProxy** is set then these connections will go through that proxy. +
- +
-    The separation between **ReachableORAddresses** and
-    **ReachableDirAddresses** is only interesting when you are connecting
-    through proxies (see **HTTPProxy** and **HTTPSProxy**). Most proxies limit
-    TLS connections (which Tor uses to connect to Onion Routers) to port 443,
-    and some limit HTTP GET requests (which Tor uses for fetching directory
-    information) to port 80.
-
 [[HidServAuth]] **HidServAuth** __onion-address__ __auth-cookie__ [__service-name__]::
-    Client authorization for a hidden service. Valid onion addresses contain 16
+    Client authorization for a v2 hidden service. Valid onion addresses contain 16
     characters in a-z2-7 plus ".onion", and valid auth cookies contain 22
     characters in A-Za-z0-9+/. The service name is only used for internal
     purposes, e.g., for Tor controllers. This option may be used multiple times
@@ -1092,17 +1188,15 @@ The following options are useful only for clients (that is, if
     services can be configured to require authorization using the
     **HiddenServiceAuthorizeClient** option.
 
-[[ClientOnionAuthDir]] **ClientOnionAuthDir** __path__::
-    Path to the directory containing v3 hidden service authorization files.
-    Each file is for a single onion address, and the files MUST have the suffix
-    ".auth_private" (i.e. "bob_onion.auth_private"). The content format MUST be:
- +
-      <onion-address>:descriptor:x25519:<base32-encoded-privkey>
- +
-    The <onion-address> MUST NOT have the ".onion" suffix. The
-    <base32-encoded-privkey> is the base32 representation of the raw key bytes
-    only (32 bytes for x25519). See Appendix G in the rend-spec-v3.txt file of
-    https://spec.torproject.org/[torspec] for more information.
+[[HTTPTunnelPort]] **HTTPTunnelPort** ['address'**:**]{empty}__port__|**auto** [_isolation flags_]::
+    Open this port to listen for proxy connections using the "HTTP CONNECT"
+    protocol instead of SOCKS. Set this to
+    0 if you don't want to allow "HTTP CONNECT" connections. Set the port
+    to "auto" to have Tor pick a port for you. This directive can be
+    specified multiple times to bind to multiple addresses/ports. If multiple
+    entries of this option are present in your configuration file, Tor will
+    perform stream isolation between listeners by default. See
+    <<SocksPort,SocksPort>> for an explanation of isolation flags. (Default: 0)
 
 [[LongLivedPorts]] **LongLivedPorts** __PORTS__::
     A list of ports for services that tend to have long-running connections
@@ -1126,23 +1220,26 @@ The following options are useful only for clients (that is, if
     "MapAddress \*.example.com \*.example.com.torserver.exit". (Note the
     leading "*." in each part of the directive.) You can also redirect all
     subdomains of a domain to a single address. For example, "MapAddress
-    *.example.com www.example.com". +
- +
+    *.example.com www.example.com". If the specified exit is not available,
+    or the exit can not connect to the site, Tor will fail any connections
+    to the mapped address.+
+     +
     NOTES:
 
     1. When evaluating MapAddress expressions Tor stops when it hits the most
     recently added expression that matches the requested address. So if you
-    have the following in your torrc, www.torproject.org will map to 1.1.1.1:
+    have the following in your torrc, www.torproject.org will map to
+    198.51.100.1:
 
-     MapAddress www.torproject.org 2.2.2.2
-     MapAddress www.torproject.org 1.1.1.1
+     MapAddress www.torproject.org 192.0.2.1
+     MapAddress www.torproject.org 198.51.100.1
 
     2. Tor evaluates the MapAddress configuration until it finds no matches. So
     if you have the following in your torrc, www.torproject.org will map to
-    2.2.2.2:
+    203.0.113.1:
 
-      MapAddress 1.1.1.1 2.2.2.2
-      MapAddress www.torproject.org 1.1.1.1
+      MapAddress 198.51.100.1 203.0.113.1
+      MapAddress www.torproject.org 198.51.100.1
 
     3. The following MapAddress expression is invalid (and will be
     ignored) because you cannot map from a specific address to a wildcard
@@ -1153,9 +1250,14 @@ The following options are useful only for clients (that is, if
     4. Using a wildcard to match only part of a string (as in *ample.com) is
     also invalid.
 
-[[NewCircuitPeriod]] **NewCircuitPeriod** __NUM__::
-    Every NUM seconds consider whether to build a new circuit. (Default: 30
-    seconds)
+    5. Tor maps hostnames and IP addresses separately. If you MapAddress
+       a DNS name, but use an IP address to connect, then Tor will ignore the
+       DNS name mapping.
+
+    6. MapAddress does not apply to redirects in the application protocol.
+       For example, HTTP redirects and alt-svc headers will ignore mappings
+       for the original address. You can use a wildcard mapping to handle
+       redirects within the same site.
 
 [[MaxCircuitDirtiness]] **MaxCircuitDirtiness** __NUM__::
     Feel free to reuse a circuit that was first used at most NUM seconds ago,
@@ -1171,37 +1273,183 @@ The following options are useful only for clients (that is, if
     client streams. A circuit is pending if we have begun constructing it,
     but it has not yet been completely constructed.  (Default: 32)
 
-[[NodeFamily]] **NodeFamily** __node__,__node__,__...__::
-    The Tor servers, defined by their identity fingerprints,
-    constitute a "family" of similar or co-administered servers, so never use
-    any two of them in the same circuit. Defining a NodeFamily is only needed
-    when a server doesn't list the family itself (with MyFamily). This option
-    can be used multiple times; each instance defines a separate family.  In
-    addition to nodes, you can also list IP address and ranges and country
-    codes in {curly braces}. See the **ExcludeNodes** option for more
-    information on how to specify nodes.
+[[NATDPort]] **NATDPort** ['address'**:**]{empty}__port__|**auto** [_isolation flags_]::
+    Open this port to listen for connections from old versions of ipfw (as
+    included in old versions of FreeBSD, etc) using the NATD protocol.
+    Use 0 if you don't want to allow NATD connections.  Set the port
+    to "auto" to have Tor pick a port for you. This directive can be
+    specified multiple times to bind to multiple addresses/ports.  If multiple
+    entries of this option are present in your configuration file, Tor will
+    perform stream isolation between listeners by default. See
+    <<SocksPort,SocksPort>> for an explanation of isolation flags. +
+     +
+    This option is only for people who cannot use TransPort. (Default: 0)
 
-[[EnforceDistinctSubnets]] **EnforceDistinctSubnets** **0**|**1**::
-    If 1, Tor will not put two servers whose IP addresses are "too close" on
-    the same circuit. Currently, two addresses are "too close" if they lie in
-    the same /16 range. (Default: 1)
+[[NewCircuitPeriod]] **NewCircuitPeriod** __NUM__::
+    Every NUM seconds consider whether to build a new circuit. (Default: 30
+    seconds)
+
+[[OptimisticData]] **OptimisticData** **0**|**1**|**auto**::
+    When this option is set, and Tor is using an exit node that supports
+    the feature, it will try optimistically to send data to the exit node
+    without waiting for the exit node to report whether the connection
+    succeeded.  This can save a round-trip time for protocols like HTTP
+    where the client talks first.  If OptimisticData is set to **auto**,
+    Tor will look at the UseOptimisticData parameter in the networkstatus.
+    (Default: auto)
+
+// These are out of order because they logically belong together
+[[PathBiasCircThreshold]] **PathBiasCircThreshold** __NUM__ +
+
+[[PathBiasDropGuards]] **PathBiasDropGuards** __NUM__ +
+
+[[PathBiasExtremeRate]] **PathBiasExtremeRate** __NUM__ +
+
+[[PathBiasNoticeRate]] **PathBiasNoticeRate** __NUM__ +
+
+[[PathBiasWarnRate]] **PathBiasWarnRate** __NUM__ +
+
+[[PathBiasScaleThreshold]] **PathBiasScaleThreshold** __NUM__::
+    These options override the default behavior of Tor's (**currently
+    experimental**) path bias detection algorithm. To try to find broken or
+    misbehaving guard nodes, Tor looks for nodes where more than a certain
+    fraction of circuits through that guard fail to get built. +
+     +
+    The PathBiasCircThreshold option controls how many circuits we need to build
+    through a guard before we make these checks.  The PathBiasNoticeRate,
+    PathBiasWarnRate and PathBiasExtremeRate options control what fraction of
+    circuits must succeed through a guard so we won't write log messages.
+    If less than PathBiasExtremeRate circuits succeed *and* PathBiasDropGuards
+    is set to 1, we disable use of that guard. +
+     +
+    When we have seen more than PathBiasScaleThreshold
+    circuits through a guard, we scale our observations by 0.5 (governed by
+    the consensus) so that new observations don't get swamped by old ones. +
+     +
+    By default, or if a negative value is provided for one of these options,
+    Tor uses reasonable defaults from the networkstatus consensus document.
+    If no defaults are available there, these options default to 150, .70,
+    .50, .30, 0, and 300 respectively.
+
+// These are out of order because they logically belong together
+[[PathBiasUseThreshold]] **PathBiasUseThreshold** __NUM__ +
+
+[[PathBiasNoticeUseRate]] **PathBiasNoticeUseRate** __NUM__ +
+
+[[PathBiasExtremeUseRate]] **PathBiasExtremeUseRate** __NUM__ +
+
+[[PathBiasScaleUseThreshold]] **PathBiasScaleUseThreshold** __NUM__::
+    Similar to the above options, these options override the default behavior
+    of Tor's (**currently experimental**) path use bias detection algorithm. +
+     +
+    Where as the path bias parameters govern thresholds for successfully
+    building circuits, these four path use bias parameters govern thresholds
+    only for circuit usage. Circuits which receive no stream usage
+    are not counted by this detection algorithm. A used circuit is considered
+    successful if it is capable of carrying streams or otherwise receiving
+    well-formed responses to RELAY cells. +
+     +
+    By default, or if a negative value is provided for one of these options,
+    Tor uses reasonable defaults from the networkstatus consensus document.
+    If no defaults are available there, these options default to 20, .80,
+    .60, and 100, respectively.
+
+[[PathsNeededToBuildCircuits]] **PathsNeededToBuildCircuits** __NUM__::
+    Tor clients don't build circuits for user traffic until they know
+    about enough of the network so that they could potentially construct
+    enough of the possible paths through the network. If this option
+    is set to a fraction between 0.25 and 0.95, Tor won't build circuits
+    until it has enough descriptors or microdescriptors to construct
+    that fraction of possible paths. Note that setting this option too low
+    can make your Tor client less anonymous, and setting it too high can
+    prevent your Tor client from bootstrapping.  If this option is negative,
+    Tor will use a default value chosen by the directory authorities. If the
+    directory authorities do not choose a value, Tor will default to 0.6.
+    (Default: -1)
+
+[[ReachableAddresses]] **ReachableAddresses** __IP__[/__MASK__][:__PORT__]...::
+    A comma-separated list of IP addresses and ports that your firewall allows
+    you to connect to. The format is as for the addresses in ExitPolicy, except
+    that "accept" is understood unless "reject" is explicitly provided. For
+    example, \'ReachableAddresses 99.0.0.0/8, reject 18.0.0.0/8:80, accept
+    \*:80' means that your firewall allows connections to everything inside net
+    99, rejects port 80 connections to net 18, and accepts connections to port
+    80 otherwise. (Default: \'accept \*:*'.)
+
+[[ReachableDirAddresses]] **ReachableDirAddresses** __IP__[/__MASK__][:__PORT__]...::
+    Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
+    these restrictions when fetching directory information, using standard HTTP
+    GET requests. If not set explicitly then the value of
+    **ReachableAddresses** is used. If **HTTPProxy** is set then these
+    connections will go through that proxy.   (DEPRECATED: This option has
+    had no effect for some time.)
+
+[[ReachableORAddresses]] **ReachableORAddresses** __IP__[/__MASK__][:__PORT__]...::
+    Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
+    these restrictions when connecting to Onion Routers, using TLS/SSL. If not
+    set explicitly then the value of **ReachableAddresses** is used. If
+    **HTTPSProxy** is set then these connections will go through that proxy. +
+     +
+    The separation between **ReachableORAddresses** and
+    **ReachableDirAddresses** is only interesting when you are connecting
+    through proxies (see <<HTTPProxy,HTTPProxy>> and <<HTTPSProxy,HTTPSProxy>>). Most proxies limit
+    TLS connections (which Tor uses to connect to Onion Routers) to port 443,
+    and some limit HTTP GET requests (which Tor uses for fetching directory
+    information) to port 80.
+
+[[SafeSocks]] **SafeSocks** **0**|**1**::
+    When this option is enabled, Tor will reject application connections that
+    use unsafe variants of the socks protocol -- ones that only provide an IP
+    address, meaning the application is doing a DNS resolve first.
+    Specifically, these are socks4 and socks5 when not doing remote DNS.
+    (Default: 0)
+
+// Out of order because it logically belongs after SafeSocks
+[[TestSocks]] **TestSocks** **0**|**1**::
+    When this option is enabled, Tor will make a notice-level log entry for
+    each connection to the Socks port indicating whether the request used a
+    safe socks protocol or an unsafe one (see <<SafeSocks,SafeSocks>>). This
+    helps to determine whether an application using Tor is possibly leaking
+    DNS requests. (Default: 0)
 
-[[SocksPort]] **SocksPort** \['address':]__port__|**unix:**__path__|**auto** [_flags_] [_isolation flags_]::
+// Out of order because it logically belongs with SafeSocks
+[[WarnPlaintextPorts]] **WarnPlaintextPorts** __port__,__port__,__...__::
+    Tells Tor to issue a warnings whenever the user tries to make an anonymous
+    connection to one of these ports. This option is designed to alert users
+    to services that risk sending passwords in the clear. (Default:
+    23,109,110,143)
+
+// Out of order because it logically belongs with SafeSocks
+[[RejectPlaintextPorts]] **RejectPlaintextPorts** __port__,__port__,__...__::
+    Like WarnPlaintextPorts, but instead of warning about risky port uses, Tor
+    will instead refuse to make the connection. (Default: None)
+
+[[SocksPolicy]] **SocksPolicy** __policy__,__policy__,__...__::
+    Set an entrance policy for this server, to limit who can connect to the
+    SocksPort and DNSPort ports. The policies have the same form as exit
+    policies below, except that port specifiers are ignored. Any address
+    not matched by some entry in the policy is accepted.
+
+[[SocksPort]] **SocksPort** ['address'**:**]{empty}__port__|**unix:**__path__|**auto** [_flags_] [_isolation flags_]::
     Open this port to listen for connections from SOCKS-speaking
     applications. Set this to 0 if you don't want to allow application
     connections via SOCKS. Set it to "auto" to have Tor pick a port for
     you. This directive can be specified multiple times to bind
     to multiple addresses/ports. If a unix domain socket is used, you may
-    quote the path using standard C escape sequences.
+    quote the path using standard C escape sequences. Most flags are off by
+    default, except where specified. Flags that are on by default can be
+    disabled by putting "No" before the flag name.
     (Default: 9050) +
- +
+     +
         NOTE: Although this option allows you to specify an IP address
         other than localhost, you should do so only with extreme caution.
         The SOCKS protocol is unencrypted and (as we use it)
         unauthenticated, so exposing it in this way could leak your
         information to anybody watching your network, and allow anybody
         to use your computer as an open proxy. +
- +
+     +
+    If multiple entries of this option are present in your configuration
+    file, Tor will perform stream isolation between listeners by default.
     The _isolation flags_ arguments give Tor rules for which streams
     received on this SocksPort are allowed to share circuits with one
     another.  Recognized isolation flags are:
@@ -1218,8 +1466,9 @@ The following options are useful only for clients (that is, if
         you can disable it with **NoIsolateSOCKSAuth**.)
     **IsolateClientProtocol**;;
         Don't share circuits with streams using a different protocol.
-        (SOCKS 4, SOCKS 5, TransPort connections, NATDPort connections,
-        and DNSPort requests are all considered to be different protocols.)
+        (SOCKS 4, SOCKS 5, HTTPTunnelPort connections, TransPort connections,
+        NATDPort connections, and DNSPort requests are all considered to be
+        different protocols.)
     **IsolateDestPort**;;
         Don't share circuits with streams targeting a different
         destination port.
@@ -1228,8 +1477,9 @@ The following options are useful only for clients (that is, if
         destination address.
     **KeepAliveIsolateSOCKSAuth**;;
         If **IsolateSOCKSAuth** is enabled, keep alive circuits while they have
-        at least one stream with SOCKS authentication active. After such a circuit
-        is idle for more than MaxCircuitDirtiness seconds, it can be closed.
+        at least one stream with SOCKS authentication active. After such a
+        circuit is idle for more than MaxCircuitDirtiness seconds, it can be
+        closed.
     **SessionGroup=**__INT__;;
         If no other isolation rules would prevent it, allow streams
         on this port to share circuits with streams from every other
@@ -1287,11 +1537,11 @@ The following options are useful only for clients (that is, if
     **UseDNSCache**;;
         Tells the client to use any cached DNS answers we have when making
         requests via this connection.
-    **PreferIPv6Automap**;;
+    **NoPreferIPv6Automap**;;
         When serving a hostname lookup request on this port that
         should get automapped (according to AutomapHostsOnResolve),
         if we could return either an IPv4 or an IPv6 answer, prefer
-        an IPv6 answer. (On by default.)
+        an IPv4 answer. (Tor prefers IPv6 by default.)
     **PreferSOCKSNoAuth**;;
         Ordinarily, when an application offers both "username/password
         authentication" and "no authentication" to Tor via SOCKS5, Tor
@@ -1301,6 +1551,53 @@ The following options are useful only for clients (that is, if
         one. You can disable this behavior, so that Tor will select "No
         authentication" when IsolateSOCKSAuth is disabled, or when this
         option is set.
+    **ExtendedErrors**;;
+        Return extended error code in the SOCKS reply. So far, the possible
+        errors are:
+
+          X'F0' Onion Service Descriptor Can Not be Found
+
+            The requested onion service descriptor can't be found on the
+            hashring and thus not reachable by the client. (v3 only)
+
+          X'F1' Onion Service Descriptor Is Invalid
+
+            The requested onion service descriptor can't be parsed or
+            signature validation failed. (v3 only)
+
+          X'F2' Onion Service Introduction Failed
+
+            All introduction attempts failed either due to a combination of
+            NACK by the intro point or time out. (v3 only)
+
+          X'F3' Onion Service Rendezvous Failed
+
+            Every rendezvous circuit has timed out and thus the client is
+            unable to rendezvous with the service. (v3 only)
+
+          X'F4' Onion Service Missing Client Authorization
+
+            Client was able to download the requested onion service descriptor
+            but is unable to decrypt its content because it is missing client
+            authorization information. (v3 only)
+
+          X'F5' Onion Service Wrong Client Authorization
+
+            Client was able to download the requested onion service descriptor
+            but is unable to decrypt its content using the client
+            authorization information it has. This means the client access
+            were revoked. (v3 only)
+
+          X'F6' Onion Service Invalid Address
+
+            The given .onion address is invalid. In one of these cases this
+            error is returned: address checksum doesn't match, ed25519 public
+            key is invalid or the encoding is invalid. (v3 only)
+
+          X'F7' Onion Service Introduction Timed Out
+
+            Similar to X'F2' code but in this case, all introduction attempts
+            have failed due to a time out. (v3 only)
 
 // Anchor only for formatting, not visible in the man page.
 [[SocksPortFlagsMisc]]::
@@ -1308,17 +1605,6 @@ The following options are useful only for clients (that is, if
     line is used, and all earlier flags are ignored. No error is issued for
     conflicting flags.
 
-[[SocksPolicy]] **SocksPolicy** __policy__,__policy__,__...__::
-    Set an entrance policy for this server, to limit who can connect to the
-    SocksPort and DNSPort ports. The policies have the same form as exit
-    policies below, except that port specifiers are ignored. Any address
-    not matched by some entry in the policy is accepted.
-
-[[SocksTimeout]] **SocksTimeout** __NUM__::
-    Let a socks connection wait NUM seconds handshaking, and NUM seconds
-    unattached waiting for an appropriate circuit, before we fail it. (Default:
-    2 minutes)
-
 [[TokenBucketRefillInterval]] **TokenBucketRefillInterval** __NUM__ [**msec**|**second**]::
     Set the refill delay interval of Tor's token bucket to NUM milliseconds.
     NUM must be between 1 and 1000, inclusive.  When Tor is out of bandwidth,
@@ -1346,6 +1632,44 @@ The following options are useful only for clients (that is, if
     association between host and exit server after NUM seconds. The default is
     1800 seconds (30 minutes).
 
+[[TransPort]] **TransPort** ['address'**:**]{empty}__port__|**auto** [_isolation flags_]::
+    Open this port to listen for transparent proxy connections.  Set this to
+    0 if you don't want to allow transparent proxy connections.  Set the port
+    to "auto" to have Tor pick a port for you. This directive can be
+    specified multiple times to bind to multiple addresses/ports. If multiple
+    entries of this option are present in your configuration file, Tor will
+    perform stream isolation between listeners by default.  See
+    <<SocksPort,SocksPort>> for an explanation of isolation flags. +
+     +
+    TransPort requires OS support for transparent proxies, such as BSDs' pf or
+    Linux's IPTables. If you're planning to use Tor as a transparent proxy for
+    a network, you'll want to examine and change VirtualAddrNetwork from the
+    default setting. (Default: 0)
+
+[[TransProxyType]] **TransProxyType** **default**|**TPROXY**|**ipfw**|**pf-divert**::
+    TransProxyType may only be enabled when there is transparent proxy listener
+    enabled. +
+     +
+    Set this to "TPROXY" if you wish to be able to use the TPROXY Linux module
+    to transparently proxy connections that are configured using the TransPort
+    option. Detailed information on how to configure the TPROXY
+    feature can be found in the Linux kernel source tree in the file
+    Documentation/networking/tproxy.txt. +
+     +
+    Set this option to "ipfw" to use the FreeBSD ipfw interface. +
+     +
+    On *BSD operating systems when using pf, set this to "pf-divert" to take
+    advantage of +divert-to+ rules, which do not modify the packets like
+    +rdr-to+ rules do. Detailed information on how to configure pf to use
+    +divert-to+ rules can be found in the pf.conf(5) manual page. On OpenBSD,
+    +divert-to+ is available to use on versions greater than or equal to
+    OpenBSD 4.4. +
+     +
+    Set this to "default", or leave it unconfigured, to use regular IPTables
+    on Linux, or to use pf +rdr-to+ rules on *BSD systems. +
+     +
+    (Default: "default")
+
 [[UpdateBridgesFromAuthority]] **UpdateBridgesFromAuthority** **0**|**1**::
     When set (along with UseBridges), Tor will try to fetch bridge descriptors
     from the configured bridge authorities when feasible. It will fall back to
@@ -1364,23 +1688,33 @@ The following options are useful only for clients (that is, if
     Authorities or Single Onion Services. In these cases,
     this option is ignored. (Default: 1)
 
-[[GuardfractionFile]] **GuardfractionFile** __FILENAME__::
-    V3 authoritative directories only. Configures the location of the
-    guardfraction file which contains information about how long relays
-    have been guards. (Default: unset)
-
 [[UseGuardFraction]] **UseGuardFraction** **0**|**1**|**auto**::
     This option specifies whether clients should use the
     guardfraction information found in the consensus during path
     selection. If it's set to 'auto', clients will do what the
     UseGuardFraction consensus parameter tells them to do. (Default: auto)
 
+//Out of order because it logically belongs after the UseEntryGuards option
+[[GuardLifetime]] **GuardLifetime**  __N__ **days**|**weeks**|**months**::
+    If UseEntryGuards is set, minimum time to keep a guard on our guard list
+    before  picking a new one.  If less than one day, we use defaults from the
+    consensus directory.  (Default: 0)
+
+//Out of order because it logically belongs after the UseEntryGuards option
+[[NumDirectoryGuards]] **NumDirectoryGuards** __NUM__::
+    If UseEntryGuards is set to 1, we try to make sure we have at least NUM
+    routers to use as directory guards. If this option is set to 0, use the
+    value from the guard-n-primary-dir-guards-to-use consensus parameter, and
+    default to 3 if the consensus parameter isn't set.  (Default: 0)
+
+//Out of order because it logically belongs after the UseEntryGuards option
 [[NumEntryGuards]] **NumEntryGuards** __NUM__::
     If UseEntryGuards is set to 1, we will try to pick a total of NUM routers
     as long-term entries for our circuits. If NUM is 0, we try to learn the
     number from the guard-n-primary-guards-to-use consensus parameter, and
     default to 1 if the consensus parameter isn't set. (Default: 0)
 
+//Out of order because it logically belongs after the UseEntryGuards option
 [[NumPrimaryGuards]] **NumPrimaryGuards** __NUM__::
     If UseEntryGuards is set to 1, we will try to pick NUM routers for our
     primary guard list, which is the set of routers we strongly prefer when
@@ -1388,31 +1722,13 @@ The following options are useful only for clients (that is, if
     the guard-n-primary-guards consensus parameter, and default to 3 if the
     consensus parameter isn't set. (Default: 0)
 
-[[NumDirectoryGuards]] **NumDirectoryGuards** __NUM__::
-    If UseEntryGuards is set to 1, we try to make sure we have at least NUM
-    routers to use as directory guards. If this option is set to 0, use the
-    value from the guard-n-primary-dir-guards-to-use consensus parameter, and
-    default to 3 if the consensus parameter isn't set.  (Default: 0)
-
-[[GuardLifetime]] **GuardLifetime**  __N__ **days**|**weeks**|**months**::
-    If nonzero, and UseEntryGuards is set, minimum time to keep a guard before
-    picking a new one. If zero, we use the GuardLifetime parameter from the
-    consensus directory.  No value here may  be less than 1 month or greater
-    than 5 years; out-of-range values are clamped. (Default: 0)
-
-[[SafeSocks]] **SafeSocks** **0**|**1**::
-    When this option is enabled, Tor will reject application connections that
-    use unsafe variants of the socks protocol -- ones that only provide an IP
-    address, meaning the application is doing a DNS resolve first.
-    Specifically, these are socks4 and socks5 when not doing remote DNS.
-    (Default: 0)
-
-[[TestSocks]] **TestSocks** **0**|**1**::
-    When this option is enabled, Tor will make a notice-level log entry for
-    each connection to the Socks port indicating whether the request used a
-    safe socks protocol or an unsafe one (see above entry on SafeSocks). This
-    helps to determine whether an application using Tor is possibly leaking
-    DNS requests. (Default: 0)
+[[UseMicrodescriptors]] **UseMicrodescriptors** **0**|**1**|**auto**::
+    Microdescriptors are a smaller version of the information that Tor needs
+    in order to build its circuits.  Using microdescriptors makes Tor clients
+    download less directory information, thus saving bandwidth.  Directory
+    caches need to fetch regular descriptors and microdescriptors, so this
+    option doesn't save any bandwidth for them.  For legacy reasons, auto is
+    accepted, but it has the same effect as 1. (Default: auto)
 
 [[VirtualAddrNetworkIPv4]] **VirtualAddrNetworkIPv4** __IPv4Address__/__bits__ +
 
@@ -1421,7 +1737,7 @@ The following options are useful only for clients (that is, if
     command from the controller or the AutomapHostsOnResolve feature, Tor
     picks an unassigned address from this range. (Defaults:
     127.192.0.0/10 and [FE80::]/10 respectively.) +
- +
+     +
     When providing proxy server service to a network of computers using a tool
     like dns-proxy-tor, change the IPv4 network to "10.192.0.0/10" or
     "172.16.0.0/12" and change the IPv6 network to "[FC00::]/7".
@@ -1433,123 +1749,178 @@ The following options are useful only for clients (that is, if
     used IP. For local use, no change to the default VirtualAddrNetwork setting
     is needed.
 
-[[AllowNonRFC953Hostnames]] **AllowNonRFC953Hostnames** **0**|**1**::
-    When this option is disabled, Tor blocks hostnames containing illegal
-    characters (like @ and :) rather than sending them to an exit node to be
-    resolved. This helps trap accidental attempts to resolve URLs and so on.
-    (Default: 0)
+== CIRCUIT TIMEOUT OPTIONS
 
-[[HTTPTunnelPort]] **HTTPTunnelPort**  \['address':]__port__|**auto** [_isolation flags_]::
-    Open this port to listen for proxy connections using the "HTTP CONNECT"
-    protocol instead of SOCKS. Set this to
-    0 if you don't want to allow "HTTP CONNECT" connections. Set the port
-    to "auto" to have Tor pick a port for you. This directive can be
-    specified multiple times to bind to multiple addresses/ports.  See
-    SOCKSPort for an explanation of isolation flags. (Default: 0)
+// These options are in alphabetical order, with exceptions as noted.
+// Please keep them that way!
 
-[[TransPort]] **TransPort**  \['address':]__port__|**auto** [_isolation flags_]::
-    Open this port to listen for transparent proxy connections.  Set this to
-    0 if you don't want to allow transparent proxy connections.  Set the port
-    to "auto" to have Tor pick a port for you. This directive can be
-    specified multiple times to bind to multiple addresses/ports.  See
-    SOCKSPort for an explanation of isolation flags. +
- +
-    TransPort requires OS support for transparent proxies, such as BSDs' pf or
-    Linux's IPTables. If you're planning to use Tor as a transparent proxy for
-    a network, you'll want to examine and change VirtualAddrNetwork from the
-    default setting. (Default: 0)
+The following options are useful for configuring timeouts related
+to building Tor circuits and using them:
 
-[[TransProxyType]] **TransProxyType** **default**|**TPROXY**|**ipfw**|**pf-divert**::
-    TransProxyType may only be enabled when there is transparent proxy listener
-    enabled. +
- +
-    Set this to "TPROXY" if you wish to be able to use the TPROXY Linux module
-    to transparently proxy connections that are configured using the TransPort
-    option. Detailed information on how to configure the TPROXY
-    feature can be found in the Linux kernel source tree in the file
-    Documentation/networking/tproxy.txt. +
- +
-    Set this option to "ipfw" to use the FreeBSD ipfw interface. +
- +
-    On *BSD operating systems when using pf, set this to "pf-divert" to take
-    advantage of +divert-to+ rules, which do not modify the packets like
-    +rdr-to+ rules do. Detailed information on how to configure pf to use
-    +divert-to+ rules can be found in the pf.conf(5) manual page. On OpenBSD,
-    +divert-to+ is available to use on versions greater than or equal to
-    OpenBSD 4.4. +
- +
-    Set this to "default", or leave it unconfigured, to use regular IPTables
-    on Linux, or to use pf +rdr-to+ rules on *BSD systems. +
- +
-    (Default: "default")
+[[CircuitsAvailableTimeout]] **CircuitsAvailableTimeout** __NUM__::
+    Tor will attempt to keep at least one open, unused circuit available for
+    this amount of time. This option governs how long idle circuits are kept
+    open, as well as the amount of time Tor will keep a circuit open to each
+    of the recently used ports. This way when the Tor client is entirely
+    idle, it can expire all of its circuits, and then expire its TLS
+    connections. Note that the actual timeout value is uniformly randomized
+    from the specified value to twice that amount. (Default: 30 minutes;
+    Max: 24 hours)
 
-[[NATDPort]] **NATDPort** \['address':]__port__|**auto** [_isolation flags_]::
-    Open this port to listen for connections from old versions of ipfw (as
-    included in old versions of FreeBSD, etc) using the NATD protocol.
-    Use 0 if you don't want to allow NATD connections.  Set the port
-    to "auto" to have Tor pick a port for you. This directive can be
-    specified multiple times to bind to multiple addresses/ports.  See
-    SocksPort for an explanation of isolation flags. +
- +
-    This option is only for people who cannot use TransPort. (Default: 0)
+// Out of order because it logically belongs before the CircuitBuildTimeout option
+[[LearnCircuitBuildTimeout]] **LearnCircuitBuildTimeout** **0**|**1**::
+    If 0, CircuitBuildTimeout adaptive learning is disabled. (Default: 1)
 
-[[AutomapHostsOnResolve]] **AutomapHostsOnResolve** **0**|**1**::
-    When this option is enabled, and we get a request to resolve an address
-    that ends with one of the suffixes in **AutomapHostsSuffixes**, we map an
-    unused virtual address to that address, and return the new virtual address.
-    This is handy for making ".onion" addresses work with applications that
-    resolve an address and then connect to it. (Default: 0)
+[[CircuitBuildTimeout]] **CircuitBuildTimeout** __NUM__::
+    Try for at most NUM seconds when building circuits. If the circuit isn't
+    open in that time, give up on it. If LearnCircuitBuildTimeout is 1, this
+    value serves as the initial value to use before a timeout is learned. If
+    LearnCircuitBuildTimeout is 0, this value is the only value used.
+    (Default: 60 seconds)
 
-[[AutomapHostsSuffixes]] **AutomapHostsSuffixes** __SUFFIX__,__SUFFIX__,__...__::
-    A comma-separated list of suffixes to use with **AutomapHostsOnResolve**.
-    The "." suffix is equivalent to "all addresses." (Default: .exit,.onion).
+[[CircuitStreamTimeout]] **CircuitStreamTimeout** __NUM__::
+    If non-zero, this option overrides our internal timeout schedule for how
+    many seconds until we detach a stream from a circuit and try a new circuit.
+    If your network is particularly slow, you might want to set this to a
+    number like 60. (Default: 0)
 
-[[DNSPort]] **DNSPort** \['address':]__port__|**auto** [_isolation flags_]::
-    If non-zero, open this port to listen for UDP DNS requests, and resolve
-    them anonymously.  This port only handles A, AAAA, and PTR requests---it
-    doesn't handle arbitrary DNS request types. Set the port to "auto" to
-    have Tor pick a port for
-    you. This directive can be specified multiple times to bind to multiple
-    addresses/ports. See SocksPort for an explanation of isolation
-    flags. (Default: 0)
+[[SocksTimeout]] **SocksTimeout** __NUM__::
+    Let a socks connection wait NUM seconds handshaking, and NUM seconds
+    unattached waiting for an appropriate circuit, before we fail it. (Default:
+    2 minutes)
 
-[[ClientDNSRejectInternalAddresses]] **ClientDNSRejectInternalAddresses** **0**|**1**::
-    If true, Tor does not believe any anonymously retrieved DNS answer that
-    tells it that an address resolves to an internal address (like 127.0.0.1 or
-    192.168.0.1). This option prevents certain browser-based attacks; it
-    is not allowed to be set on the default network. (Default: 1)
+== DORMANT MODE OPTIONS
+
+// These options are in alphabetical order, with exceptions as noted.
+// Please keep them that way!
+
+Tor can enter dormant mode to conserve power and network bandwidth.
+The following options control when Tor enters and leaves dormant mode:
+
+[[DormantCanceledByStartup]] **DormantCanceledByStartup** **0**|**1**::
+    By default, Tor starts in active mode if it was active the last time
+    it was shut down, and in dormant mode if it was dormant.  But if
+    this option is true, Tor treats every startup event as user
+    activity, and Tor will never start in Dormant mode, even if it has
+    been unused for a long time on previous runs. (Default: 0)
+     +
+    Note: Packagers and application developers should change the value of
+    this option only with great caution: it has the potential to
+    create spurious traffic on the network.  This option should only
+    be used if Tor is started by an affirmative user activity (like
+    clicking on an applcation or running a command), and not if Tor
+    is launched for some other reason (for example, by a startup
+    process, or by an application that launches itself on every login.)
+
+[[DormantClientTimeout]] **DormantClientTimeout**  __N__ **minutes**|**hours**|**days**|**weeks**::
+    If Tor spends this much time without any client activity,
+    enter a dormant state where automatic circuits are not built, and
+    directory information is not fetched.
+    Does not affect servers or onion services. Must be at least 10 minutes.
+    (Default: 24 hours)
+
+[[DormantOnFirstStartup]] **DormantOnFirstStartup** **0**|**1**::
+    If true, then the first time Tor starts up with a fresh DataDirectory,
+    it starts in dormant mode, and takes no actions until the user has made
+    a request.  (This mode is recommended if installing a Tor client for a
+    user who might not actually use it.)  If false, Tor bootstraps the first
+    time it is started, whether it sees a user request or not.
+     +
+    After the first time Tor starts, it begins in dormant mode if it was
+    dormant before, and not otherwise. (Default: 0)
+
+[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams**  **0**|**1**::
+    If true, then any open client stream (even one not reading or writing)
+    counts as client activity for the purpose of DormantClientTimeout.
+    If false, then only network activity counts. (Default: 1)
+
+== NODE SELECTION OPTIONS
+
+// These options are in alphabetical order, with exceptions as noted.
+// Please keep them that way!
+
+The following options restrict the nodes that a tor client
+(or onion service) can use while building a circuit.
+These options can weaken your anonymity by making your client behavior
+different from other Tor clients:
 
-[[ClientRejectInternalAddresses]] **ClientRejectInternalAddresses** **0**|**1**::
-    If true, Tor does not try to fulfill requests to connect to an internal
-    address (like 127.0.0.1 or 192.168.0.1) __unless an exit node is
-    specifically requested__ (for example, via a .exit hostname, or a
-    controller request).  If true, multicast DNS hostnames for machines on the
-    local network (of the form *.local) are also rejected.  (Default: 1)
+[[EntryNodes]] **EntryNodes** __node__,__node__,__...__::
+    A list of identity fingerprints and country codes of nodes
+    to use for the first hop in your normal circuits.
+    Normal circuits include all
+    circuits except for direct connections to directory servers.  The Bridge
+    option overrides this option; if you have configured bridges and
+    UseBridges is 1, the Bridges are used as your entry nodes. +
+     +
+    The ExcludeNodes option overrides this option: any node listed in both
+    EntryNodes and ExcludeNodes is treated as excluded. See
+    <<ExcludeNodes,ExcludeNodes>> for more information on how to specify nodes.
 
-[[DownloadExtraInfo]] **DownloadExtraInfo** **0**|**1**::
-    If true, Tor downloads and caches "extra-info" documents. These documents
-    contain information about servers other than the information in their
-    regular server descriptors. Tor does not use this information for anything
-    itself; to save bandwidth, leave this option turned off. (Default: 0)
+[[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__::
+    A list of identity fingerprints, country codes, and address
+    patterns of nodes to avoid when building a circuit. Country codes are
+    2-letter ISO3166 codes, and must
+    be wrapped in braces; fingerprints may be preceded by a dollar sign.
+    (Example:
+    ExcludeNodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
+     +
+    By default, this option is treated as a preference that Tor is allowed
+    to override in order to keep working.
+    For example, if you try to connect to a hidden service,
+    but you have excluded all of the hidden service's introduction points,
+    Tor will connect to one of them anyway.  If you do not want this
+    behavior, set the StrictNodes option (documented below).  +
+     +
+    Note also that if you are a relay, this (and the other node selection
+    options below) only affects your own circuits that Tor builds for you.
+    Clients can still build circuits through you to any node.  Controllers
+    can tell Tor to build circuits through any node. +
+     +
+    Country codes are case-insensitive. The code "\{??}" refers to nodes whose
+    country can't be identified. No country code, including \{??}, works if
+    no GeoIPFile can be loaded. See also the <<GeoIPExcludeUnknown,GeoIPExcludeUnknown>> option below.
 
-[[WarnPlaintextPorts]] **WarnPlaintextPorts** __port__,__port__,__...__::
-    Tells Tor to issue a warnings whenever the user tries to make an anonymous
-    connection to one of these ports. This option is designed to alert users
-    to services that risk sending passwords in the clear. (Default:
-    23,109,110,143)
+// Out of order because it logically belongs after the ExcludeNodes option
+[[ExcludeExitNodes]] **ExcludeExitNodes** __node__,__node__,__...__::
+    A list of identity fingerprints, country codes, and address
+    patterns of nodes to never use when picking an exit node---that is, a
+    node that delivers traffic for you *outside* the Tor network.   Note that any
+    node listed in ExcludeNodes is automatically considered to be part of this
+    list too.  See
+    <<ExcludeNodes,ExcludeNodes>> for more information on how to specify
+    nodes. See also the caveats on the <<ExitNodes,ExitNodes>> option below.
 
-[[RejectPlaintextPorts]] **RejectPlaintextPorts** __port__,__port__,__...__::
-    Like WarnPlaintextPorts, but instead of warning about risky port uses, Tor
-    will instead refuse to make the connection. (Default: None)
+[[ExitNodes]] **ExitNodes** __node__,__node__,__...__::
+    A list of identity fingerprints, country codes, and address
+    patterns of nodes to use as exit node---that is, a
+    node that delivers traffic for you *outside* the Tor network.  See
+    <<ExcludeNodes,ExcludeNodes>> for more information on how to specify nodes. +
+     +
+    Note that if you list too few nodes here, or if you exclude too many exit
+    nodes with ExcludeExitNodes, you can degrade functionality.  For example,
+    if none of the exits you list allows traffic on port 80 or 443, you won't
+    be able to browse the web. +
+     +
+    Note also that not every circuit is used to deliver traffic *outside* of
+    the Tor network.  It is normal to see non-exit circuits (such as those
+    used to connect to hidden services, those that do directory fetches,
+    those used for relay reachability self-tests, and so on) that end
+    at a non-exit node.  To
+    keep a node from being used entirely, see <<ExcludeNodes,ExcludeNodes>> and <<StrictNodes,StrictNodes>>. +
+     +
+    The ExcludeNodes option overrides this option: any node listed in both
+    ExitNodes and ExcludeNodes is treated as excluded. +
+     +
+    The .exit address notation, if enabled via MapAddress, overrides
+    this option.
 
-[[OptimisticData]] **OptimisticData** **0**|**1**|**auto**::
-    When this option is set, and Tor is using an exit node that supports
-    the feature, it will try optimistically to send data to the exit node
-    without waiting for the exit node to report whether the connection
-    succeeded.  This can save a round-trip time for protocols like HTTP
-    where the client talks first.  If OptimisticData is set to **auto**,
-    Tor will look at the UseOptimisticData parameter in the networkstatus.
-    (Default: auto)
+[[GeoIPExcludeUnknown]] **GeoIPExcludeUnknown** **0**|**1**|**auto**::
+    If this option is set to 'auto', then whenever any country code is set in
+    ExcludeNodes or ExcludeExitNodes, all nodes with unknown country (\{??} and
+    possibly \{A1}) are treated as excluded as well. If this option is set to
+    '1', then all unknown countries are treated as excluded in ExcludeNodes
+    and ExcludeExitNodes.  This option has no effect when a GeoIP file isn't
+    configured or can't be found.  (Default: auto)
 
 [[HSLayer2Nodes]] **HSLayer2Nodes** __node__,__node__,__...__::
     A list of identity fingerprints, nicknames, country codes, and
@@ -1559,39 +1930,39 @@ The following options are useful only for clients (that is, if
     and induces your client or service to create many circuits, in order
     to discover your primary guard node.
     (Default: Any node in the network may be used in the second hop.)
- +
+     +
     (Example:
     HSLayer2Nodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
- +
+     +
     When this is set, the resulting hidden service paths will
     look like:
- +
+     +
         C - G - L2 - M - Rend +
         C - G - L2 - M - HSDir +
         C - G - L2 - M - Intro +
         S - G - L2 - M - Rend +
         S - G - L2 - M - HSDir +
         S - G - L2 - M - Intro +
- +
+     +
     where C is this client, S is the service, G is the Guard node,
     L2 is a node from this option, and M is a random middle node.
     Rend, HSDir, and Intro point selection is not affected by this
     option.
- +
+     +
     This option may be combined with HSLayer3Nodes to create
     paths of the form:
- +
+     +
         C - G - L2 - L3 - Rend +
         C - G - L2 - L3 - M - HSDir +
         C - G - L2 - L3 - M - Intro +
         S - G - L2 - L3 - M - Rend +
         S - G - L2 - L3 - HSDir +
         S - G - L2 - L3 - Intro +
- +
+     +
     ExcludeNodes have higher priority than HSLayer2Nodes,
     which means that nodes specified in ExcludeNodes will not be
     picked.
- +
+     +
     When either this option or HSLayer3Nodes are set, the /16 subnet
     and node family restrictions are removed for hidden service
     circuits. Additionally, we allow the guard node to be present
@@ -1599,7 +1970,7 @@ The following options are useful only for clients (that is, if
     is done to prevent the adversary from inferring information
     about our guard, layer2, and layer3 node choices at later points
     in the path.
- +
+     +
     This option is meant to be managed by a Tor controller such as
     https://github.com/mikeperry-tor/vanguards that selects and
     updates this set of nodes for you. Hence it does not do load
@@ -1615,10 +1986,10 @@ The following options are useful only for clients (that is, if
     and induces your client or service to create many circuits, in order
     to discover your primary or Layer2 guard nodes.
     (Default: Any node in the network may be used in the third hop.)
- +
+     +
     (Example:
     HSLayer3Nodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
- +
+     +
     When this is set by itself, the resulting hidden service paths
     will look like: +
         C - G - M - L3 - Rend +
@@ -1631,21 +2002,21 @@ The following options are useful only for clients (that is, if
     L2 is a node from this option, and M is a random middle node.
     Rend, HSDir, and Intro point selection is not affected by this
     option.
- +
+     +
     While it is possible to use this option by itself, it should be
     combined with HSLayer2Nodes to create paths of the form:
- +
+     +
         C - G - L2 - L3 - Rend +
         C - G - L2 - L3 - M - HSDir +
         C - G - L2 - L3 - M - Intro +
         S - G - L2 - L3 - M - Rend +
         S - G - L2 - L3 - HSDir +
         S - G - L2 - L3 - Intro +
- +
+     +
     ExcludeNodes have higher priority than HSLayer3Nodes,
     which means that nodes specified in ExcludeNodes will not be
     picked.
- +
+     +
     When either this option or HSLayer2Nodes are set, the /16 subnet
     and node family restrictions are removed for hidden service
     circuits. Additionally, we allow the guard node to be present
@@ -1661,144 +2032,103 @@ The following options are useful only for clients (that is, if
     HSLayer3Nodes are currently available for use, Tor will not work.
     Please use extreme care if you are setting this option manually.
 
-[[UseMicrodescriptors]] **UseMicrodescriptors** **0**|**1**|**auto**::
-    Microdescriptors are a smaller version of the information that Tor needs
-    in order to build its circuits.  Using microdescriptors makes Tor clients
-    download less directory information, thus saving bandwidth.  Directory
-    caches need to fetch regular descriptors and microdescriptors, so this
-    option doesn't save any bandwidth for them.  For legacy reasons, auto is
-    accepted, but it has the same effect as 1. (Default: auto)
-
-[[PathBiasCircThreshold]] **PathBiasCircThreshold** __NUM__ +
-
-[[PathBiasNoticeRate]] **PathBiasNoticeRate** __NUM__ +
-
-[[PathBiasWarnRate]] **PathBiasWarnRate** __NUM__ +
-
-[[PathBiasExtremeRate]] **PathBiasExtremeRate** __NUM__ +
-
-[[PathBiasDropGuards]] **PathBiasDropGuards** __NUM__ +
-
-[[PathBiasScaleThreshold]] **PathBiasScaleThreshold** __NUM__::
-    These options override the default behavior of Tor's (**currently
-    experimental**) path bias detection algorithm. To try to find broken or
-    misbehaving guard nodes, Tor looks for nodes where more than a certain
-    fraction of circuits through that guard fail to get built. +
- +
-    The PathBiasCircThreshold option controls how many circuits we need to build
-    through a guard before we make these checks.  The PathBiasNoticeRate,
-    PathBiasWarnRate and PathBiasExtremeRate options control what fraction of
-    circuits must succeed through a guard so we won't write log messages.
-    If less than PathBiasExtremeRate circuits succeed *and* PathBiasDropGuards
-    is set to 1, we disable use of that guard. +
- +
-    When we have seen more than PathBiasScaleThreshold
-    circuits through a guard, we scale our observations by 0.5 (governed by
-    the consensus) so that new observations don't get swamped by old ones. +
- +
-    By default, or if a negative value is provided for one of these options,
-    Tor uses reasonable defaults from the networkstatus consensus document.
-    If no defaults are available there, these options default to 150, .70,
-    .50, .30, 0, and 300 respectively.
-
-[[PathBiasUseThreshold]] **PathBiasUseThreshold** __NUM__ +
-
-[[PathBiasNoticeUseRate]] **PathBiasNoticeUseRate** __NUM__ +
-
-[[PathBiasExtremeUseRate]] **PathBiasExtremeUseRate** __NUM__ +
-
-[[PathBiasScaleUseThreshold]] **PathBiasScaleUseThreshold** __NUM__::
-    Similar to the above options, these options override the default behavior
-    of Tor's (**currently experimental**) path use bias detection algorithm. +
- +
-    Where as the path bias parameters govern thresholds for successfully
-    building circuits, these four path use bias parameters govern thresholds
-    only for circuit usage. Circuits which receive no stream usage
-    are not counted by this detection algorithm. A used circuit is considered
-    successful if it is capable of carrying streams or otherwise receiving
-    well-formed responses to RELAY cells. +
- +
-    By default, or if a negative value is provided for one of these options,
-    Tor uses reasonable defaults from the networkstatus consensus document.
-    If no defaults are available there, these options default to 20, .80,
-    .60, and 100, respectively.
-
-[[ClientUseIPv4]] **ClientUseIPv4** **0**|**1**::
-    If this option is set to 0, Tor will avoid connecting to directory servers
-    and entry nodes over IPv4. Note that clients with an IPv4
-    address in a **Bridge**, proxy, or pluggable transport line will try
-    connecting over IPv4 even if **ClientUseIPv4** is set to 0. (Default: 1)
-
-[[ClientUseIPv6]] **ClientUseIPv6** **0**|**1**::
-    If this option is set to 1, Tor might connect to directory servers or
-    entry nodes over IPv6. For IPv6 only hosts, you need to also set
-    **ClientUseIPv4** to 0 to disable IPv4. Note that clients configured with
-    an IPv6 address in a **Bridge**, proxy, or pluggable transportline will
-    try connecting over IPv6 even if **ClientUseIPv6** is set to 0. (Default: 0)
-
-[[ClientPreferIPv6DirPort]] **ClientPreferIPv6DirPort** **0**|**1**|**auto**::
-    If this option is set to 1, Tor prefers a directory port with an IPv6
-    address over one with IPv4, for direct connections, if a given directory
-    server has both. (Tor also prefers an IPv6 DirPort if IPv4Client is set to
-    0.) If this option is set to auto, clients prefer IPv4. Other things may
-    influence the choice. This option breaks a tie to the favor of IPv6.
-    (Default: auto)  (DEPRECATED: This option has had no effect for some
-    time.)
-
-[[ClientPreferIPv6ORPort]] **ClientPreferIPv6ORPort** **0**|**1**|**auto**::
-    If this option is set to 1, Tor prefers an OR port with an IPv6
-    address over one with IPv4 if a given entry node has both. (Tor also
-    prefers an IPv6 ORPort if IPv4Client is set to 0.) If this option is set
-    to auto, Tor bridge clients prefer the configured bridge address, and
-    other clients prefer IPv4. Other things may influence the choice. This
-    option breaks a tie to the favor of IPv6. (Default: auto)
-
-[[PathsNeededToBuildCircuits]] **PathsNeededToBuildCircuits** __NUM__::
-    Tor clients don't build circuits for user traffic until they know
-    about enough of the network so that they could potentially construct
-    enough of the possible paths through the network. If this option
-    is set to a fraction between 0.25 and 0.95, Tor won't build circuits
-    until it has enough descriptors or microdescriptors to construct
-    that fraction of possible paths. Note that setting this option too low
-    can make your Tor client less anonymous, and setting it too high can
-    prevent your Tor client from bootstrapping.  If this option is negative,
-    Tor will use a default value chosen by the directory authorities. If the
-    directory authorities do not choose a value, Tor will default to 0.6.
-    (Default: -1)
+[[MiddleNodes]] **MiddleNodes** __node__,__node__,__...__::
+    A list of identity fingerprints and country codes of nodes
+    to use for "middle" hops in your normal circuits.
+    Normal circuits include all circuits except for direct connections
+    to directory servers. Middle hops are all hops other than exit and entry. +
++
+    This is an **experimental** feature that is meant to be used by researchers
+    and developers to test new features in the Tor network safely. Using it
+    without care will strongly influence your anonymity. This feature might get
+    removed in the future.
++
+    The HSLayer2Node and HSLayer3Node options override this option for onion
+    service circuits, if they are set. The vanguards addon will read this
+    option, and if set, it will set HSLayer2Nodes and HSLayer3Nodes to nodes
+    from this set.
++
+    The ExcludeNodes option overrides this option: any node listed in both
+    MiddleNodes and ExcludeNodes is treated as excluded. See
+    the <<ExcludeNodes,ExcludeNodes>> for more information on how to specify nodes.
 
-[[ClientBootstrapConsensusAuthorityDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses from authorities
-    if they are bootstrapping (that is, they don't have a usable, reasonably
-    live consensus). Only used by clients fetching from a list of fallback
-    directory mirrors. This schedule is advanced by (potentially concurrent)
-    connection attempts, unlike other schedules, which are advanced by
-    connection failures. (Default: 6)
+[[NodeFamily]] **NodeFamily** __node__,__node__,__...__::
+    The Tor servers, defined by their identity fingerprints,
+    constitute a "family" of similar or co-administered servers, so never use
+    any two of them in the same circuit. Defining a NodeFamily is only needed
+    when a server doesn't list the family itself (with MyFamily). This option
+    can be used multiple times; each instance defines a separate family.  In
+    addition to nodes, you can also list IP address and ranges and country
+    codes in {curly braces}. See <<ExcludeNodes,ExcludeNodes>> for more
+    information on how to specify nodes.
 
-[[ClientBootstrapConsensusFallbackDownloadInitialDelay]] **ClientBootstrapConsensusFallbackDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses from fallback
-    directory mirrors if they are bootstrapping (that is, they don't have a
-    usable, reasonably live consensus). Only used by clients fetching from a
-    list of fallback directory mirrors. This schedule is advanced by
-    (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0)
+[[StrictNodes]] **StrictNodes** **0**|**1**::
+    If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option
+    as a requirement to follow for all the circuits you generate, even if
+    doing so will break functionality for you (StrictNodes does not apply to
+    ExcludeExitNodes, ExitNodes, MiddleNodes, or MapAddress).  If StrictNodes
+    is set to 0, Tor will still try to avoid nodes in the ExcludeNodes list,
+    but it will err on the side of avoiding unexpected errors.
+    Specifically, StrictNodes 0 tells Tor that it is okay to use an excluded
+    node when it is *necessary* to perform relay reachability self-tests,
+    connect to a hidden service, provide a hidden service to a client,
+    fulfill a .exit request, upload directory information, or download
+    directory information.  (Default: 0)
+
+[[server-options]]
+== SERVER OPTIONS
+
+// These options are in alphabetical order, with exceptions as noted.
+// Please keep them that way!
 
-[[ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses from authorities
-    if they are bootstrapping (that is, they don't have a usable, reasonably
-    live consensus). Only used by clients which don't have or won't fetch
-    from a list of fallback directory mirrors. This schedule is advanced by
-    (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0)
+The following options are useful only for servers (that is, if ORPort
+is non-zero):
 
-[[ClientBootstrapConsensusMaxInProgressTries]] **ClientBootstrapConsensusMaxInProgressTries** __NUM__::
-    Try this many simultaneous connections to download a consensus before
-    waiting for one to complete, timeout, or error out. (Default: 3)
+[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    Limits the max number of bytes sent and received within a set time period
+    using a given calculation rule (see <<AccountingStart,AccountingStart>> and <<AccountingRule,AccountingRule>>).
+    Useful if you need to stay under a specific bandwidth. By default, the
+    number used for calculation is the max of either the bytes sent or
+    received. For example, with AccountingMax set to 1 TByte, a server
+    could send 900 GBytes and receive 800 GBytes and continue running.
+    It will only hibernate once one of the two reaches 1 TByte. This can
+    be changed to use the sum of the both bytes received and sent by setting
+    the AccountingRule option to "sum" (total bandwidth in/out). When the
+    number of bytes remaining gets low, Tor will stop accepting new connections
+    and circuits. When the number of bytes is exhausted, Tor will hibernate
+    until some time in the next accounting period. To prevent all servers
+    from waking at the same time, Tor will also wait until a random point
+    in each period before waking up. If you have bandwidth cost issues,
+    enabling hibernation is preferable to setting a low bandwidth, since
+    it provides users with a collection of fast servers that are up some
+    of the time, which is more useful than a set of slow servers that are
+    always "available". +
+     +
+    Note that (as also described in the Bandwidth section) Tor uses
+    powers of two, not powers of ten: 1 GByte is 1024*1024*1024, not
+    one billion. Be careful: some internet service providers might count
+    GBytes differently.
 
-SERVER OPTIONS
---------------
+[[AccountingRule]] **AccountingRule** **sum**|**max**|**in**|**out**::
+    How we determine when our AccountingMax has been reached (when we
+    should hibernate) during a time interval. Set to "max" to calculate
+    using the higher of either the sent or received bytes (this is the
+    default functionality). Set to "sum" to calculate using the sent
+    plus received bytes. Set to "in" to calculate using only the
+    received bytes. Set to "out" to calculate using only the sent bytes.
+    (Default: max)
 
-The following options are useful only for servers (that is, if ORPort
-is non-zero):
+[[AccountingStart]] **AccountingStart** **day**|**week**|**month** [__day__] __HH:MM__::
+    Specify how long accounting periods last. If **month** is given,
+    each accounting period runs from the time __HH:MM__ on the __dayth__ day of one
+    month to the same day and time of the next. The relay will go at full speed,
+    use all the quota you specify, then hibernate for the rest of the period. (The
+    day must be between 1 and 28.) If **week** is given, each accounting period
+    runs from the time __HH:MM__ of the __dayth__ day of one week to the same day
+    and time of the next week, with Monday as day 1 and Sunday as day 7. If **day**
+    is given, each accounting period runs from the time __HH:MM__ each day to the
+    same time on the next day. All times are local, and given in 24-hour time.
+    (Default: "month 1 0:00")
 
 [[Address]] **Address** __address__::
     The IPv4 address of this server, or a fully qualified domain name of
@@ -1821,20 +2151,26 @@ is non-zero):
     from bridge users to the Tor network. It mainly causes Tor to publish a
     server descriptor to the bridge database, rather than
     to the public directory authorities. +
- +
+     +
     Note: make sure that no MyFamily lines are present in your torrc when
     relay is configured in bridge mode.
 
+//Out of order because it logically belongs after BridgeRelay.
+[[BridgeRecordUsageByCountry]] **BridgeRecordUsageByCountry** **0**|**1**::
+    When this option is enabled and BridgeRelay is also enabled, and we have
+    GeoIP data, Tor keeps a per-country count of how many client
+    addresses have contacted it so that it can help the bridge authority guess
+    which countries have blocked access to it. If ExtraInfoStatistics is
+    enabled, it will be published as part of the extra-info document.
+    (Default: 1)
+
+//Out of order because it logically belongs after BridgeRelay.
 [[BridgeDistribution]] **BridgeDistribution** __string__::
     If set along with BridgeRelay, Tor will include a new line in its
     bridge descriptor which indicates to the BridgeDB service how it
     would like its bridge address to be given out. Set it to "none" if
     you want BridgeDB to avoid distributing your bridge address, or "any" to
     let BridgeDB decide. (Default: any)
- +
-    Note: as of Oct 2017, the BridgeDB part of this option is not yet
-    implemented.  Until BridgeDB is updated to obey this option, your
-    bridge will make this request, but it will not (yet) be obeyed.
 
 [[ContactInfo]] **ContactInfo** __email_address__::
     Administrative contact information for this relay or bridge. This line
@@ -1844,25 +2180,16 @@ is non-zero):
     spammers might also collect them. You may want to obscure the fact
     that it's an email address and/or generate a new address for this
     purpose. +
- +
+     +
     ContactInfo **must** be set to a working address if you run more than one
     relay or bridge.  (Really, everybody running a relay or bridge should set
     it.)
 
-
-[[ExitRelay]] **ExitRelay** **0**|**1**|**auto**::
-    Tells Tor whether to run as an exit relay.  If Tor is running as a
-    non-bridge server, and ExitRelay is set to 1, then Tor allows traffic to
-    exit according to the ExitPolicy option, the ReducedExitPolicy option,
-    or the default ExitPolicy (if no other exit policy option is specified). +
- +
-    If ExitRelay is set to 0, no traffic is allowed to
-    exit, and the ExitPolicy and ReducedExitPolicy options are ignored. +
- +
-    If ExitRelay is set to "auto", then Tor checks the ExitPolicy and
-    ReducedExitPolicy options. If either is set, Tor behaves as if ExitRelay
-    were set to 1. If neither exit policy option is set, Tor behaves as if
-    ExitRelay were set to 0. (Default: auto)
+[[DisableOOSCheck]] **DisableOOSCheck** **0**|**1**::
+    This option disables the code that closes connections when Tor notices
+    that it is running low on sockets. Right now, it is on by default,
+    since the existing out-of-sockets mechanism tends to kill OR connections
+    more than it should. (Default: 1)
 
 [[ExitPolicy]] **ExitPolicy** __policy__,__policy__,__...__::
     Set an exit policy for this server. Each policy is of the form
@@ -1874,29 +2201,29 @@ is non-zero):
     __PORT__ can be a single port number, an interval of ports
     "__FROM_PORT__-__TO_PORT__", or "\*". If __PORT__ is omitted, that means
     "\*". +
- +
+     +
     For example, "accept 18.7.22.69:\*,reject 18.0.0.0/8:\*,accept \*:\*" would
     reject any IPv4 traffic destined for MIT except for web.mit.edu, and accept
     any other IPv4 or IPv6 traffic. +
- +
+     +
     Tor also allows IPv6 exit policy entries. For instance, "reject6 [FC00::]/7:\*"
     rejects all destinations that share 7 most significant bit prefix with
     address FC00::. Respectively, "accept6 [C000::]/3:\*" accepts all destinations
     that share 3 most significant bit prefix with address C000::. +
- +
+     +
     accept6 and reject6 only produce IPv6 exit policy entries. Using an IPv4
     address with accept6 or reject6 is ignored and generates a warning.
     accept/reject allows either IPv4 or IPv6 addresses. Use \*4 as an IPv4
     wildcard address, and \*6 as an IPv6 wildcard address. accept/reject *
     expands to matching IPv4 and IPv6 wildcard address rules. +
- +
+     +
     To specify all IPv4 and IPv6 internal and link-local networks (including
     0.0.0.0/8, 169.254.0.0/16, 127.0.0.0/8, 192.168.0.0/16, 10.0.0.0/8,
     172.16.0.0/12, [::]/8, [FC00::]/7, [FE80::]/10, [FEC0::]/10, [FF00::]/8,
     and [::]/127), you can use the "private" alias instead of an address.
     ("private" always produces rules for IPv4 and IPv6 addresses, even when
     used with accept6/reject6.) +
- +
+     +
     Private addresses are rejected by default (at the beginning of your exit
     policy), along with any configured primary public IPv4 and IPv6 addresses.
     These private addresses are rejected unless you set the
@@ -1906,12 +2233,12 @@ is non-zero):
     may also allow connections to your own computer that are addressed to its
     public (external) IP address. See RFC 1918 and RFC 3330 for more details
     about internal and reserved IP address space. See
-    ExitPolicyRejectLocalInterfaces if you want to block every address on the
+    <<ExitPolicyRejectLocalInterfaces,ExitPolicyRejectLocalInterfaces>> if you want to block every address on the
     relay, even those that aren't advertised in the descriptor. +
- +
+     +
     This directive can be specified multiple times so you don't have to put it
     all on one line. +
- +
+     +
     Policies are considered first to last, and the first match wins. If you
     want to allow the same ports on IPv4 and IPv6, write your rules using
     accept/reject \*. If you want to allow different ports on IPv4 and IPv6,
@@ -1919,13 +2246,13 @@ is non-zero):
     accept/reject \*4. If you want to \_replace_ the default exit policy, end
     your exit policy with either a reject \*:* or an accept \*:*. Otherwise,
     you're \_augmenting_ (prepending to) the default exit policy. +
- +
+     +
     If you want to use a reduced exit policy rather than the default exit
     policy, set "ReducedExitPolicy 1". If you want to _replace_ the default
     exit policy with your custom exit policy, end your exit policy with either
     a reject *:* or an accept *:*. Otherwise, you're _augmenting_ (prepending
     to) the default or reduced exit policy. +
- +
+     +
     The default exit policy is:
 
        reject *:25
@@ -1945,33 +2272,186 @@ is non-zero):
     Since the default exit policy uses accept/reject *, it applies to both
     IPv4 and IPv6 addresses.
 
-[[ExitPolicyRejectPrivate]] **ExitPolicyRejectPrivate** **0**|**1**::
-    Reject all private (local) networks, along with the relay's advertised
-    public IPv4 and IPv6 addresses, at the beginning of your exit policy.
-    See above entry on ExitPolicy.
-    (Default: 1)
-
 [[ExitPolicyRejectLocalInterfaces]] **ExitPolicyRejectLocalInterfaces** **0**|**1**::
     Reject all IPv4 and IPv6 addresses that the relay knows about, at the
     beginning of your exit policy. This includes any OutboundBindAddress, the
     bind addresses of any port options, such as ControlPort or DNSPort, and any
     public IPv4 and IPv6 addresses on any interface on the relay. (If IPv6Exit
     is not set, all IPv6 addresses will be rejected anyway.)
-    See above entry on ExitPolicy.
+    See above entry on <<ExitPolicy,ExitPolicy>>.
     This option is off by default, because it lists all public relay IP
     addresses in the ExitPolicy, even those relay operators might prefer not
     to disclose.
     (Default: 0)
 
+[[ExitPolicyRejectPrivate]] **ExitPolicyRejectPrivate** **0**|**1**::
+    Reject all private (local) networks, along with the relay's advertised
+    public IPv4 and IPv6 addresses, at the beginning of your exit policy.
+    See above entry on <<ExitPolicy,ExitPolicy>>.
+    (Default: 1)
+
+[[ExitRelay]] **ExitRelay** **0**|**1**|**auto**::
+    Tells Tor whether to run as an exit relay.  If Tor is running as a
+    non-bridge server, and ExitRelay is set to 1, then Tor allows traffic to
+    exit according to the ExitPolicy option, the ReducedExitPolicy option,
+    or the default ExitPolicy (if no other exit policy option is specified). +
+     +
+    If ExitRelay is set to 0, no traffic is allowed to exit, and the
+    ExitPolicy, ReducedExitPolicy, and IPv6Exit options are ignored. +
+     +
+    If ExitRelay is set to "auto", then Tor checks the ExitPolicy,
+    ReducedExitPolicy, and IPv6Exit options. If at least one of these options
+    is set, Tor behaves as if ExitRelay were set to 1. If none of these exit
+    policy options are set, Tor behaves as if ExitRelay were set to 0.
+    (Default: auto)
+
+[[ExtendAllowPrivateAddresses]] **ExtendAllowPrivateAddresses** **0**|**1**::
+    When this option is enabled, Tor will connect to relays on localhost,
+    RFC1918 addresses, and so on. In particular, Tor will make direct OR
+    connections, and Tor routers allow EXTEND requests, to these private
+    addresses. (Tor will always allow connections to bridges, proxies, and
+    pluggable transports configured on private addresses.) Enabling this
+    option can create security issues; you should probably leave it off.
+    (Default: 0)
+
+[[GeoIPFile]] **GeoIPFile** __filename__::
+    A filename containing IPv4 GeoIP data, for use with by-country statistics.
+
+[[GeoIPv6File]] **GeoIPv6File** __filename__::
+    A filename containing IPv6 GeoIP data, for use with by-country statistics.
+
+[[HeartbeatPeriod]] **HeartbeatPeriod**  __N__ **minutes**|**hours**|**days**|**weeks**::
+    Log a heartbeat message every **HeartbeatPeriod** seconds. This is
+    a log level __notice__ message, designed to let you know your Tor
+    server is still alive and doing useful things. Settings this
+    to 0 will disable the heartbeat. Otherwise, it must be at least 30
+    minutes. (Default: 6 hours)
+
+[[IPv6Exit]] **IPv6Exit** **0**|**1**::
+    If set, and we are an exit node, allow clients to use us for IPv6 traffic.
+    When this option is set and ExitRelay is auto, we act as if ExitRelay
+    is 1. (Default: 0)
+
+[[KeyDirectory]] **KeyDirectory** __DIR__::
+    Store secret keys in DIR. Can not be changed while tor is
+    running.
+    (Default: the "keys" subdirectory of DataDirectory.)
+
+[[KeyDirectoryGroupReadable]] **KeyDirectoryGroupReadable** **0**|**1**|**auto**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    KeyDirectory. If the option is set to 1, make the KeyDirectory readable
+    by the default GID. If the option is "auto", then we use the
+    setting for DataDirectoryGroupReadable when the KeyDirectory is the
+    same as the DataDirectory, and 0 otherwise. (Default: auto)
+
+[[MainloopStats]] **MainloopStats** **0**|**1**::
+    Log main loop statistics every **HeartbeatPeriod** seconds. This is a log
+    level __notice__ message designed to help developers instrumenting Tor's
+    main event loop. (Default: 0)
+
+[[MaxMemInQueues]] **MaxMemInQueues**  __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**::
+    This option configures a threshold above which Tor will assume that it
+    needs to stop queueing or buffering data because it's about to run out of
+    memory.  If it hits this threshold, it will begin killing circuits until
+    it has recovered at least 10% of this memory.  Do not set this option too
+    low, or your relay may be unreliable under load.  This option only
+    affects some queues, so the actual process size will be larger than
+    this.  If this option is set to 0, Tor will try to pick a reasonable
+    default based on your system's physical memory.  (Default: 0)
+
+[[MaxOnionQueueDelay]] **MaxOnionQueueDelay** __NUM__ [**msec**|**second**]::
+    If we have more onionskins queued for processing than we can process in
+    this amount of time, reject new ones. (Default: 1750 msec)
+
+[[MyFamily]] **MyFamily** __fingerprint__,__fingerprint__,...::
+    Declare that this Tor relay is controlled or administered by a group or
+    organization identical or similar to that of the other relays, defined by
+    their (possibly $-prefixed) identity fingerprints.
+    This option can be repeated many times, for
+    convenience in defining large families: all fingerprints in all MyFamily
+    lines are merged into one list.
+    When two relays both declare that they are in the
+    same \'family', Tor clients will not use them in the same circuit. (Each
+    relay only needs to list the other servers in its family; it doesn't need to
+    list itself, but it won't hurt if it does.) Do not list any bridge relay as it would
+    compromise its concealment. +
+     +
+    If you run more than one relay, the MyFamily option on each relay
+    **must** list all other relays, as described above. +
+     +
+    Note: do not use MyFamily when configuring your Tor instance as a
+    bridge.
+
+[[Nickname]] **Nickname** __name__::
+    Set the server's nickname to \'name'. Nicknames must be between 1 and 19
+    characters inclusive, and must contain only the characters [a-zA-Z0-9].
+    If not set, **Unnamed** will be used. Relays can always be uniquely identified
+    by their identity fingerprints.
+
+[[NumCPUs]] **NumCPUs** __num__::
+    How many processes to use at once for decrypting onionskins and other
+    parallelizable operations.  If this is set to 0, Tor will try to detect
+    how many CPUs you have, defaulting to 1 if it can't tell.  (Default: 0)
+
+[[OfflineMasterKey]] **OfflineMasterKey** **0**|**1**::
+    If non-zero, the Tor relay will never generate or load its master secret
+    key.  Instead, you'll have to use "tor --keygen" to manage the permanent
+    ed25519 master identity key, as well as the corresponding temporary
+    signing keys and certificates. (Default: 0)
+
+[[ORPort]] **ORPort** ['address'**:**]{empty}__PORT__|**auto** [_flags_]::
+    Advertise this port to listen for connections from Tor clients and
+    servers.  This option is required to be a Tor server.
+    Set it to "auto" to have Tor pick a port for you. Set it to 0 to not
+    run an ORPort at all. This option can occur more than once. (Default: 0) +
+     +
+    Tor recognizes these flags on each ORPort:
+    **NoAdvertise**;;
+        By default, we bind to a port and tell our users about it. If
+        NoAdvertise is specified, we don't advertise, but listen anyway.  This
+        can be useful if the port everybody will be connecting to (for
+        example, one that's opened on our firewall) is somewhere else.
+    **NoListen**;;
+        By default, we bind to a port and tell our users about it. If
+        NoListen is specified, we don't bind, but advertise anyway.  This
+        can be useful if something else  (for example, a firewall's port
+        forwarding configuration) is causing connections to reach us.
+    **IPv4Only**;;
+        If the address is absent, or resolves to both an IPv4 and an IPv6
+        address, only listen to the IPv4 address.
+    **IPv6Only**;;
+        If the address is absent, or resolves to both an IPv4 and an IPv6
+        address, only listen to the IPv6 address.
+
+// Anchor only for formatting, not visible in the man page.
+[[ORPortFlagsExclusive]]::
+    For obvious reasons, NoAdvertise and NoListen are mutually exclusive, and
+    IPv4Only and IPv6Only are mutually exclusive.
+
+[[PublishServerDescriptor]] **PublishServerDescriptor** **0**|**1**|**v3**|**bridge**,**...**::
+    This option specifies which descriptors Tor will publish when acting as
+    a relay. You can
+    choose multiple arguments, separated by commas. +
+     +
+    If this option is set to 0, Tor will not publish its
+    descriptors to any directories. (This is useful if you're testing
+    out your server, or if you're using a Tor controller that handles
+    directory publishing for you.) Otherwise, Tor will publish its
+    descriptors of all type(s) specified. The default is "1", which
+    means "if running as a relay or bridge, publish descriptors to the
+    appropriate authorities". Other possibilities are "v3", meaning
+    "publish as if you're a relay", and "bridge", meaning "publish as
+    if you're a bridge".
+
 [[ReducedExitPolicy]] **ReducedExitPolicy** **0**|**1**::
     If set, use a reduced exit policy rather than the default one. +
- +
+     +
     The reduced exit policy is an alternative to the default exit policy. It
     allows as many Internet services as possible while still blocking the
     majority of TCP ports. Currently, the policy allows approximately 65 ports.
     This reduces the odds that your node will be used for peer-to-peer
     applications. +
- +
+     +
     The reduced exit policy is:
 
         accept *:20-21
@@ -2056,155 +2536,6 @@ is non-zero):
 
     (Default: 0)
 
-[[IPv6Exit]] **IPv6Exit** **0**|**1**::
-    If set, and we are an exit node, allow clients to use us for IPv6
-    traffic. (Default: 0)
-
-[[MaxOnionQueueDelay]] **MaxOnionQueueDelay** __NUM__ [**msec**|**second**]::
-    If we have more onionskins queued for processing than we can process in
-    this amount of time, reject new ones. (Default: 1750 msec)
-
-[[MyFamily]] **MyFamily** __fingerprint__,__fingerprint__,...::
-    Declare that this Tor relay is controlled or administered by a group or
-    organization identical or similar to that of the other relays, defined by
-    their (possibly $-prefixed) identity fingerprints.
-    This option can be repeated many times, for
-    convenience in defining large families: all fingerprints in all MyFamily
-    lines are merged into one list.
-    When two relays both declare that they are in the
-    same \'family', Tor clients will not use them in the same circuit. (Each
-    relay only needs to list the other servers in its family; it doesn't need to
-    list itself, but it won't hurt if it does.) Do not list any bridge relay as it would
-    compromise its concealment. +
- +
-    When listing a node, it's better to list it by fingerprint than by
-    nickname: fingerprints are more reliable. +
- +
-    If you run more than one relay, the MyFamily option on each relay
-    **must** list all other relays, as described above. +
- +
-    Note: do not use MyFamily when configuring your Tor instance as a
-    brigde.
-
-[[Nickname]] **Nickname** __name__::
-    Set the server's nickname to \'name'. Nicknames must be between 1 and 19
-    characters inclusive, and must contain only the characters [a-zA-Z0-9].
-    If not set, **Unnamed** will be used. Relays can always be uniquely identified
-    by their identity fingerprints.
-
-[[NumCPUs]] **NumCPUs** __num__::
-    How many processes to use at once for decrypting onionskins and other
-    parallelizable operations.  If this is set to 0, Tor will try to detect
-    how many CPUs you have, defaulting to 1 if it can't tell.  (Default: 0)
-
-[[ORPort]] **ORPort** \['address':]__PORT__|**auto** [_flags_]::
-    Advertise this port to listen for connections from Tor clients and
-    servers.  This option is required to be a Tor server.
-    Set it to "auto" to have Tor pick a port for you. Set it to 0 to not
-    run an ORPort at all. This option can occur more than once. (Default: 0) +
- +
-    Tor recognizes these flags on each ORPort:
-    **NoAdvertise**;;
-        By default, we bind to a port and tell our users about it. If
-        NoAdvertise is specified, we don't advertise, but listen anyway.  This
-        can be useful if the port everybody will be connecting to (for
-        example, one that's opened on our firewall) is somewhere else.
-    **NoListen**;;
-        By default, we bind to a port and tell our users about it. If
-        NoListen is specified, we don't bind, but advertise anyway.  This
-        can be useful if something else  (for example, a firewall's port
-        forwarding configuration) is causing connections to reach us.
-    **IPv4Only**;;
-        If the address is absent, or resolves to both an IPv4 and an IPv6
-        address, only listen to the IPv4 address.
-    **IPv6Only**;;
-        If the address is absent, or resolves to both an IPv4 and an IPv6
-        address, only listen to the IPv6 address.
-
-// Anchor only for formatting, not visible in the man page.
-[[ORPortFlagsExclusive]]::
-    For obvious reasons, NoAdvertise and NoListen are mutually exclusive, and
-    IPv4Only and IPv6Only are mutually exclusive.
-
-[[PublishServerDescriptor]] **PublishServerDescriptor** **0**|**1**|**v3**|**bridge**,**...**::
-    This option specifies which descriptors Tor will publish when acting as
-    a relay. You can
-    choose multiple arguments, separated by commas. +
- +
-    If this option is set to 0, Tor will not publish its
-    descriptors to any directories. (This is useful if you're testing
-    out your server, or if you're using a Tor controller that handles
-    directory publishing for you.) Otherwise, Tor will publish its
-    descriptors of all type(s) specified. The default is "1", which
-    means "if running as a relay or bridge, publish descriptors to the
-    appropriate authorities". Other possibilities are "v3", meaning
-    "publish as if you're a relay", and "bridge", meaning "publish as
-    if you're a bridge".
-
-[[ShutdownWaitLength]] **ShutdownWaitLength** __NUM__::
-    When we get a SIGINT and we're a server, we begin shutting down:
-    we close listeners and start refusing new circuits. After **NUM**
-    seconds, we exit. If we get a second SIGINT, we exit immediately.
-    (Default: 30 seconds)
-
-[[SSLKeyLifetime]] **SSLKeyLifetime** __N__ **minutes**|**hours**|**days**|**weeks**::
-    When creating a link certificate for our outermost SSL handshake,
-    set its lifetime to this amount of time. If set to 0, Tor will choose
-    some reasonable random defaults. (Default: 0)
-
-[[HeartbeatPeriod]] **HeartbeatPeriod**  __N__ **minutes**|**hours**|**days**|**weeks**::
-    Log a heartbeat message every **HeartbeatPeriod** seconds. This is
-    a log level __notice__ message, designed to let you know your Tor
-    server is still alive and doing useful things. Settings this
-    to 0 will disable the heartbeat. Otherwise, it must be at least 30
-    minutes. (Default: 6 hours)
-
-[[MainloopStats]] **MainloopStats** **0**|**1**::
-    Log main loop statistics every **HeartbeatPeriod** seconds. This is a log
-    level __notice__ message designed to help developers instrumenting Tor's
-    main event loop. (Default: 0)
-
-[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    Limits the max number of bytes sent and received within a set time period
-    using a given calculation rule (see: AccountingStart, AccountingRule).
-    Useful if you need to stay under a specific bandwidth. By default, the
-    number used for calculation is the max of either the bytes sent or
-    received. For example, with AccountingMax set to 1 GByte, a server
-    could send 900 MBytes and receive 800 MBytes and continue running.
-    It will only hibernate once one of the two reaches 1 GByte. This can
-    be changed to use the sum of the both bytes received and sent by setting
-    the AccountingRule option to "sum" (total bandwidth in/out). When the
-    number of bytes remaining gets low, Tor will stop accepting new connections
-    and circuits. When the number of bytes is exhausted, Tor will hibernate
-    until some time in the next accounting period. To prevent all servers
-    from waking at the same time, Tor will also wait until a random point
-    in each period before waking up. If you have bandwidth cost issues,
-    enabling hibernation is preferable to setting a low bandwidth, since
-    it provides users with a collection of fast servers that are up some
-    of the time, which is more useful than a set of slow servers that are
-    always "available".
-
-[[AccountingRule]] **AccountingRule** **sum**|**max**|**in**|**out**::
-    How we determine when our AccountingMax has been reached (when we
-    should hibernate) during a time interval. Set to "max" to calculate
-    using the higher of either the sent or received bytes (this is the
-    default functionality). Set to "sum" to calculate using the sent
-    plus received bytes. Set to "in" to calculate using only the
-    received bytes. Set to "out" to calculate using only the sent bytes.
-    (Default: max)
-
-[[AccountingStart]] **AccountingStart** **day**|**week**|**month** [__day__] __HH:MM__::
-    Specify how long accounting periods last. If **month** is given,
-    each accounting period runs from the time __HH:MM__ on the __dayth__ day of one
-    month to the same day and time of the next. The relay will go at full speed,
-    use all the quota you specify, then hibernate for the rest of the period. (The
-    day must be between 1 and 28.) If **week** is given, each accounting period
-    runs from the time __HH:MM__ of the __dayth__ day of one week to the same day
-    and time of the next week, with Monday as day 1 and Sunday as day 7. If **day**
-    is given, each accounting period runs from the time __HH:MM__ each day to the
-    same time on the next day. All times are local, and given in 24-hour time.
-    (Default: "month 1 0:00")
-
 [[RefuseUnknownExits]] **RefuseUnknownExits** **0**|**1**|**auto**::
     Prevent nodes that don't appear in the consensus from exiting using this
     relay.  If the option is 1, we always block exit attempts from such
@@ -2212,25 +2543,18 @@ is non-zero):
     whatever the authorities suggest in the consensus (and block if the consensus
     is quiet on the issue). (Default: auto)
 
-[[ServerDNSResolvConfFile]] **ServerDNSResolvConfFile** __filename__::
-    Overrides the default DNS configuration with the configuration in
-    __filename__. The file format is the same as the standard Unix
-    "**resolv.conf**" file (7). This option, like all other ServerDNS options,
-    only affects name lookups that your server does on behalf of clients.
-    (Defaults to use the system DNS configuration.)
-
 [[ServerDNSAllowBrokenConfig]] **ServerDNSAllowBrokenConfig** **0**|**1**::
     If this option is false, Tor exits immediately if there are problems
     parsing the system DNS configuration or connecting to nameservers.
     Otherwise, Tor continues to periodically retry the system nameservers until
     it eventually succeeds. (Default: 1)
 
-[[ServerDNSSearchDomains]] **ServerDNSSearchDomains** **0**|**1**::
-    If set to 1, then we will search for addresses in the local search domain.
-    For example, if this system is configured to believe it is in
-    "example.com", and a client tries to connect to "www", the client will be
-    connected to "www.example.com". This option only affects name lookups that
-    your server does on behalf of clients. (Default: 0)
+[[ServerDNSAllowNonRFC953Hostnames]] **ServerDNSAllowNonRFC953Hostnames** **0**|**1**::
+    When this option is disabled, Tor does not try to resolve hostnames
+    containing illegal characters (like @ and :) rather than sending them to an
+    exit node to be resolved. This helps trap accidental attempts to resolve
+    URLs and so on. This option only affects name lookups that your server does
+    on behalf of clients. (Default: 0)
 
 [[ServerDNSDetectHijacking]] **ServerDNSDetectHijacking** **0**|**1**::
     When this option is set to 1, we will test periodically to determine
@@ -2239,6 +2563,29 @@ is non-zero):
     correct this. This option only affects name lookups that your server does
     on behalf of clients. (Default: 1)
 
+[[ServerDNSRandomizeCase]] **ServerDNSRandomizeCase** **0**|**1**::
+    When this option is set, Tor sets the case of each character randomly in
+    outgoing DNS requests, and makes sure that the case matches in DNS replies.
+    This so-called "0x20 hack" helps resist some types of DNS poisoning attack.
+    For more information, see "Increased DNS Forgery Resistance through
+    0x20-Bit Encoding". This option only affects name lookups that your server
+    does on behalf of clients. (Default: 1)
+
+[[ServerDNSResolvConfFile]] **ServerDNSResolvConfFile** __filename__::
+    Overrides the default DNS configuration with the configuration in
+    __filename__. The file format is the same as the standard Unix
+    "**resolv.conf**" file (7). This option, like all other ServerDNS options,
+    only affects name lookups that your server does on behalf of clients.
+    (Defaults to use the system DNS configuration or a localhost DNS service
+    in case no nameservers are found in a given configuration.)
+
+[[ServerDNSSearchDomains]] **ServerDNSSearchDomains** **0**|**1**::
+    If set to 1, then we will search for addresses in the local search domain.
+    For example, if this system is configured to believe it is in
+    "example.com", and a client tries to connect to "www", the client will be
+    connected to "www.example.com". This option only affects name lookups that
+    your server does on behalf of clients. (Default: 0)
+
 [[ServerDNSTestAddresses]] **ServerDNSTestAddresses** __hostname__,__hostname__,__...__::
     When we're detecting DNS hijacking, make sure that these __valid__ addresses
     aren't getting redirected. If they are, then our DNS is completely useless,
@@ -2246,32 +2593,32 @@ is non-zero):
     name lookups that your server does on behalf of clients. (Default:
     "www.google.com, www.mit.edu, www.yahoo.com, www.slashdot.org")
 
-[[ServerDNSAllowNonRFC953Hostnames]] **ServerDNSAllowNonRFC953Hostnames** **0**|**1**::
-    When this option is disabled, Tor does not try to resolve hostnames
-    containing illegal characters (like @ and :) rather than sending them to an
-    exit node to be resolved. This helps trap accidental attempts to resolve
-    URLs and so on. This option only affects name lookups that your server does
-    on behalf of clients. (Default: 0)
+[[ShutdownWaitLength]] **ShutdownWaitLength** __NUM__::
+    When we get a SIGINT and we're a server, we begin shutting down:
+    we close listeners and start refusing new circuits. After **NUM**
+    seconds, we exit. If we get a second SIGINT, we exit immediately.
+    (Default: 30 seconds)
 
-[[BridgeRecordUsageByCountry]] **BridgeRecordUsageByCountry** **0**|**1**::
-    When this option is enabled and BridgeRelay is also enabled, and we have
-    GeoIP data, Tor keeps a per-country count of how many client
-    addresses have contacted it so that it can help the bridge authority guess
-    which countries have blocked access to it. (Default: 1)
+[[SigningKeyLifetime]] **SigningKeyLifetime** __N__ **days**|**weeks**|**months**::
+    For how long should each Ed25519 signing key be valid?  Tor uses a
+    permanent master identity key that can be kept offline, and periodically
+    generates new "signing" keys that it uses online.  This option
+    configures their lifetime.
+    (Default: 30 days)
 
-[[ServerDNSRandomizeCase]] **ServerDNSRandomizeCase** **0**|**1**::
-    When this option is set, Tor sets the case of each character randomly in
-    outgoing DNS requests, and makes sure that the case matches in DNS replies.
-    This so-called "0x20 hack" helps resist some types of DNS poisoning attack.
-    For more information, see "Increased DNS Forgery Resistance through
-    0x20-Bit Encoding". This option only affects name lookups that your server
-    does on behalf of clients. (Default: 1)
+[[SSLKeyLifetime]] **SSLKeyLifetime** __N__ **minutes**|**hours**|**days**|**weeks**::
+    When creating a link certificate for our outermost SSL handshake,
+    set its lifetime to this amount of time. If set to 0, Tor will choose
+    some reasonable random defaults. (Default: 0)
 
-[[GeoIPFile]] **GeoIPFile** __filename__::
-    A filename containing IPv4 GeoIP data, for use with by-country statistics.
+== STATISTICS OPTIONS
 
-[[GeoIPv6File]] **GeoIPv6File** __filename__::
-    A filename containing IPv6 GeoIP data, for use with by-country statistics.
+// These options are in alphabetical order, with exceptions as noted.
+// Please keep them that way!
+
+Relays publish most statistics in a document called the
+extra-info document. The following options affect the different
+types of statistics that Tor relays collect and publish:
 
 [[CellStatistics]] **CellStatistics** **0**|**1**::
     Relays only.
@@ -2281,16 +2628,16 @@ is non-zero):
     circuit) and writes them into disk every 24 hours. Onion router
     operators may use the statistics for performance monitoring.
     If ExtraInfoStatistics is enabled, it will published as part of
-    extra-info document. (Default: 0)
+    the extra-info document. (Default: 0)
 
-[[PaddingStatistics]] **PaddingStatistics** **0**|**1**::
-    Relays and bridges only.
-    When this option is enabled, Tor collects statistics for padding cells
-    sent and received by this relay, in addition to total cell counts.
-    These statistics are rounded, and omitted if traffic is low. This
-    information is important for load balancing decisions related to padding.
-    If ExtraInfoStatistics is enabled, it will be published
-    as a part of extra-info document. (Default: 1)
+[[ConnDirectionStatistics]] **ConnDirectionStatistics** **0**|**1**::
+    Relays only.
+    When this option is enabled, Tor writes statistics on the amounts of
+    traffic it passes between itself and other relays to disk every 24
+    hours. Enables relay operators to monitor how much their relay is
+    being used as middle node in the circuit. If ExtraInfoStatistics is
+    enabled, it will be published as part of the extra-info document.
+    (Default: 0)
 
 [[DirReqStatistics]] **DirReqStatistics** **0**|**1**::
     Relays and bridges only.
@@ -2299,7 +2646,7 @@ is non-zero):
     hours. Enables relay and bridge operators to monitor how much their
     server is being used by clients to learn about Tor network.
     If ExtraInfoStatistics is enabled, it will published as part of
-    extra-info document. (Default: 1)
+    the extra-info document. (Default: 1)
 
 [[EntryStatistics]] **EntryStatistics** **0**|**1**::
     Relays only.
@@ -2308,7 +2655,7 @@ is non-zero):
     operators to monitor how much inbound traffic that originates from
     Tor clients passes through their server to go further down the
     Tor network. If ExtraInfoStatistics is enabled, it will be published
-    as part of extra-info document. (Default: 0)
+    as part of the extra-info document. (Default: 0)
 
 [[ExitPortStatistics]] **ExitPortStatistics** **0**|**1**::
     Exit relays only.
@@ -2316,112 +2663,41 @@ is non-zero):
     relayed bytes and opened stream per exit port to disk every 24 hours.
     Enables exit relay operators to measure and monitor amounts of traffic
     that leaves Tor network through their exit node. If ExtraInfoStatistics
-    is enabled, it will be published as part of extra-info document.
+    is enabled, it will be published as part of the extra-info document.
     (Default: 0)
 
-[[ConnDirectionStatistics]] **ConnDirectionStatistics** **0**|**1**::
-    Relays only.
-    When this option is enabled, Tor writes statistics on the amounts of
-    traffic it passes between itself and other relays to disk every 24
-    hours. Enables relay operators to monitor how much their relay is
-    being used as middle node in the circuit. If ExtraInfoStatistics is
-    enabled, it will be published as part of extra-info document.
-    (Default: 0)
+[[ExtraInfoStatistics]] **ExtraInfoStatistics** **0**|**1**::
+    When this option is enabled, Tor includes previously gathered statistics in
+    its extra-info documents that it uploads to the directory authorities.
+    Disabling this option also removes bandwidth usage statistics, and
+    GeoIPFile and GeoIPv6File hashes from the extra-info file. Bridge
+    ServerTransportPlugin lines are always included in the extra-info file,
+    because they are required by BridgeDB.
+    (Default: 1)
 
 [[HiddenServiceStatistics]] **HiddenServiceStatistics** **0**|**1**::
     Relays only.
     When this option is enabled, a Tor relay writes obfuscated
     statistics on its role as hidden-service directory, introduction
-    point, or rendezvous point to disk every 24 hours. If
-    ExtraInfoStatistics is also enabled, these statistics are further
-    published to the directory authorities. (Default: 1)
-
-[[ExtraInfoStatistics]] **ExtraInfoStatistics** **0**|**1**::
-    When this option is enabled, Tor includes previously gathered statistics in
-    its extra-info documents that it uploads to the directory authorities.
+    point, or rendezvous point to disk every 24 hours.  If ExtraInfoStatistics
+    is enabled, it will be published as part of the extra-info document.
     (Default: 1)
 
-[[ExtendAllowPrivateAddresses]] **ExtendAllowPrivateAddresses** **0**|**1**::
-    When this option is enabled, Tor will connect to relays on localhost,
-    RFC1918 addresses, and so on. In particular, Tor will make direct OR
-    connections, and Tor routers allow EXTEND requests, to these private
-    addresses. (Tor will always allow connections to bridges, proxies, and
-    pluggable transports configured on private addresses.) Enabling this
-    option can create security issues; you should probably leave it off.
-    (Default: 0)
-
-[[MaxMemInQueues]] **MaxMemInQueues**  __N__ **bytes**|**KB**|**MB**|**GB**::
-    This option configures a threshold above which Tor will assume that it
-    needs to stop queueing or buffering data because it's about to run out of
-    memory.  If it hits this threshold, it will begin killing circuits until
-    it has recovered at least 10% of this memory.  Do not set this option too
-    low, or your relay may be unreliable under load.  This option only
-    affects some queues, so the actual process size will be larger than
-    this.  If this option is set to 0, Tor will try to pick a reasonable
-    default based on your system's physical memory.  (Default: 0)
-
-[[DisableOOSCheck]] **DisableOOSCheck** **0**|**1**::
-    This option disables the code that closes connections when Tor notices
-    that it is running low on sockets. Right now, it is on by default,
-    since the existing out-of-sockets mechanism tends to kill OR connections
-    more than it should. (Default: 1)
-
-[[SigningKeyLifetime]] **SigningKeyLifetime** __N__ **days**|**weeks**|**months**::
-    For how long should each Ed25519 signing key be valid?  Tor uses a
-    permanent master identity key that can be kept offline, and periodically
-    generates new "signing" keys that it uses online.  This option
-    configures their lifetime.
-    (Default: 30 days)
-
-[[OfflineMasterKey]] **OfflineMasterKey** **0**|**1**::
-    If non-zero, the Tor relay will never generate or load its master secret
-    key.  Instead, you'll have to use "tor --keygen" to manage the permanent
-    ed25519 master identity key, as well as the corresponding temporary
-    signing keys and certificates. (Default: 0)
-
-[[KeyDirectory]] **KeyDirectory** __DIR__::
-    Store secret keys in DIR. Can not be changed while tor is
-    running.
-    (Default: the "keys" subdirectory of DataDirectory.)
-
-[[KeyDirectoryGroupReadable]] **KeyDirectoryGroupReadable** **0**|**1**::
-    If this option is set to 0, don't allow the filesystem group to read the
-    KeywDirectory. If the option is set to 1, make the KeyDirectory readable
-    by the default GID. (Default: 0)
-
-[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
-    Tells an authority, or other node tracking node reliability and history,
-    that fine-grained information about nodes can be discarded when it hasn't
-    changed for a given amount of time.  (Default: 24 hours)
-
+[[PaddingStatistics]] **PaddingStatistics** **0**|**1**::
+    Relays and bridges only.
+    When this option is enabled, Tor collects statistics for padding cells
+    sent and received by this relay, in addition to total cell counts.
+    These statistics are rounded, and omitted if traffic is low. This
+    information is important for load balancing decisions related to padding.
+    If ExtraInfoStatistics is enabled, it will be published
+    as a part of the extra-info document. (Default: 1)
 
-DIRECTORY SERVER OPTIONS
-------------------------
+== DIRECTORY SERVER OPTIONS
 
 The following options are useful only for directory servers. (Relays with
-enough bandwidth automatically become directory servers; see DirCache for
+enough bandwidth automatically become directory servers; see <<DirCache,DirCache>> for
 details.)
 
-[[DirPortFrontPage]] **DirPortFrontPage** __FILENAME__::
-    When this option is set, it takes an HTML file and publishes it as "/" on
-    the DirPort. Now relay operators can provide a disclaimer without needing
-    to set up a separate webserver. There's a sample disclaimer in
-    contrib/operator-tools/tor-exit-notice.html.
-
-[[DirPort]] **DirPort** \['address':]__PORT__|**auto** [_flags_]::
-    If this option is nonzero, advertise the directory service on this port.
-    Set it to "auto" to have Tor pick a port for you.  This option can occur
-    more than once, but only one advertised DirPort is supported: all
-    but one DirPort must have the **NoAdvertise** flag set. (Default: 0) +
- +
-    The same flags are supported here as are supported by ORPort.
-
-[[DirPolicy]] **DirPolicy** __policy__,__policy__,__...__::
-    Set an entrance policy for this server, to limit who can connect to the
-    directory ports. The policies have the same form as exit policies above,
-    except that port specifiers are ignored. Any address not matched by
-    some entry in the policy is accepted.
-
 [[DirCache]] **DirCache** **0**|**1**::
     When this option is set, Tor caches all current directory documents except
     extra info documents, and accepts client requests for them. If
@@ -2430,6 +2706,26 @@ details.)
     connect via the ORPort by default. Setting either DirPort or BridgeRelay
     and setting DirCache to 0 is not supported.  (Default: 1)
 
+[[DirPolicy]] **DirPolicy** __policy__,__policy__,__...__::
+    Set an entrance policy for this server, to limit who can connect to the
+    directory ports. The policies have the same form as exit policies above,
+    except that port specifiers are ignored. Any address not matched by
+    some entry in the policy is accepted.
+
+[[DirPort]] **DirPort** ['address'**:**]{empty}__PORT__|**auto** [_flags_]::
+    If this option is nonzero, advertise the directory service on this port.
+    Set it to "auto" to have Tor pick a port for you.  This option can occur
+    more than once, but only one advertised DirPort is supported: all
+    but one DirPort must have the **NoAdvertise** flag set. (Default: 0) +
+     +
+    The same flags are supported here as are supported by ORPort.
+
+[[DirPortFrontPage]] **DirPortFrontPage** __FILENAME__::
+    When this option is set, it takes an HTML file and publishes it as "/" on
+    the DirPort. Now relay operators can provide a disclaimer without needing
+    to set up a separate webserver. There's a sample disclaimer in
+    contrib/operator-tools/tor-exit-notice.html.
+
 [[MaxConsensusAgeForDiffs]] **MaxConsensusAgeForDiffs**  __N__ **minutes**|**hours**|**days**|**weeks**::
     When this option is nonzero, Tor caches will not try to generate
     consensus diffs for any consensus older than this amount of time.
@@ -2441,8 +2737,7 @@ details.)
     (Default: 0)
 
 
-DENIAL OF SERVICE MITIGATION OPTIONS
-------------------------------------
+== DENIAL OF SERVICE MITIGATION OPTIONS
 
 Tor has three built-in mitigation options that can be individually
 enabled/disabled and fine-tuned, but by default Tor directory authorities will
@@ -2453,14 +2748,16 @@ and are as follows:
   1. If a single client address makes too many concurrent connections (this is
      configurable via DoSConnectionMaxConcurrentCount), hang up on further
      connections.
- +
+     +
   2. If a single client IP address (v4 or v6) makes circuits too quickly
      (default values are more than 3 per second, with an allowed burst of 90,
-     see DoSCircuitCreationRate and DoSCircuitCreationBurst) while also having
+     see <<DoSCircuitCreationRate,DoSCircuitCreationRate>> and
+     <<DoSCircuitCreationBurst,DoSCircuitCreationBurst>>) while also having
      too many connections open (default is 3, see
-     DoSCircuitCreationMinConnections), tor will refuse any new circuit (CREATE
+     <<DoSCircuitCreationMinConnections,DoSCircuitCreationMinConnections>>),
+     tor will refuse any new circuit (CREATE
      cells) for the next while (random value between 1 and 2 hours).
- +
+     +
   3. If a client asks to establish a rendezvous point to you directly (ex:
      Tor2Web client), ignore the request.
 
@@ -2480,32 +2777,17 @@ your log at NOTICE level which looks like:
 The following options are useful only for a public relay. They control the
 Denial of Service mitigation subsystem described above.
 
+//Out of order because it logically belongs before the other DoSCircuitCreation options.
 [[DoSCircuitCreationEnabled]] **DoSCircuitCreationEnabled** **0**|**1**|**auto**::
 
     Enable circuit creation DoS mitigation. If set to 1 (enabled), tor will
     cache client IPs along with statistics in order to detect circuit DoS
     attacks. If an address is positively identified, tor will activate
-    defenses against the address. See the DoSCircuitCreationDefenseType option
-    for more details.  This is a client to relay detection only. "auto" means
+    defenses against the address. See <<DoSCircuitCreationDefenseType,DoSCircuitCreationDefenseType>>
+     option for more details.  This is a client to relay detection only. "auto" means
     use the consensus parameter. If not defined in the consensus, the value is 0.
     (Default: auto)
 
-[[DoSCircuitCreationMinConnections]] **DoSCircuitCreationMinConnections** __NUM__::
-
-    Minimum threshold of concurrent connections before a client address can be
-    flagged as executing a circuit creation DoS. In other words, once a client
-    address reaches the circuit rate and has a minimum of NUM concurrent
-    connections, a detection is positive. "0" means use the consensus
-    parameter. If not defined in the consensus, the value is 3.
-    (Default: 0)
-
-[[DoSCircuitCreationRate]] **DoSCircuitCreationRate** __NUM__::
-
-    The allowed circuit creation rate per second applied per client IP
-    address. If this option is 0, it obeys a consensus parameter. If not
-    defined in the consensus, the value is 3.
-    (Default: 0)
-
 [[DoSCircuitCreationBurst]] **DoSCircuitCreationBurst** __NUM__::
 
     The allowed circuit creation burst per client IP address. If the circuit
@@ -2514,26 +2796,43 @@ Denial of Service mitigation subsystem described above.
     consensus, the value is 90.
     (Default: 0)
 
+[[DoSCircuitCreationDefenseTimePeriod]] **DoSCircuitCreationDefenseTimePeriod** __N__ **seconds**|**minutes**|**hours**::
+
+    The base time period in seconds that the DoS defense is activated for. The
+    actual value is selected randomly for each activation from N+1 to 3/2 * N.
+    "0" means use the consensus parameter. If not defined in the consensus,
+    the value is 3600 seconds (1 hour).
+    (Default: 0)
+
 [[DoSCircuitCreationDefenseType]] **DoSCircuitCreationDefenseType** __NUM__::
 
     This is the type of defense applied to a detected client address. The
     possible values are:
- +
+     +
       1: No defense.
- +
+     +
       2: Refuse circuit creation for the DoSCircuitCreationDefenseTimePeriod period of time.
- +
+     +
     "0" means use the consensus parameter. If not defined in the consensus, the value is 2.
     (Default: 0)
 
-[[DoSCircuitCreationDefenseTimePeriod]] **DoSCircuitCreationDefenseTimePeriod** __N__ **seconds**|**minutes**|**hours**::
+[[DoSCircuitCreationMinConnections]] **DoSCircuitCreationMinConnections** __NUM__::
 
-    The base time period in seconds that the DoS defense is activated for. The
-    actual value is selected randomly for each activation from N+1 to 3/2 * N.
-    "0" means use the consensus parameter. If not defined in the consensus,
-    the value is 3600 seconds (1 hour).
+    Minimum threshold of concurrent connections before a client address can be
+    flagged as executing a circuit creation DoS. In other words, once a client
+    address reaches the circuit rate and has a minimum of NUM concurrent
+    connections, a detection is positive. "0" means use the consensus
+    parameter. If not defined in the consensus, the value is 3.
+    (Default: 0)
+
+[[DoSCircuitCreationRate]] **DoSCircuitCreationRate** __NUM__::
+
+    The allowed circuit creation rate per second applied per client IP
+    address. If this option is 0, it obeys a consensus parameter. If not
+    defined in the consensus, the value is 3.
     (Default: 0)
 
+//out of order because it logically belongs before the other DoSConnection options.
 [[DoSConnectionEnabled]] **DoSConnectionEnabled** **0**|**1**|**auto**::
 
     Enable the connection DoS mitigation. If set to 1 (enabled), for client
@@ -2542,26 +2841,26 @@ Denial of Service mitigation subsystem described above.
     consensus parameter. If not defined in the consensus, the value is 0.
     (Default: auto)
 
-[[DoSConnectionMaxConcurrentCount]] **DoSConnectionMaxConcurrentCount** __NUM__::
-
-    The maximum threshold of concurrent connection from a client IP address.
-    Above this limit, a defense selected by DoSConnectionDefenseType is
-    applied. "0" means use the consensus parameter. If not defined in the
-    consensus, the value is 100.
-    (Default: 0)
-
 [[DoSConnectionDefenseType]] **DoSConnectionDefenseType** __NUM__::
 
     This is the type of defense applied to a detected client address for the
     connection mitigation. The possible values are:
- +
+     +
       1: No defense.
- +
+     +
       2: Immediately close new connections.
- +
+     +
     "0" means use the consensus parameter. If not defined in the consensus, the value is 2.
     (Default: 0)
 
+[[DoSConnectionMaxConcurrentCount]] **DoSConnectionMaxConcurrentCount** __NUM__::
+
+    The maximum threshold of concurrent connection from a client IP address.
+    Above this limit, a defense selected by DoSConnectionDefenseType is
+    applied. "0" means use the consensus parameter. If not defined in the
+    consensus, the value is 100.
+    (Default: 0)
+
 [[DoSRefuseSingleHopClientRendezvous]] **DoSRefuseSingleHopClientRendezvous** **0**|**1**|**auto**::
 
     Refuse establishment of rendezvous points for single hop clients. In other
@@ -2571,14 +2870,14 @@ Denial of Service mitigation subsystem described above.
     (Default: auto)
 
 
-DIRECTORY AUTHORITY SERVER OPTIONS
-----------------------------------
+== DIRECTORY AUTHORITY SERVER OPTIONS
 
 The following options enable operation as a directory authority, and
 control how Tor behaves as a directory authority.  You should not need
 to adjust any of them if you're running a regular relay or exit server
 on the public Tor network.
 
+// Out of order because it logically belongs first in this section
 [[AuthoritativeDirectory]] **AuthoritativeDirectory** **0**|**1**::
     When this option is set to 1, Tor operates as an authoritative directory
     server. Instead of caching the directory, it generates its own list of
@@ -2586,95 +2885,68 @@ on the public Tor network.
     already have you listed as a trusted directory, you probably do not want
     to set this option.
 
-[[V3AuthoritativeDirectory]] **V3AuthoritativeDirectory** **0**|**1**::
-    When this option is set in addition to **AuthoritativeDirectory**, Tor
-    generates version 3 network statuses and serves descriptors, etc as
-    described in dir-spec.txt file of https://spec.torproject.org/[torspec]
-    (for Tor clients and servers running at least 0.2.0.x).
-
-[[VersioningAuthoritativeDirectory]] **VersioningAuthoritativeDirectory** **0**|**1**::
-    When this option is set to 1, Tor adds information on which versions of
-    Tor are still believed safe for use to the published directory. Each
-    version 1 authority is automatically a versioning authority; version 2
-    authorities provide this service optionally. See **RecommendedVersions**,
-    **RecommendedClientVersions**, and **RecommendedServerVersions**.
-
-[[RecommendedVersions]] **RecommendedVersions** __STRING__::
-    STRING is a comma-separated list of Tor versions currently believed to be
-    safe. The list is included in each directory, and nodes which pull down the
-    directory learn whether they need to upgrade. This option can appear
-    multiple times: the values from multiple lines are spliced together. When
-    this is set then **VersioningAuthoritativeDirectory** should be set too.
-
-[[RecommendedPackages]] **RecommendedPackages** __PACKAGENAME__ __VERSION__ __URL__ __DIGESTTYPE__**=**__DIGEST__ ::
-    Adds "package" line to the directory authority's vote.  This information
-    is used to vote on the correct URL and digest for the released versions
-    of different Tor-related packages, so that the consensus can certify
-    them.  This line may appear any number of times.
-
-[[RecommendedClientVersions]] **RecommendedClientVersions** __STRING__::
-    STRING is a comma-separated list of Tor versions currently believed to be
-    safe for clients to use. This information is included in version 2
-    directories. If this is not set then the value of **RecommendedVersions**
-    is used. When this is set then **VersioningAuthoritativeDirectory** should
-    be set too.
-
+//Out of order because it belongs with the AuthoritativeDirectory option.
 [[BridgeAuthoritativeDir]] **BridgeAuthoritativeDir** **0**|**1**::
     When this option is set in addition to **AuthoritativeDirectory**, Tor
     accepts and serves server descriptors, but it caches and serves the main
     networkstatus documents rather than generating its own. (Default: 0)
 
-[[MinUptimeHidServDirectoryV2]] **MinUptimeHidServDirectoryV2** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
-    Minimum uptime of a v2 hidden service directory to be accepted as such by
-    authoritative directories. (Default: 25 hours)
-
-[[RecommendedServerVersions]] **RecommendedServerVersions** __STRING__::
-    STRING is a comma-separated list of Tor versions currently believed to be
-    safe for servers to use. This information is included in version 2
-    directories. If this is not set then the value of **RecommendedVersions**
-    is used. When this is set then **VersioningAuthoritativeDirectory** should
-    be set too.
-
-[[ConsensusParams]] **ConsensusParams** __STRING__::
-    STRING is a space-separated list of key=value pairs that Tor will include
-    in the "params" line of its networkstatus vote.
-
-[[DirAllowPrivateAddresses]] **DirAllowPrivateAddresses** **0**|**1**::
-    If set to 1, Tor will accept server descriptors with arbitrary "Address"
-    elements. Otherwise, if the address is not an IP address or is a private IP
-    address, it will reject the server descriptor. Additionally, Tor
-    will allow exit policies for private networks to fulfill Exit flag
-    requirements. (Default: 0)
+//Out of order because it belongs with the AuthoritativeDirectory option.
+[[V3AuthoritativeDirectory]] **V3AuthoritativeDirectory** **0**|**1**::
+    When this option is set in addition to **AuthoritativeDirectory**, Tor
+    generates version 3 network statuses and serves descriptors, etc as
+    described in dir-spec.txt file of https://spec.torproject.org/[torspec]
+    (for Tor clients and servers running at least 0.2.0.x).
 
 [[AuthDirBadExit]] **AuthDirBadExit** __AddressPattern...__::
     Authoritative directories only. A set of address patterns for servers that
     will be listed as bad exits in any network status document this authority
     publishes, if **AuthDirListBadExits** is set. +
- +
+     +
     (The address pattern syntax here and in the options below
     is the same as for exit policies, except that you don't need to say
     "accept" or "reject", and ports are not needed.)
 
-[[AuthDirInvalid]] **AuthDirInvalid** __AddressPattern...__::
-    Authoritative directories only. A set of address patterns for servers that
-    will never be listed as "valid" in any network status document that this
-    authority publishes.
+[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    Authoritative directories only. If non-zero, always vote the
+    Fast flag for any relay advertising this amount of capacity or
+    more. (Default: 100 KBytes)
 
-[[AuthDirReject]] **AuthDirReject** __AddressPattern__...::
-    Authoritative directories only. A set of address patterns for servers that
-    will never be listed at all in any network status document that this
-    authority publishes, or accepted as an OR address in any descriptor
-    submitted for publication by this authority.
+[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    Authoritative directories only. If non-zero, this advertised capacity
+    or more is always sufficient to satisfy the bandwidth requirement
+    for the Guard flag. (Default: 2 MBytes)
 
-[[AuthDirBadExitCCs]] **AuthDirBadExitCCs** __CC__,... +
+[[AuthDirHasIPv6Connectivity]] **AuthDirHasIPv6Connectivity** **0**|**1**::
+    Authoritative directories only. When set to 0, OR ports with an
+    IPv6 address are not included in the authority's votes. When set to 1,
+    IPv6 OR ports are tested for reachability like IPv4 OR ports. If the
+    reachability test succeeds, the authority votes for the IPv6 ORPort, and
+    votes Running for the relay. If the reachability test fails, the authority
+    does not vote for the IPv6 ORPort, and does not vote Running (Default: 0)  +
++
+    The content of the consensus depends on the number of voting authorities
+    that set AuthDirHasIPv6Connectivity:
 
-[[AuthDirInvalidCCs]] **AuthDirInvalidCCs** __CC__,... +
+      If no authorities set AuthDirHasIPv6Connectivity 1, there will be no
+      IPv6 ORPorts in the consensus.
 
-[[AuthDirRejectCCs]] **AuthDirRejectCCs** __CC__,...::
-    Authoritative directories only. These options contain a comma-separated
-    list of country codes such that any server in one of those country codes
-    will be marked as a bad exit/invalid for use, or rejected
-    entirely.
+      If a minority of authorities set AuthDirHasIPv6Connectivity 1,
+      unreachable IPv6 ORPorts will be removed from the consensus. But the
+      majority of IPv4-only authorities will still vote the relay as Running.
+      Reachable IPv6 ORPort lines will be included in the consensus
+
+      If a majority of voting authorities set AuthDirHasIPv6Connectivity 1,
+      relays with unreachable IPv6 ORPorts will not be listed as Running.
+      Reachable IPv6 ORPort lines will be included in the consensus
+      (To ensure that any valid majority will vote relays with unreachable
+      IPv6 ORPorts not Running, 75% of authorities must set
+      AuthDirHasIPv6Connectivity 1.)
+
+[[AuthDirInvalid]] **AuthDirInvalid** __AddressPattern...__::
+    Authoritative directories only. A set of address patterns for servers that
+    will never be listed as "valid" in any network status document that this
+    authority publishes.
 
 [[AuthDirListBadExits]] **AuthDirListBadExits** **0**|**1**::
     Authoritative directories only. If set to 1, this directory has some
@@ -2687,16 +2959,6 @@ on the public Tor network.
     list as acceptable on a single IP address. Set this to "0" for "no limit".
     (Default: 2)
 
-[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    Authoritative directories only. If non-zero, always vote the
-    Fast flag for any relay advertising this amount of capacity or
-    more. (Default: 100 KBytes)
-
-[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    Authoritative directories only. If non-zero, this advertised capacity
-    or more is always sufficient to satisfy the bandwidth requirement
-    for the Guard flag. (Default: 2 MBytes)
-
 [[AuthDirPinKeys]] **AuthDirPinKeys** **0**|**1**::
     Authoritative directories only. If non-zero, do not allow any relay to
     publish a descriptor if any other relay has reserved its <Ed25519,RSA>
@@ -2704,6 +2966,31 @@ on the public Tor network.
     in a journal if it is new, or if it differs from the most recently
     accepted pinning for one of the keys it contains. (Default: 1)
 
+[[AuthDirReject]] **AuthDirReject** __AddressPattern__...::
+    Authoritative directories only. A set of address patterns for servers that
+    will never be listed at all in any network status document that this
+    authority publishes, or accepted as an OR address in any descriptor
+    submitted for publication by this authority.
+
+//Out of order because it logically belongs with the other CCs options.
+[[AuthDirBadExitCCs]] **AuthDirBadExitCCs** __CC__,... +
+
+//Out of order because it logically belongs with the other CCs options.
+[[AuthDirInvalidCCs]] **AuthDirInvalidCCs** __CC__,... +
+
+
+[[AuthDirRejectRequestsUnderLoad]] **AuthDirRejectRequestsUnderLoad** **0**|**1**::
+    If set, the directory authority will start rejecting directory requests
+    from non relay connections by sending a 503 error code if it is under
+    bandwidth pressure (reaching the configured limit if any). Relays will
+    always tried to be answered even if this is on. (Default: 1)
+
+[[AuthDirRejectCCs]] **AuthDirRejectCCs** __CC__,...::
+    Authoritative directories only. These options contain a comma-separated
+    list of country codes such that any server in one of those country codes
+    will be marked as a bad exit/invalid for use, or rejected
+    entirely.
+
 [[AuthDirSharedRandomness]] **AuthDirSharedRandomness** **0**|**1**::
     Authoritative directories only. Switch for the shared random protocol.
     If zero, the authority won't participate in the protocol. If non-zero
@@ -2725,19 +3012,55 @@ on the public Tor network.
     and their target user audience can periodically fetch the list of
     available community bridges to stay up-to-date. (Default: not set)
 
-[[V3AuthVotingInterval]] **V3AuthVotingInterval** __N__ **minutes**|**hours**::
-    V3 authoritative directories only. Configures the server's preferred voting
-    interval. Note that voting will __actually__ happen at an interval chosen
-    by consensus from all the authorities' preferred intervals. This time
-    SHOULD divide evenly into a day. (Default: 1 hour)
+[[ConsensusParams]] **ConsensusParams** __STRING__::
+    STRING is a space-separated list of key=value pairs that Tor will include
+    in the "params" line of its networkstatus vote. This directive can be
+    specified multiple times so you don't have to put it all on one line.
 
-[[V3AuthVoteDelay]] **V3AuthVoteDelay** __N__ **minutes**|**hours**::
-    V3 authoritative directories only. Configures the server's preferred delay
-    between publishing its vote and assuming it has all the votes from all the
-    other authorities. Note that the actual time used is not the server's
-    preferred time, but the consensus of all preferences. (Default: 5 minutes)
+[[DirAllowPrivateAddresses]] **DirAllowPrivateAddresses** **0**|**1**::
+    If set to 1, Tor will accept server descriptors with arbitrary "Address"
+    elements. Otherwise, if the address is not an IP address or is a private IP
+    address, it will reject the server descriptor. Additionally, Tor
+    will allow exit policies for private networks to fulfill Exit flag
+    requirements. (Default: 0)
+
+[[GuardfractionFile]] **GuardfractionFile** __FILENAME__::
+    V3 authoritative directories only. Configures the location of the
+    guardfraction file which contains information about how long relays
+    have been guards. (Default: unset)
+
+[[MinMeasuredBWsForAuthToIgnoreAdvertised]] **MinMeasuredBWsForAuthToIgnoreAdvertised** __N__::
+    A total value, in abstract bandwidth units, describing how much
+    measured total bandwidth an authority should have observed on the network
+    before it will treat advertised bandwidths as wholly
+    unreliable. (Default: 500)
+
+[[MinUptimeHidServDirectoryV2]] **MinUptimeHidServDirectoryV2** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
+    Minimum uptime of a relay to be accepted as a hidden service directory
+    by directory authorities. (Default: 96 hours)
 
-[[V3AuthDistDelay]] **V3AuthDistDelay** __N__ **minutes**|**hours**::
+[[RecommendedClientVersions]] **RecommendedClientVersions** __STRING__::
+    STRING is a comma-separated list of Tor versions currently believed to be
+    safe for clients to use. This information is included in version 2
+    directories. If this is not set then the value of **RecommendedVersions**
+    is used. When this is set then **VersioningAuthoritativeDirectory** should
+    be set too.
+
+[[RecommendedServerVersions]] **RecommendedServerVersions** __STRING__::
+    STRING is a comma-separated list of Tor versions currently believed to be
+    safe for servers to use. This information is included in version 2
+    directories. If this is not set then the value of **RecommendedVersions**
+    is used. When this is set then **VersioningAuthoritativeDirectory** should
+    be set too.
+
+[[RecommendedVersions]] **RecommendedVersions** __STRING__::
+    STRING is a comma-separated list of Tor versions currently believed to be
+    safe. The list is included in each directory, and nodes which pull down the
+    directory learn whether they need to upgrade. This option can appear
+    multiple times: the values from multiple lines are spliced together. When
+    this is set then **VersioningAuthoritativeDirectory** should be set too.
+
+[[V3AuthDistDelay]] **V3AuthDistDelay** __N__ **seconds**|**minutes**|**hours**::
     V3 authoritative directories only. Configures the server's preferred  delay
     between publishing its consensus and signature and assuming  it has all the
     signatures from all the other authorities. Note that the actual time used
@@ -2752,90 +3075,57 @@ on the public Tor network.
     server's preferred number, but the consensus of all preferences. Must be at
     least 2. (Default: 3)
 
-[[V3BandwidthsFile]] **V3BandwidthsFile** __FILENAME__::
-    V3 authoritative directories only. Configures the location of the
-    bandwidth-authority generated file storing information on relays' measured
-    bandwidth capacities. To avoid inconsistent reads, bandwidth data should
-    be written to temporary file, then renamed to the configured filename.
-    (Default: unset)
-
 [[V3AuthUseLegacyKey]] **V3AuthUseLegacyKey** **0**|**1**::
     If set, the directory authority will sign consensuses not only with its
     own signing key, but also with a "legacy" key and certificate with a
     different identity.  This feature is used to migrate directory authority
     keys in the event of a compromise.  (Default: 0)
 
-[[AuthDirHasIPv6Connectivity]] **AuthDirHasIPv6Connectivity** **0**|**1**::
-    Authoritative directories only. When set to 0, OR ports with an
-    IPv6 address are not included in the authority's votes. When set to 1,
-    IPv6 OR ports are tested for reachability like IPv4 OR ports. If the
-    reachability test succeeds, the authority votes for the IPv6 ORPort, and
-    votes Running for the relay. If the reachability test fails, the authority
-    does not vote for the IPv6 ORPort, and does not vote Running (Default: 0)  +
-+
-    The content of the consensus depends on the number of voting authorities
-    that set AuthDirHasIPv6Connectivity:
-
-      If no authorities set AuthDirHasIPv6Connectivity 1, there will be no
-      IPv6 ORPorts in the consensus.
-
-      If a minority of authorities set AuthDirHasIPv6Connectivity 1,
-      unreachable IPv6 ORPorts will be removed from the consensus. But the
-      majority of IPv4-only authorities will still vote the relay as Running.
-      Reachable IPv6 ORPort lines will be included in the consensus
+[[V3AuthVoteDelay]] **V3AuthVoteDelay** __N__ **seconds**|**minutes**|**hours**::
+    V3 authoritative directories only. Configures the server's preferred delay
+    between publishing its vote and assuming it has all the votes from all the
+    other authorities. Note that the actual time used is not the server's
+    preferred time, but the consensus of all preferences. (Default: 5
+    minutes)
 
-      If a majority of voting authorities set AuthDirHasIPv6Connectivity 1,
-      relays with unreachable IPv6 ORPorts will not be listed as Running.
-      Reachable IPv6 ORPort lines will be included in the consensus
-      (To ensure that any valid majority will vote relays with unreachable
-      IPv6 ORPorts not Running, 75% of authorities must set
-      AuthDirHasIPv6Connectivity 1.)
+[[V3AuthVotingInterval]] **V3AuthVotingInterval** __N__ **minutes**|**hours**::
+    V3 authoritative directories only. Configures the server's preferred voting
+    interval. Note that voting will __actually__ happen at an interval chosen
+    by consensus from all the authorities' preferred intervals. This time
+    SHOULD divide evenly into a day. (Default: 1 hour)
 
-[[MinMeasuredBWsForAuthToIgnoreAdvertised]] **MinMeasuredBWsForAuthToIgnoreAdvertised** __N__::
-    A total value, in abstract bandwidth units, describing how much
-    measured total bandwidth an authority should have observed on the network
-    before it will treat advertised bandwidths as wholly
-    unreliable. (Default: 500)
+[[V3BandwidthsFile]] **V3BandwidthsFile** __FILENAME__::
+    V3 authoritative directories only. Configures the location of the
+    bandwidth-authority generated file storing information on relays' measured
+    bandwidth capacities. To avoid inconsistent reads, bandwidth data should
+    be written to temporary file, then renamed to the configured filename.
+    (Default: unset)
 
-HIDDEN SERVICE OPTIONS
-----------------------
+[[VersioningAuthoritativeDirectory]] **VersioningAuthoritativeDirectory** **0**|**1**::
+    When this option is set to 1, Tor adds information on which versions of
+    Tor are still believed safe for use to the published directory. Each
+    version 1 authority is automatically a versioning authority; version 2
+    authorities provide this service optionally. See <<RecommendedVersions,RecommendedVersions>>,
+    <<RecommendedClientVersions,RecommendedClientVersions>>, and <<RecommendedServerVersions,RecommendedServerVersions>>.
 
-The following options are used to configure a hidden service.
+== HIDDEN SERVICE OPTIONS
 
-[[HiddenServiceDir]] **HiddenServiceDir** __DIRECTORY__::
-    Store data files for a hidden service in DIRECTORY. Every hidden service
-    must have a separate directory. You may use this option  multiple times to
-    specify multiple services. If DIRECTORY does not exist, Tor will create it.
-    (Note: in current versions of Tor, if DIRECTORY is a relative path,
-    it will be relative to the current
-    working directory of Tor instance, not to its DataDirectory.  Do not
-    rely on this behavior; it is not guaranteed to remain the same in future
-    versions.)
+The following options are used to configure a hidden service. Some options
+apply per service and some apply for the whole tor instance.
 
-[[HiddenServicePort]] **HiddenServicePort** __VIRTPORT__ [__TARGET__]::
-    Configure a virtual port VIRTPORT for a hidden service. You may use this
-    option multiple times; each time applies to the service using the most
-    recent HiddenServiceDir. By default, this option maps the virtual port to
-    the same port on 127.0.0.1 over TCP. You may override the target port,
-    address, or both by specifying a target of addr, port, addr:port, or
-    **unix:**__path__.  (You can specify an IPv6 target as [addr]:port. Unix
-    paths may be quoted, and may use standard C escapes.)
-    You may also have multiple lines with  the same VIRTPORT: when a user
-    connects to that VIRTPORT, one of the TARGETs from those lines will be
-    chosen at random. Note that address-port pairs have to be comma-separated.
+The next section describes the per service options that can only be set
+**after** the **HiddenServiceDir** directive
 
-[[PublishHidServDescriptors]] **PublishHidServDescriptors** **0**|**1**::
-    If set to 0, Tor will run any hidden services you configure, but it won't
-    advertise them to the rendezvous directory. This option is only useful if
-    you're using a Tor controller that handles hidserv publishing for you.
-    (Default: 1)
+**PER SERVICE OPTIONS:**
 
-[[HiddenServiceVersion]] **HiddenServiceVersion** **2**|**3**::
-    A list of rendezvous service descriptor versions to publish for the hidden
-    service. Currently, versions 2 and 3 are supported. (Default: 3)
+[[HiddenServiceAllowUnknownPorts]] **HiddenServiceAllowUnknownPorts** **0**|**1**::
+   If set to 1, then connections to unrecognized ports do not cause the
+   current hidden service to close rendezvous circuits. (Setting this to 0 is
+   not an authorization mechanism; it is instead meant to be a mild
+   inconvenience to port-scanners.) (Default: 0)
 
 [[HiddenServiceAuthorizeClient]] **HiddenServiceAuthorizeClient** __auth-type__ __client-name__,__client-name__,__...__::
-    If configured, the hidden service is accessible for authorized clients
+    If configured, the v2 hidden service is accessible for authorized clients
     only. The auth-type can either be \'basic' for a general-purpose
     authorization protocol or \'stealth' for a less scalable protocol that also
     hides service activity from unauthorized clients. Only clients that are
@@ -2846,41 +3136,90 @@ The following options are used to configure a hidden service.
     found in the hostname file. Clients need to put this authorization data in
     their configuration file using **HidServAuth**. This option is only for v2
     services; v3 services configure client authentication in a subdirectory of
-    HiddenServiceDir instead (see the **Client Authorization** section).
+    HiddenServiceDir instead (see <<client-authorization,CLIENT AUTHORIZATION>>).
 
-[[HiddenServiceAllowUnknownPorts]] **HiddenServiceAllowUnknownPorts** **0**|**1**::
-   If set to 1, then connections to unrecognized ports do not cause the
-   current hidden service to close rendezvous circuits. (Setting this to 0 is
-   not an authorization mechanism; it is instead meant to be a mild
-   inconvenience to port-scanners.) (Default: 0)
+[[HiddenServiceDir]] **HiddenServiceDir** __DIRECTORY__::
+    Store data files for a hidden service in DIRECTORY. Every hidden service
+    must have a separate directory. You may use this option  multiple times to
+    specify multiple services. If DIRECTORY does not exist, Tor will create it.
+    Please note that you cannot add new Onion Service to already running Tor
+    instance if **Sandbox** is enabled.
+    (Note: in current versions of Tor, if DIRECTORY is a relative path,
+    it will be relative to the current
+    working directory of Tor instance, not to its DataDirectory.  Do not
+    rely on this behavior; it is not guaranteed to remain the same in future
+    versions.)
+
+[[HiddenServiceDirGroupReadable]] **HiddenServiceDirGroupReadable** **0**|**1**::
+    If this option is set to 1, allow the filesystem group to read the
+    hidden service directory and hostname file. If the option is set to 0,
+    only owner is able to read the hidden service directory. (Default: 0)
+    Has no effect on Windows.
+
+[[HiddenServiceEnableIntroDoSDefense]] **HiddenServiceEnableIntroDoSDefense** **0**|**1**::
+    Enable DoS defense at the intropoint level. When this is enabled, the
+    rate and burst parameter (see below) will be sent to the intro point which
+    will then use them to apply rate limiting for introduction request to this
+    service.
+  +
+    The introduction point honors the consensus parameters except if this is
+    specifically set by the service operator using this option. The service
+    never looks at the consensus parameters in order to enable or disable this
+    defense. (Default: 0)
+
+//Out of order because it logically belongs after HiddenServiceEnableIntroDoSDefense.
+[[HiddenServiceEnableIntroDoSBurstPerSec]] **HiddenServiceEnableIntroDoSBurstPerSec** __NUM__::
+    The allowed client introduction burst per second at the introduction
+    point. If this option is 0, it is considered infinite and thus if
+    **HiddenServiceEnableIntroDoSDefense** is set, it then effectively
+    disables the defenses. (Default: 200)
+
+[[HiddenServiceEnableIntroDoSRatePerSec]] **HiddenServiceEnableIntroDoSRatePerSec** __NUM__::
+    The allowed client introduction rate per second at the introduction
+    point. If this option is 0, it is considered infinite and thus if
+    **HiddenServiceEnableIntroDoSDefense** is set, it then effectively
+    disables the defenses. (Default: 25)
 
 [[HiddenServiceExportCircuitID]] **HiddenServiceExportCircuitID** __protocol__::
    The onion service will use the given protocol to expose the global circuit
-   identifier of each inbound client circuit via the selected protocol. The only
+   identifier of each inbound client circuit. The only
    protocol supported right now \'haproxy'. This option is only for v3
    services. (Default: none) +
- +
+     +
    The haproxy option works in the following way: when the feature is
    enabled, the Tor process will write a header line when a client is connecting
    to the onion service. The header will look like this: +
- +
+     +
    "PROXY TCP6 fc00:dead:beef:4dad::ffff:ffff ::1 65535 42\r\n" +
- +
+     +
    We encode the "global circuit identifier" as the last 32-bits of the first
    IPv6 address. All other values in the header can safely be ignored. You can
    compute the global circuit identifier using the following formula given the
    IPv6 address "fc00:dead:beef:4dad::AABB:CCDD": +
- +
+     +
       global_circuit_id = (0xAA << 24) + (0xBB << 16) + (0xCC << 8) + 0xDD; +
- +
-   In the case above, where the last 32-bit is 0xffffffff, the global circuit
+     +
+   In the case above, where the last 32-bits are 0xffffffff, the global circuit
    identifier would be 4294967295. You can use this value together with Tor's
-   control port where it is possible to terminate a circuit given the global
-   circuit identifier. For more information about this see controls-spec.txt. +
- +
-   The HAProxy version 1 proxy protocol is described in detail at
+   control port to terminate particular circuits using their global
+   circuit identifiers. For more information about this see control-spec.txt. +
+     +
+   The HAProxy version 1 protocol is described in detail at
    https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
 
+[[HiddenServiceOnionBalanceInstance]] **HiddenServiceOnionBalanceInstance** **0**|**1**::
+
+   If set to 1, this onion service becomes an OnionBalance instance and will
+   accept client connections destined to an OnionBalance frontend. In this
+   case, Tor expects to find a file named "ob_config" inside the
+   **HiddenServiceDir** directory with content:
+     +
+      MasterOnionAddress <frontend_onion_address>
+     +
+   where <frontend_onion_address> is the onion address of the OnionBalance
+   frontend (e.g. wrxdvcaqpuzakbfww5sxs6r2uybczwijzfn2ezy2osaj7iox7kl7nhad.onion).
+
+
 [[HiddenServiceMaxStreams]] **HiddenServiceMaxStreams** __N__::
    The maximum number of simultaneous streams (connections) per rendezvous
    circuit. The maximum value allowed is 65535. (Setting this to 0 will allow
@@ -2891,6 +3230,26 @@ The following options are used to configure a hidden service.
    offending rendezvous circuit to be torn down, as opposed to stream creation
    requests that exceed the limit being silently ignored. (Default: 0)
 
+[[HiddenServiceNumIntroductionPoints]] **HiddenServiceNumIntroductionPoints** __NUM__::
+    Number of introduction points the hidden service will have. You can't
+    have more than 10 for v2 service and 20 for v3. (Default: 3)
+
+[[HiddenServicePort]] **HiddenServicePort** __VIRTPORT__ [__TARGET__]::
+    Configure a virtual port VIRTPORT for a hidden service. You may use this
+    option multiple times; each time applies to the service using the most
+    recent HiddenServiceDir. By default, this option maps the virtual port to
+    the same port on 127.0.0.1 over TCP. You may override the target port,
+    address, or both by specifying a target of addr, port, addr:port, or
+    **unix:**__path__.  (You can specify an IPv6 target as [addr]:port. Unix
+    paths may be quoted, and may use standard C escapes.)
+    You may also have multiple lines with  the same VIRTPORT: when a user
+    connects to that VIRTPORT, one of the TARGETs from those lines will be
+    chosen at random. Note that address-port pairs have to be comma-separated.
+
+[[HiddenServiceVersion]] **HiddenServiceVersion** **2**|**3**::
+    A list of rendezvous service descriptor versions to publish for the hidden
+    service. Currently, versions 2 and 3 are supported. (Default: 3)
+
 [[RendPostPeriod]] **RendPostPeriod** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
     Every time the specified period elapses, Tor uploads any rendezvous
     service descriptors to the directory servers. This information is also
@@ -2898,15 +3257,9 @@ The following options are used to configure a hidden service.
     maximum is 3.5 days. This option is only for v2 services.
     (Default: 1 hour)
 
-[[HiddenServiceDirGroupReadable]] **HiddenServiceDirGroupReadable** **0**|**1**::
-    If this option is set to 1, allow the filesystem group to read the
-    hidden service directory and hostname file. If the option is set to 0,
-    only owner is able to read the hidden service directory. (Default: 0)
-    Has no effect on Windows.
 
-[[HiddenServiceNumIntroductionPoints]] **HiddenServiceNumIntroductionPoints** __NUM__::
-    Number of introduction points the hidden service will have. You can't
-    have more than 10 for v2 service and 20 for v3. (Default: 3)
+
+**PER INSTANCE OPTIONS:**
 
 [[HiddenServiceSingleHopMode]] **HiddenServiceSingleHopMode** **0**|**1**::
     **Experimental - Non Anonymous** Hidden Services on a tor instance in
@@ -2919,7 +3272,7 @@ The following options are used to configure a hidden service.
     locatable, but clients remain location-anonymous. However, the fact that a
     client is accessing a Single Onion rather than a Hidden Service may be
     statistically distinguishable. +
- +
+     +
     **WARNING:** Once a hidden service directory has been used by a tor
     instance in HiddenServiceSingleHopMode, it can **NEVER** be used again for
     a hidden service. It is best practice to create a new hidden service
@@ -2927,13 +3280,14 @@ The following options are used to configure a hidden service.
     Service. It is not possible to run Single Onion Services and Hidden
     Services from the same tor instance: they should be run on different
     servers with different IP addresses. +
- +
+     +
     HiddenServiceSingleHopMode requires HiddenServiceNonAnonymousMode to be set
     to 1. Since a Single Onion service is non-anonymous, you can not configure
     a SOCKSPort on a tor instance that is running in
     **HiddenServiceSingleHopMode**. Can not be changed while tor is running.
     (Default: 0)
 
+//Out of order because it belongs after HiddenServiceSingleHopMode.
 [[HiddenServiceNonAnonymousMode]] **HiddenServiceNonAnonymousMode** **0**|**1**::
     Makes hidden services non-anonymous on this tor instance. Allows the
     non-anonymous HiddenServiceSingleHopMode. Enables direct connections in the
@@ -2942,42 +3296,59 @@ The following options are used to configure a hidden service.
     including setting SOCKSPort to "0". Can not be changed while tor is
     running. (Default: 0)
 
-Client Authorization
---------------------
+[[PublishHidServDescriptors]] **PublishHidServDescriptors** **0**|**1**::
+    If set to 0, Tor will run any hidden services you configure, but it won't
+    advertise them to the rendezvous directory. This option is only useful if
+    you're using a Tor controller that handles hidserv publishing for you.
+    (Default: 1)
+
+[[client-authorization]]
+== CLIENT AUTHORIZATION
 
 (Version 3 only)
 
-To configure client authorization on the service side, the
-"<HiddenServiceDir>/authorized_clients/" directory needs to exist. Each file
-in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the
-file name is irrelevant) and its content format MUST be:
+Service side:
+
+  To configure client authorization on the service side, the
+  "<HiddenServiceDir>/authorized_clients/" directory needs to exist. Each file
+  in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the
+  file name is irrelevant) and its content format MUST be:
+
+          <auth-type>:<key-type>:<base32-encoded-public-key>
 
-        <auth-type>:<key-type>:<base32-encoded-public-key>
+  The supported <auth-type> are: "descriptor". The supported <key-type> are:
+  "x25519". The <base32-encoded-public-key> is the base32 representation of
+  the raw key bytes only (32 bytes for x25519).
 
-The supported <auth-type> are: "descriptor". The supported <key-type> are:
-"x25519". The <base32-encoded-public-key> is the base32 representation of
-the raw key bytes only (32 bytes for x25519).
+  Each file MUST contain one line only. Any malformed file will be
+  ignored. Client authorization will only be enabled for the service if tor
+  successfully loads at least one authorization file.
 
-Each file MUST contain one line only. Any malformed file will be
-ignored. Client authorization will only be enabled for the service if tor
-successfully loads at least one authorization file.
+  Note that once you've configured client authorization, anyone else with the
+  address won't be able to access it from this point on. If no authorization is
+  configured, the service will be accessible to anyone with the onion address.
 
-Note that once you've configured client authorization, anyone else with the
-address won't be able to access it from this point on. If no authorization is
-configured, the service will be accessible to anyone with the onion address.
+  Revoking a client can be done by removing their ".auth" file, however the
+  revocation will be in effect only after the tor process gets restarted even if
+  a SIGHUP takes place.
 
-Revoking a client can be done by removing their ".auth" file, however the
-revocation will be in effect only after the tor process gets restarted even if
-a SIGHUP takes place.
+Client side:
 
-See the Appendix G in the rend-spec-v3.txt file of
-https://spec.torproject.org/[torspec] for more information.
+  To access a v3 onion service with client authorization as a client, make sure
+  you have ClientOnionAuthDir set in your torrc. Then, in the
+  <ClientOnionAuthDir> directory, create an .auth_private file for the onion
+  service corresponding to this key (i.e. 'bob_onion.auth_private').  The
+  contents of the <ClientOnionAuthDir>/<user>.auth_private file should look like:
 
-TESTING NETWORK OPTIONS
------------------------
+      <56-char-onion-addr-without-.onion-part>:descriptor:x25519:<x25519 private key in base32>
+
+For more information, please see https://2019.www.torproject.org/docs/tor-onion-service.html.en#ClientAuthorization .
+
+== TESTING NETWORK OPTIONS
 
 The following options are used for running a testing Tor network.
 
+//Out of order because it logically belongs first in this section.
 [[TestingTorNetwork]] **TestingTorNetwork** **0**|**1**::
     If set to 1, Tor adjusts default values of the configuration options below,
     so that it is easier to set up a testing Tor network. May only be set if
@@ -2985,12 +3356,10 @@ The following options are used for running a testing Tor network.
     running.
     (Default: 0) +
 
-       ServerDNSAllowBrokenConfig 1
        DirAllowPrivateAddresses 1
        EnforceDistinctSubnets 0
        AssumeReachable 1
        AuthDirMaxServersPerAddr 0
-       AuthDirMaxServersPerAuthAddr 0
        ClientBootstrapConsensusAuthorityDownloadInitialDelay 0
        ClientBootstrapConsensusFallbackDownloadInitialDelay 0
        ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay 0
@@ -3002,12 +3371,11 @@ The following options are used for running a testing Tor network.
        V3AuthVotingInterval 5 minutes
        V3AuthVoteDelay 20 seconds
        V3AuthDistDelay 20 seconds
-       MinUptimeHidServDirectoryV2 0 seconds
-       TestingV3AuthInitialVotingInterval 5 minutes
+       TestingV3AuthInitialVotingInterval 150 seconds
        TestingV3AuthInitialVoteDelay 20 seconds
        TestingV3AuthInitialDistDelay 20 seconds
        TestingAuthDirTimeToLearnReachability 0 minutes
-       TestingEstimatedDescriptorPropagationTime 0 minutes
+       MinUptimeHidServDirectoryV2 0 minutes
        TestingServerDownloadInitialDelay 0
        TestingClientDownloadInitialDelay 0
        TestingServerConsensusDownloadInitialDelay 0
@@ -3018,65 +3386,37 @@ The following options are used for running a testing Tor network.
        TestingDirConnectionMaxStall 30 seconds
        TestingEnableConnBwEvent 1
        TestingEnableCellStatsEvent 1
+       RendPostPeriod 2 minutes
 
-[[TestingV3AuthInitialVotingInterval]] **TestingV3AuthInitialVotingInterval** __N__ **minutes**|**hours**::
-    Like V3AuthVotingInterval, but for initial voting interval before the first
-    consensus has been created. Changing this requires that
-    **TestingTorNetwork** is set. (Default: 30 minutes)
-
-[[TestingV3AuthInitialVoteDelay]] **TestingV3AuthInitialVoteDelay** __N__ **minutes**|**hours**::
-    Like V3AuthVoteDelay, but for initial voting interval before
-    the first consensus has been created. Changing this requires that
-    **TestingTorNetwork** is set. (Default: 5 minutes)
-
-[[TestingV3AuthInitialDistDelay]] **TestingV3AuthInitialDistDelay** __N__ **minutes**|**hours**::
-    Like V3AuthDistDelay, but for initial voting interval before
-    the first consensus has been created. Changing this requires that
-    **TestingTorNetwork** is set. (Default: 5 minutes)
-
-[[TestingV3AuthVotingStartOffset]] **TestingV3AuthVotingStartOffset** __N__ **seconds**|**minutes**|**hours**::
-    Directory authorities offset voting start time by this much.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 0)
-
-[[TestingAuthDirTimeToLearnReachability]] **TestingAuthDirTimeToLearnReachability** __N__ **minutes**|**hours**::
+[[TestingAuthDirTimeToLearnReachability]] **TestingAuthDirTimeToLearnReachability** __N__ **seconds**|**minutes**|**hours**::
     After starting as an authority, do not make claims about whether routers
     are Running until this much time has passed. Changing this requires
     that **TestingTorNetwork** is set.  (Default: 30 minutes)
 
-[[TestingEstimatedDescriptorPropagationTime]] **TestingEstimatedDescriptorPropagationTime** __N__ **minutes**|**hours**::
-    Clients try downloading server descriptors from directory caches after this
-    time. Changing this requires that **TestingTorNetwork** is set. (Default:
-    10 minutes)
-
-[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    Minimum value for the Fast flag.  Overrides the ordinary minimum taken
-    from the consensus when TestingTorNetwork is set. (Default: 0.)
-
-[[TestingServerDownloadInitialDelay]] **TestingServerDownloadInitialDelay** __N__::
-    Initial delay in seconds for when servers should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
-
-[[TestingClientDownloadInitialDelay]] **TestingClientDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingAuthKeyLifetime]] **TestingAuthKeyLifetime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**|**months**::
+    Overrides the default lifetime for a signing Ed25519 TLS Link authentication
+    key.
+    (Default: 2 days)
 
-[[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__::
-    Initial delay in seconds for when servers should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingAuthKeySlop]] **TestingAuthKeySlop** __N__ **seconds**|**minutes**|**hours** +
 
-[[TestingClientConsensusDownloadInitialDelay]] **TestingClientConsensusDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingBridgeBootstrapDownloadInitialDelay]] **TestingBridgeBootstrapDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download each bridge descriptor when they
+    have just started, or when they can not contact any of their bridges.
+    Changing this requires that **TestingTorNetwork** is set. (Default: 0)
 
 [[TestingBridgeDownloadInitialDelay]] **TestingBridgeDownloadInitialDelay** __N__::
     Initial delay in seconds for when clients should download each bridge descriptor when they
     know that one or more of their configured bridges are running. Changing
     this requires that **TestingTorNetwork** is set. (Default: 10800)
 
-[[TestingBridgeBootstrapDownloadInitialDelay]] **TestingBridgeBootstrapDownloadInitialDelay** __N__::
-    Initial delay in seconds for when clients should download each bridge descriptor when they
-    have just started, or when they can not contact any of their bridges.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 0)
+[[TestingClientConsensusDownloadInitialDelay]] **TestingClientConsensusDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
+
+[[TestingClientDownloadInitialDelay]] **TestingClientDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
 [[TestingClientMaxIntervalWithoutRequest]] **TestingClientMaxIntervalWithoutRequest** __N__ **seconds**|**minutes**::
     When directory clients have only a few descriptors to request, they batch
@@ -3084,110 +3424,133 @@ The following options are used for running a testing Tor network.
     Changing this requires that **TestingTorNetwork** is set. (Default: 10
     minutes)
 
-[[TestingDirConnectionMaxStall]] **TestingDirConnectionMaxStall** __N__ **seconds**|**minutes**::
-    Let a directory connection stall this long before expiring it.
-    Changing this requires that **TestingTorNetwork** is set. (Default:
-    5 minutes)
-
 [[TestingDirAuthVoteExit]] **TestingDirAuthVoteExit** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and
     address patterns of nodes to vote Exit for regardless of their
-    uptime, bandwidth, or exit policy. See the **ExcludeNodes**
-    option for more information on how to specify nodes. +
- +
+    uptime, bandwidth, or exit policy. See <<ExcludeNodes,ExcludeNodes>>
+     for more information on how to specify nodes. +
+     +
     In order for this option to have any effect, **TestingTorNetwork**
-    has to be set. See the **ExcludeNodes** option for more
+    has to be set. See <<ExcludeNodes,ExcludeNodes>> for more
     information on how to specify nodes.
 
 [[TestingDirAuthVoteExitIsStrict]] **TestingDirAuthVoteExitIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Exit flag unless it is specified
     in the **TestingDirAuthVoteExit** list, regardless of its uptime, bandwidth,
     or exit policy. +
- +
+     +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
 
 [[TestingDirAuthVoteGuard]] **TestingDirAuthVoteGuard** __node__,__node__,__...__::
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote Guard for regardless of their
-    uptime and bandwidth. See the **ExcludeNodes** option for more
+    uptime and bandwidth. See <<ExcludeNodes,ExcludeNodes>> for more
     information on how to specify nodes. +
- +
+     +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
 
 [[TestingDirAuthVoteGuardIsStrict]] **TestingDirAuthVoteGuardIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Guard flag unless it is specified
     in the **TestingDirAuthVoteGuard** list, regardless of its uptime and bandwidth. +
- +
+     +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
 
 [[TestingDirAuthVoteHSDir]] **TestingDirAuthVoteHSDir** __node__,__node__,__...__::
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote HSDir for regardless of their
-    uptime and DirPort. See the **ExcludeNodes** option for more
+    uptime and DirPort. See <<ExcludeNodes,ExcludeNodes>> for more
     information on how to specify nodes. +
- +
+     +
     In order for this option to have any effect, **TestingTorNetwork**
     must be set.
 
 [[TestingDirAuthVoteHSDirIsStrict]] **TestingDirAuthVoteHSDirIsStrict** **0**|**1** ::
     If True (1), a node will never receive the HSDir flag unless it is specified
     in the **TestingDirAuthVoteHSDir** list, regardless of its uptime and DirPort. +
- +
+     +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
 
-[[TestingEnableConnBwEvent]] **TestingEnableConnBwEvent** **0**|**1**::
-    If this option is set, then Tor controllers may register for CONN_BW
-    events.  Changing this requires that **TestingTorNetwork** is set.
-    (Default: 0)
+[[TestingDirConnectionMaxStall]] **TestingDirConnectionMaxStall** __N__ **seconds**|**minutes**::
+    Let a directory connection stall this long before expiring it.
+    Changing this requires that **TestingTorNetwork** is set. (Default:
+    5 minutes)
 
 [[TestingEnableCellStatsEvent]] **TestingEnableCellStatsEvent** **0**|**1**::
     If this option is set, then Tor controllers may register for CELL_STATS
     events.  Changing this requires that **TestingTorNetwork** is set.
     (Default: 0)
 
-[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
-    Sets a lower-bound for assigning an exit flag when running as an
-    authority on a testing network. Overrides the usual default lower bound
-    of 4 KB. (Default: 0)
+[[TestingEnableConnBwEvent]] **TestingEnableConnBwEvent** **0**|**1**::
+    If this option is set, then Tor controllers may register for CONN_BW
+    events.  Changing this requires that **TestingTorNetwork** is set.
+    (Default: 0)
 
 [[TestingLinkCertLifetime]] **TestingLinkCertLifetime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**|**months**::
     Overrides the default lifetime for the certificates used to authenticate
     our X509 link cert with our ed25519 signing key.
     (Default: 2 days)
 
-[[TestingAuthKeyLifetime]] **TestingAuthKeyLifetime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**|**months**::
-    Overrides the default lifetime for a signing Ed25519 TLS Link authentication
-    key.
-    (Default: 2 days)
-
 [[TestingLinkKeySlop]] **TestingLinkKeySlop** __N__ **seconds**|**minutes**|**hours** +
 
-[[TestingAuthKeySlop]] **TestingAuthKeySlop** __N__ **seconds**|**minutes**|**hours** +
+[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    Sets a lower-bound for assigning an exit flag when running as an
+    authority on a testing network. Overrides the usual default lower bound
+    of 4 KBytes. (Default: 0)
+
+[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    Minimum value for the Fast flag.  Overrides the ordinary minimum taken
+    from the consensus when TestingTorNetwork is set. (Default: 0.)
+
+[[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__::
+    Initial delay in seconds for when servers should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
+
+[[TestingServerDownloadInitialDelay]] **TestingServerDownloadInitialDelay** __N__::
+    Initial delay in seconds for when servers should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
 [[TestingSigningKeySlop]] **TestingSigningKeySlop** __N__ **seconds**|**minutes**|**hours**::
     How early before the official expiration of a an Ed25519 signing key do
     we replace it and issue a new key?
     (Default: 3 hours for link and auth; 1 day for signing.)
 
-NON-PERSISTENT OPTIONS
-----------------------
+[[TestingV3AuthInitialDistDelay]] **TestingV3AuthInitialDistDelay** __N__ **seconds**|**minutes**|**hours**::
+    Like V3AuthDistDelay, but for initial voting interval before
+    the first consensus has been created. Changing this requires that
+    **TestingTorNetwork** is set. (Default: 5 minutes)
+
+[[TestingV3AuthInitialVoteDelay]] **TestingV3AuthInitialVoteDelay** __N__ **seconds**|**minutes**|**hours**::
+    Like V3AuthVoteDelay, but for initial voting interval before
+    the first consensus has been created. Changing this requires that
+    **TestingTorNetwork** is set. (Default: 5 minutes)
+
+[[TestingV3AuthInitialVotingInterval]] **TestingV3AuthInitialVotingInterval** __N__ **seconds**|**minutes**|**hours**::
+    Like V3AuthVotingInterval, but for initial voting interval before the first
+    consensus has been created. Changing this requires that
+    **TestingTorNetwork** is set. (Default: 30 minutes)
+
+[[TestingV3AuthVotingStartOffset]] **TestingV3AuthVotingStartOffset** __N__ **seconds**|**minutes**|**hours**::
+    Directory authorities offset voting start time by this much.
+    Changing this requires that **TestingTorNetwork** is set. (Default: 0)
+
+
+== NON-PERSISTENT OPTIONS
 
 These options are not saved to the torrc file by the "SAVECONF" controller
 command.  Other options of this type are documented in control-spec.txt,
 section 5.4. End-users should mostly ignore them.
 
-[[UnderscorePorts]] **\_\_ControlPort**, **\_\_DirPort**, **\_\_DNSPort**, **\_\_ExtORPort**, **\_\_NATDPort**, **\_\_ORPort**, **\_\_SocksPort**, **\_\_TransPort**::
+[[UnderscorePorts]] **{dbl_}ControlPort**, **{dbl_}DirPort**, **{dbl_}DNSPort**, **{dbl_}ExtORPort**, **{dbl_}NATDPort**, **{dbl_}ORPort**, **{dbl_}SocksPort**, **{dbl_}TransPort**::
     These underscore-prefixed options are variants of the regular Port
     options.  They behave the same, except they are not saved to the
     torrc file by the controller's SAVECONF command.
 
 
-SIGNALS
--------
+== SIGNALS
 
 Tor catches the following signals:
 
@@ -3220,264 +3583,251 @@ Tor catches the following signals:
 [[SIGXFSZ]] **SIGXFSZ**::
     If this signal exists on your platform, Tor catches and ignores it.
 
-FILES
------
+== FILES
 
-**@CONFDIR@/torrc**::
-    The configuration file, which contains "option value" pairs.
+**`@CONFDIR@/torrc`**::
+    Default location of the configuration file.
 
-**$HOME/.torrc**::
+**`$HOME/.torrc`**::
     Fallback location for torrc, if @CONFDIR@/torrc is not found.
 
-**@LOCALSTATEDIR@/lib/tor/**::
+**`@LOCALSTATEDIR@/lib/tor/`**::
     The tor process stores keys and other data here.
 
+__CacheDirectory__/**`cached-certs`**::
+    Contains downloaded directory key certificates that are used to verify
+    authenticity of documents generated by the Tor directory authorities.
 
-__CacheDirectory__**/cached-certs**::
-    This file holds downloaded directory key certificates that are used to
-    verify authenticity of documents generated by Tor directory authorities.
-
-__CacheDirectory__**/cached-consensus** and/or **cached-microdesc-consensus**::
+__CacheDirectory__/**`cached-consensus`** and/or **`cached-microdesc-consensus`**::
     The most recent consensus network status document we've downloaded.
 
-__CacheDirectory__**/cached-descriptors** and **cached-descriptors.new**::
-    These files hold downloaded router statuses. Some routers may appear more
-    than once; if so, the most recently published descriptor is used. Lines
-    beginning with @-signs are annotations that contain more information about
-    a given router. The ".new" file is an append-only journal; when it gets
-    too large, all entries are merged into a new cached-descriptors file.
-
-__CacheDirectory__**/cached-extrainfo** and **cached-extrainfo.new**::
-   As "cached-descriptors", but holds optionally-downloaded "extra-info"
-   documents. Relays use these documents to send inessential information
-   about statistics, bandwidth history, and network health to the
-   authorities. They aren't fetched by default; see the DownloadExtraInfo
-   option for more info.
-
-__CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**::
+__CacheDirectory__/**`cached-descriptors`** and **`cached-descriptors.new`**::
+    These files contain the downloaded router statuses. Some routers may appear
+    more than once; if so, the most recently published descriptor is
+    used. Lines beginning with **`@`**-signs are annotations that contain more
+    information about a given router. The **`.new`** file is an append-only
+    journal; when it gets too large, all entries are merged into a new
+    cached-descriptors file.
+
+__CacheDirectory__/**`cached-extrainfo`** and **`cached-extrainfo.new`**::
+    Similar to **cached-descriptors**, but holds optionally-downloaded
+    "extra-info" documents. Relays use these documents to send inessential
+    information about statistics, bandwidth history, and network health to the
+    authorities. They aren't fetched by default. See <<DownloadExtraInfo,DownloadExtraInfo>>
+     for more information.
+
+__CacheDirectory__/**`cached-microdescs`** and **`cached-microdescs.new`**::
     These files hold downloaded microdescriptors.  Lines beginning with
-    @-signs are annotations that contain more information about a given
-    router. The ".new" file is an append-only journal; when it gets too
+    **`@`**-signs are annotations that contain more information about a given
+    router. The **`.new`** file is an append-only journal; when it gets too
     large, all entries are merged into a new cached-microdescs file.
 
-__CacheDirectory__**/cached-routers** and **cached-routers.new**::
-    Obsolete versions of cached-descriptors and cached-descriptors.new. When
-    Tor can't find the newer files, it looks here instead.
-
-__DataDirectory__**/state**::
-    A set of persistent key-value mappings. These are documented in
-    the file. These include:
-            - The current entry guards and their status.
-            - The current bandwidth accounting values.
-            - When the file was last written
-            - What version of Tor generated the state file
-            - A short history of bandwidth usage, as produced in the server
-            descriptors.
-
-__DataDirectory__**/sr-state**::
-    Authority only. State file used to record information about the current
+__DataDirectory__/**`state`**::
+    Contains a set of persistent key-value mappings. These include:
+        - the current entry guards and their status.
+        - the current bandwidth accounting values.
+        - when the file was last written
+        - what version of Tor generated the state file
+        - a short history of bandwidth usage, as produced in the server
+          descriptors.
+
+__DataDirectory__/**`sr-state`**::
+    _Authority only_. This file is used to record information about the current
     status of the shared-random-value voting state.
 
-__CacheDirectory__**/diff-cache**::
-    Directory cache only. Holds older consensuses, and diffs from older
-    consensuses to the most recent consensus of each type, compressed
-    in various ways. Each file contains a set of key-value arguments
-    describing its contents, followed by a single NUL byte, followed by the
-    main file contents.
-
-__DataDirectory__**/bw_accounting**::
-    Used to track bandwidth accounting values (when the current period starts
-    and ends; how much has been read and written so far this period). This file
-    is obsolete, and the data is now stored in the \'state' file instead.
-
-__DataDirectory__**/control_auth_cookie**::
-    Used for cookie authentication with the controller. Location can be
-    overridden by the CookieAuthFile config option. Regenerated on startup. See
+__CacheDirectory__/**`diff-cache`**::
+    _Directory cache only_. Holds older consensuses and diffs from oldest to
+    the most recent consensus of each type compressed in various ways. Each
+    file contains a set of key-value arguments describing its contents,
+    followed by a single NUL byte, followed by the main file contents.
+
+__DataDirectory__/**`bw_accounting`**::
+    This file is obsolete and the data is now stored in the **`state`** file
+    instead. Used to track bandwidth accounting values (when the current period
+    starts and ends; how much has been read and written so far this period).
+
+__DataDirectory__/**`control_auth_cookie`**::
+    This file can be used only when cookie authentication is enabled. Used for
+    cookie authentication with the controller. Location can be overridden by
+    the `CookieAuthFile` configuration option. Regenerated on startup. See
     control-spec.txt in https://spec.torproject.org/[torspec] for details.
-    Only used when cookie authentication is enabled.
 
-__DataDirectory__**/lock**::
-    This file is used to prevent two Tor instances from using same data
-    directory. If access to this file is locked, data directory is already
-    in use by Tor.
+__DataDirectory__/**`lock`**::
+    This file is used to prevent two Tor instances from using the same data
+    directory. If access to this file is locked, data directory is already in
+    use by Tor.
 
-__DataDirectory__**/key-pinning-journal**::
+__DataDirectory__/**`key-pinning-journal`**::
     Used by authorities. A line-based file that records mappings between
-    RSA1024 identity keys and Ed25519 identity keys. Authorities enforce
-    these mappings, so that once a relay has picked an Ed25519 key, stealing
-    or factoring the RSA1024 key will no longer let an attacker impersonate
-    the relay.
+    RSA1024 and Ed25519 identity keys. Authorities enforce these mappings, so
+    that once a relay has picked an Ed25519 key, stealing or factoring the
+    RSA1024 key will no longer let an attacker impersonate the relay.
 
-__KeyDirectory__**/authority_identity_key**::
+__KeyDirectory__/**`authority_identity_key`**::
     A v3 directory authority's master identity key, used to authenticate its
     signing key. Tor doesn't use this while it's running. The tor-gencert
-    program uses this. If you're running an authority, you should keep this
-    key offline, and not actually put it here.
+    program uses this. If you're running an authority, you should keep this key
+    offline, and not put it in this file.
 
-__KeyDirectory__**/authority_certificate**::
-    A v3 directory authority's certificate, which authenticates the authority's
-    current vote- and consensus-signing key using its master identity key.
-    Only directory authorities use this file.
+__KeyDirectory__/**`authority_certificate`**::
+    Only directory authorities use this file. A v3 directory authority's
+    certificate which authenticates the authority's current vote- and
+    consensus-signing key using its master identity key.
 
-__KeyDirectory__**/authority_signing_key**::
-    A v3 directory authority's signing key, used to sign votes and consensuses.
-    Only directory authorities use this file.  Corresponds to the
+__KeyDirectory__/**`authority_signing_key`**::
+    Only directory authorities use this file. A v3 directory authority's
+    signing key that is used to sign votes and consensuses. Corresponds to the
     **authority_certificate** cert.
 
-__KeyDirectory__**/legacy_certificate**::
-    As authority_certificate: used only when V3AuthUseLegacyKey is set.
-    See documentation for V3AuthUseLegacyKey.
+__KeyDirectory__/**`legacy_certificate`**::
+    As authority_certificate; used only when `V3AuthUseLegacyKey` is set.  See
+    documentation for <<V3AuthUseLegacyKey,V3AuthUseLegacyKey>>.
 
-__KeyDirectory__**/legacy_signing_key**::
-    As authority_signing_key: used only when V3AuthUseLegacyKey is set.
-    See documentation for V3AuthUseLegacyKey.
+__KeyDirectory__/**`legacy_signing_key`**::
+    As authority_signing_key: used only when `V3AuthUseLegacyKey` is set.  See
+    documentation for <<V3AuthUseLegacyKey,V3AuthUseLegacyKey>>.
 
-__KeyDirectory__**/secret_id_key**::
+__KeyDirectory__/**`secret_id_key`**::
     A relay's RSA1024 permanent identity key, including private and public
-    components.  Used to sign router descriptors, and to sign other keys.
+    components. Used to sign router descriptors, and to sign other keys.
 
-__KeyDirectory__**/ed25519_master_id_public_key**::
+__KeyDirectory__/**`ed25519_master_id_public_key`**::
     The public part of a relay's Ed25519 permanent identity key.
 
-__KeyDirectory__**/ed25519_master_id_secret_key**::
-    The private part of a relay's Ed25519 permanent identity key.  This key
-    is used to sign the medium-term ed25519 signing key.  This file can be
-    kept offline, or kept encrypted. If so, Tor will not be able to generate
-    new signing keys itself; you'll need to use tor --keygen yourself to do
-    so.
+__KeyDirectory__/**`ed25519_master_id_secret_key`**::
+    The private part of a relay's Ed25519 permanent identity key.  This key is
+    used to sign the medium-term ed25519 signing key.  This file can be kept
+    offline or encrypted. If so, Tor will not be able to generate new signing
+    keys automatically; you'll need to use `tor --keygen` to do so.
 
-__KeyDirectory__**/ed25519_signing_secret_key**::
+__KeyDirectory__/**`ed25519_signing_secret_key`**::
     The private and public components of a relay's medium-term Ed25519 signing
-    key. This key is authenticated by the Ed25519 master key, in turn
+    key. This key is authenticated by the Ed25519 master key, which in turn
     authenticates other keys (and router descriptors).
 
-__KeyDirectory__**/ed25519_signing_cert**::
-    The certificate which authenticates "ed25519_signing_secret_key" as
-    having been signed by the Ed25519 master key.
+__KeyDirectory__/**`ed25519_signing_cert`**::
+    The certificate which authenticates "ed25519_signing_secret_key" as having
+    been signed by the Ed25519 master key.
 
-__KeyDirectory__**/secret_onion_key** and **secret_onion_key.old**::
+__KeyDirectory__/**`secret_onion_key`** and **`secret_onion_key.old`**::
     A relay's RSA1024 short-term onion key. Used to decrypt old-style ("TAP")
-    circuit extension requests. The ".old" file holds the previously
-    generated key, which the relay uses to handle any requests that were
-    made by clients that didn't have the new one.
+    circuit extension requests. The **`.old`** file holds the previously
+    generated key, which the relay uses to handle any requests that were made
+    by clients that didn't have the new one.
 
-__KeyDirectory__**/secret_onion_key_ntor** and **secret_onion_key_ntor.old**::
+__KeyDirectory__/**`secret_onion_key_ntor`** and **`secret_onion_key_ntor.old`**::
     A relay's Curve25519 short-term onion key. Used to handle modern ("ntor")
-    circuit extension requests. The ".old" file holds the previously
-    generated key, which the relay uses to handle any requests that were
-    made by clients that didn't have the new one.
+    circuit extension requests. The **`.old`** file holds the previously
+    generated key, which the relay uses to handle any requests that were made
+    by clients that didn't have the new one.
 
-__DataDirectory__**/fingerprint**::
-    Only used by servers. Holds the fingerprint of the server's identity key.
+__DataDirectory__/**`fingerprint`**::
+    Only used by servers. Contains the fingerprint of the server's identity key.
 
-__DataDirectory__**/hashed-fingerprint**::
-    Only used by bridges. Holds the hashed fingerprint of the bridge's
+__DataDirectory__/**`hashed-fingerprint`**::
+    Only used by bridges. Contains the hashed fingerprint of the bridge's
     identity key. (That is, the hash of the hash of the identity key.)
 
-__DataDirectory__**/approved-routers**::
-    Only used by authoritative directory servers. This file lists
-    the status of routers by their identity fingerprint.
-    Each line lists a status and a fingerprint separated by
-    whitespace. See your **fingerprint** file in the __DataDirectory__ for an
-    example line. If the status is **!reject** then descriptors from the
-    given identity (fingerprint) are rejected by this server. If it is
-    **!invalid** then descriptors are accepted but marked in the directory as
-    not valid, that is, not recommended.
-
-__DataDirectory__**/v3-status-votes**::
-    Only for v3 authoritative directory servers. This file contains
-    status votes from all the authoritative directory servers.
-
-__CacheDirectory__**/unverified-consensus**::
-    This file contains a network consensus document that has been downloaded,
-    but which we didn't have the right certificates to check yet.
-
-__CacheDirectory__**/unverified-microdesc-consensus**::
-    This file contains a microdescriptor-flavored network consensus document
-    that has been downloaded, but which we didn't have the right certificates
-    to check yet.
-
-__DataDirectory__**/unparseable-desc**::
+__DataDirectory__/**`approved-routers`**::
+    Only used by authoritative directory servers. Each line lists a status and
+    an identity, separated by whitespace. Identities can be hex-encoded RSA
+    fingerprints, or base-64 encoded ed25519 public keys. See the
+    **fingerprint** file in a tor relay's __DataDirectory__ for an example
+    fingerprint line. If the status is **!reject**, then descriptors from the
+    given identity are rejected by this server. If it is **!invalid** then
+    descriptors are accepted, but marked in the directory as not valid, that
+    is, not recommended. In either case, the corresponding relays are not
+    included in the consensus.
+
+__DataDirectory__/**`v3-status-votes`**::
+    Only for v3 authoritative directory servers. This file contains status
+    votes from all the authoritative directory servers.
+
+__CacheDirectory__/**`unverified-consensus`**::
+    Contains a network consensus document that has been downloaded, but which
+    we didn't have the right certificates to check yet.
+
+__CacheDirectory__/**`unverified-microdesc-consensus`**::
+    Contains a microdescriptor-flavored network consensus document that has
+    been downloaded, but which we didn't have the right certificates to check
+    yet.
+
+__DataDirectory__/**`unparseable-desc`**::
     Onion server descriptors that Tor was unable to parse are dumped to this
     file. Only used for debugging.
 
-__DataDirectory__**/router-stability**::
+__DataDirectory__/**`router-stability`**::
     Only used by authoritative directory servers. Tracks measurements for
-    router mean-time-between-failures so that authorities have a good idea of
+    router mean-time-between-failures so that authorities have a fair idea of
     how to set their Stable flags.
 
-__DataDirectory__**/stats/dirreq-stats**::
+__DataDirectory__/**`stats/dirreq-stats`**::
     Only used by directory caches and authorities. This file is used to
     collect directory request statistics.
 
-__DataDirectory__**/stats/entry-stats**::
+__DataDirectory__/**`stats/entry-stats`**::
     Only used by servers. This file is used to collect incoming connection
     statistics by Tor entry nodes.
 
-__DataDirectory__**/stats/bridge-stats**::
+__DataDirectory__/**`stats/bridge-stats`**::
     Only used by servers. This file is used to collect incoming connection
     statistics by Tor bridges.
 
-__DataDirectory__**/stats/exit-stats**::
+__DataDirectory__/**`stats/exit-stats`**::
     Only used by servers. This file is used to collect outgoing connection
     statistics by Tor exit routers.
 
-__DataDirectory__**/stats/buffer-stats**::
+__DataDirectory__/**`stats/buffer-stats`**::
     Only used by servers. This file is used to collect buffer usage
     history.
 
-__DataDirectory__**/stats/conn-stats**::
+__DataDirectory__/**`stats/conn-stats`**::
     Only used by servers. This file is used to collect approximate connection
     history (number of active connections over time).
 
-__DataDirectory__**/stats/hidserv-stats**::
+__DataDirectory__/**`stats/hidserv-stats`**::
     Only used by servers. This file is used to collect approximate counts
     of what fraction of the traffic is hidden service rendezvous traffic, and
     approximately how many hidden services the relay has seen.
 
-__DataDirectory__**/networkstatus-bridges**::
+__DataDirectory__/**networkstatus-bridges`**::
     Only used by authoritative bridge directories. Contains information
     about bridges that have self-reported themselves to the bridge
     authority.
 
-__DataDirectory__**/approved-routers**::
-    Authorities only. This file is used to configure which relays are
-    known to be valid, invalid, and so forth.
-
-__HiddenServiceDirectory__**/hostname**::
+__HiddenServiceDirectory__/**`hostname`**::
     The <base32-encoded-fingerprint>.onion domain name for this hidden service.
     If the hidden service is restricted to authorized clients only, this file
     also contains authorization data for all clients.
- +
-    Note that clients will ignore any extra subdomains prepended to a hidden
-    service hostname. So if you have "xyz.onion" as your hostname, you
-    can tell clients to connect to "www.xyz.onion" or "irc.xyz.onion"
++
+[NOTE]
+    The clients will ignore any extra subdomains prepended to a hidden
+    service hostname. Supposing you have "xyz.onion" as your hostname, you
+    can ask your clients to connect to "www.xyz.onion" or "irc.xyz.onion"
     for virtual-hosting purposes.
 
-__HiddenServiceDirectory__**/private_key**::
-    The private key for this hidden service.
+__HiddenServiceDirectory__/**`private_key`**::
+    Contains the private key for this hidden service.
 
-__HiddenServiceDirectory__**/client_keys**::
-    Authorization data for a hidden service that is only accessible by
+__HiddenServiceDirectory__/**`client_keys`**::
+    Contains authorization data for a hidden service that is only accessible by
     authorized clients.
 
-__HiddenServiceDirectory__**/onion_service_non_anonymous**::
+__HiddenServiceDirectory__/**`onion_service_non_anonymous`**::
     This file is present if a hidden service key was created in
     **HiddenServiceNonAnonymousMode**.
 
-SEE ALSO
---------
-**torsocks**(1), **torify**(1) +
+== SEE ALSO
 
-**https://www.torproject.org/**
+For more information, refer to the Tor Project website at
+https://www.torproject.org/ and the Tor specifications at
+https://spec.torproject.org. See also **torsocks**(1) and **torify**(1).
 
-**torspec: https://spec.torproject.org **
+== BUGS
 
-BUGS
-----
+Because Tor is still under development, there may be plenty of bugs. Please
+report them at https://trac.torproject.org/.
 
-Plenty, probably. Tor is still in development. Please report them at https://trac.torproject.org/.
+== AUTHORS
 
-AUTHORS
--------
 Roger Dingledine [arma at mit.edu], Nick Mathewson [nickm at alum.mit.edu].
diff --git a/doc/torify.1.txt b/doc/torify.1.txt
index 7e49081cfc..716625f92d 100644
--- a/doc/torify.1.txt
+++ b/doc/torify.1.txt
@@ -1,7 +1,7 @@
 // Copyright (c) The Tor Project, Inc.
 // See LICENSE for licensing information
 // This is an asciidoc file used to generate the manpage/html reference.
-// Learn asciidoc on http://www.methods.co.nz/asciidoc/userguide.html
+// Learn asciidoc on https://www.methods.co.nz/asciidoc/userguide.html
 :man source:   Tor
 :man manual:   Tor Manual
 torify(1)
diff --git a/m4/ax_check_sign.m4 b/m4/ax_check_sign.m4
index d67e114dba..b6285012f2 100644
--- a/m4/ax_check_sign.m4
+++ b/m4/ax_check_sign.m4
@@ -1,5 +1,5 @@
 # ===========================================================================
-#       http://www.gnu.org/software/autoconf-archive/ax_check_sign.html
+#       https://www.gnu.org/software/autoconf-archive/ax_check_sign.html
 # ===========================================================================
 #
 # SYNOPSIS
diff --git a/m4/pkg.m4 b/m4/pkg.m4
index c5b26b52e6..12e9835fba 100644
--- a/m4/pkg.m4
+++ b/m4/pkg.m4
@@ -148,7 +148,7 @@ path to pkg-config.
 
 _PKG_TEXT
 
-To get pkg-config, see <http://pkg-config.freedesktop.org/>.])[]dnl
+To get pkg-config, see <https://pkg-config.freedesktop.org/>.])[]dnl
         ])
 else
 	$1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS
diff --git a/m4/pprint.m4 b/m4/pprint.m4
new file mode 100644
index 0000000000..4e8a71c876
--- /dev/null
+++ b/m4/pprint.m4
@@ -0,0 +1,228 @@
+# Pretty printing macros.
+#
+# Copyright (C) 2019 - Philippe Proulx <pproulx@efficios.com>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License, version 2 only, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# As a special exception to the GNU General Public License, if you distribute
+# this file as part of a program that contains a configuration script
+# generated by Autoconf, you may include it under the same distribution terms
+# that you use for the rest of that program.
+
+#serial 1
+
+# PPRINT_INIT(): initializes the pretty printing system.
+#
+# Use this macro before using any other PPRINT_* macro.
+AC_DEFUN([PPRINT_INIT], [
+  m4_define([PPRINT_CONFIG_TS], [50])
+  m4_define([PPRINT_CONFIG_INDENT], [2])
+  PPRINT_YES_MSG=yes
+  PPRINT_NO_MSG=no
+
+  # find tput, which tells us if colors are supported and gives us color codes
+  AC_PATH_PROG([pprint_tput], [tput])
+
+  AS_IF([test -n "$pprint_tput"], [
+    AS_IF([test -n "$PS1" && test `"$pprint_tput" colors` -ge 8 && test -t 1], [
+      # interactive shell and colors supported and standard output
+      # file descriptor is opened on a terminal
+      PPRINT_COLOR_TXTBLK="`"$pprint_tput" setaf 0`"
+      PPRINT_COLOR_TXTBLU="`"$pprint_tput" setaf 4`"
+      PPRINT_COLOR_TXTGRN="`"$pprint_tput" setaf 2`"
+      PPRINT_COLOR_TXTCYN="`"$pprint_tput" setaf 6`"
+      PPRINT_COLOR_TXTRED="`"$pprint_tput" setaf 1`"
+      PPRINT_COLOR_TXTPUR="`"$pprint_tput" setaf 5`"
+      PPRINT_COLOR_TXTYLW="`"$pprint_tput" setaf 3`"
+      PPRINT_COLOR_TXTWHT="`"$pprint_tput" setaf 7`"
+      PPRINT_COLOR_BLD=`"$pprint_tput" bold`
+      PPRINT_COLOR_BLDBLK="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTBLK"
+      PPRINT_COLOR_BLDBLU="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTBLU"
+      PPRINT_COLOR_BLDGRN="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTGRN"
+      PPRINT_COLOR_BLDCYN="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTCYN"
+      PPRINT_COLOR_BLDRED="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTRED"
+      PPRINT_COLOR_BLDPUR="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTPUR"
+      PPRINT_COLOR_BLDYLW="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTYLW"
+      PPRINT_COLOR_BLDWHT="$PPRINT_COLOR_BLD$PPRINT_COLOR_TXTWHT"
+      PPRINT_COLOR_RST="`"$pprint_tput" sgr0`"
+
+      # colored yes and no
+      PPRINT_YES_MSG="$PPRINT_COLOR_BLDGRN$PPRINT_YES_MSG$PPRINT_COLOR_RST"
+      PPRINT_NO_MSG="$PPRINT_COLOR_BLDRED$PPRINT_NO_MSG$PPRINT_COLOR_RST"
+
+      # subtitle color
+      PPRINT_COLOR_SUBTITLE="$PPRINT_COLOR_BLDCYN"
+    ])
+  ])
+])
+
+# PPRINT_SET_INDENT(indent): sets the current indentation.
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_SET_INDENT], [
+  m4_define([PPRINT_CONFIG_INDENT], [$1])
+])
+
+# PPRINT_SET_TS(ts): sets the current tab stop.
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_SET_TS], [
+  m4_define([PPRINT_CONFIG_TS], [$1])
+])
+
+# PPRINT_SUBTITLE(subtitle): pretty prints a subtitle.
+#
+# The subtitle is put as is in a double-quoted shell string so the user
+# needs to escape ".
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_SUBTITLE], [
+  AS_ECHO(["${PPRINT_COLOR_SUBTITLE}$1$PPRINT_COLOR_RST"])
+])
+
+AC_DEFUN([_PPRINT_INDENT], [
+  m4_if(PPRINT_CONFIG_INDENT, 0, [
+  ], [
+    m4_for([pprint_i], [0], m4_eval(PPRINT_CONFIG_INDENT * 2 - 1), [1], [
+      AS_ECHO_N([" "])
+    ])
+  ])
+])
+
+# PPRINT_PROP_STRING(title, value, title_color?): pretty prints a
+# string property.
+#
+# The title is put as is in a double-quoted shell string so the user
+# needs to escape ".
+#
+# The $PPRINT_CONFIG_INDENT variable must be set to the desired indentation
+# level.
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_PROP_STRING], [
+  m4_pushdef([pprint_title], [$1])
+  m4_pushdef([pprint_value], [$2])
+  m4_pushdef([pprint_title_color], m4_default([$3], []))
+  m4_pushdef([pprint_title_len], m4_len(pprint_title))
+  m4_pushdef([pprint_spaces_cnt], m4_eval(PPRINT_CONFIG_TS - pprint_title_len - (PPRINT_CONFIG_INDENT * 2) - 1))
+
+  m4_if(m4_eval(pprint_spaces_cnt <= 0), [1], [
+    m4_define([pprint_spaces_cnt], [1])
+  ])
+
+  m4_pushdef([pprint_spaces], [])
+
+  m4_for([pprint_i], 0, m4_eval(pprint_spaces_cnt - 1), [1], [
+    m4_append([pprint_spaces], [ ])
+  ])
+
+  _PPRINT_INDENT
+
+  AS_ECHO_N(["pprint_title_color""pprint_title$PPRINT_COLOR_RST:pprint_spaces"])
+  AS_ECHO(["${PPRINT_COLOR_BLD}pprint_value$PPRINT_COLOR_RST"])
+
+  m4_popdef([pprint_spaces])
+  m4_popdef([pprint_spaces_cnt])
+  m4_popdef([pprint_title_len])
+  m4_popdef([pprint_title_color])
+  m4_popdef([pprint_value])
+  m4_popdef([pprint_title])
+])
+
+# PPRINT_PROP_BOOL(title, value, title_color?): pretty prints a boolean
+# property.
+#
+# The title is put as is in a double-quoted shell string so the user
+# needs to escape ".
+#
+# The value is evaluated at shell runtime. Its evaluation must be
+# 0 (false) or 1 (true).
+#
+# Uses the PPRINT_PROP_STRING() with the "yes" or "no" string.
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_PROP_BOOL], [
+  m4_pushdef([pprint_title], [$1])
+  m4_pushdef([pprint_value], [$2])
+
+  test pprint_value -eq 0 && pprint_msg="$PPRINT_NO_MSG" || pprint_msg="$PPRINT_YES_MSG"
+
+  m4_if([$#], [3], [
+    PPRINT_PROP_STRING(pprint_title, [$pprint_msg], $3)
+  ], [
+    PPRINT_PROP_STRING(pprint_title, [$pprint_msg])
+  ])
+
+  m4_popdef([pprint_value])
+  m4_popdef([pprint_title])
+])
+
+# PPRINT_PROP_BOOL_CUSTOM(title, value, no_msg, title_color?): pretty prints a boolean
+# property.
+#
+# The title is put as is in a double-quoted shell string so the user
+# needs to escape ".
+#
+# The value is evaluated at shell runtime. Its evaluation must be
+# 0 (false) or 1 (true).
+#
+# Uses the PPRINT_PROP_STRING() with the "yes" or "no" string.
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_PROP_BOOL_CUSTOM], [
+  m4_pushdef([pprint_title], [$1])
+  m4_pushdef([pprint_value], [$2])
+  m4_pushdef([pprint_value_no_msg], [$3])
+
+  test pprint_value -eq 0 && pprint_msg="$PPRINT_NO_MSG (pprint_value_no_msg)" || pprint_msg="$PPRINT_YES_MSG"
+
+  m4_if([$#], [4], [
+    PPRINT_PROP_STRING(pprint_title, [$pprint_msg], $4)
+  ], [
+    PPRINT_PROP_STRING(pprint_title, [$pprint_msg])
+  ])
+
+  m4_popdef([pprint_value_no_msg])
+  m4_popdef([pprint_value])
+  m4_popdef([pprint_title])
+])
+
+# PPRINT_WARN(msg): pretty prints a warning message.
+#
+# The message is put as is in a double-quoted shell string so the user
+# needs to escape ".
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_WARN], [
+  m4_pushdef([pprint_msg], [$1])
+
+  _PPRINT_INDENT
+  AS_ECHO(["${PPRINT_COLOR_TXTYLW}WARNING:$PPRINT_COLOR_RST ${PPRINT_COLOR_BLDYLW}pprint_msg$PPRINT_COLOR_RST"])
+
+  m4_popdef([pprint_msg])
+])
+
+# PPRINT_ERROR(msg): pretty prints an error message and exits.
+#
+# The message is put as is in a double-quoted shell string so the user
+# needs to escape ".
+#
+# Use PPRINT_INIT() before using this macro.
+AC_DEFUN([PPRINT_ERROR], [
+  m4_pushdef([pprint_msg], [$1])
+
+  AC_MSG_ERROR([${PPRINT_COLOR_BLDRED}pprint_msg$PPRINT_COLOR_RST])
+
+  m4_popdef([pprint_msg])
+])
diff --git a/scripts/coccinelle/apply.sh b/scripts/coccinelle/apply.sh
new file mode 100755
index 0000000000..f531d7fa32
--- /dev/null
+++ b/scripts/coccinelle/apply.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# apply.sh:
+# run spatch with appropriate includes and builtins for the Tor source code
+
+top="$(dirname "$0")/../.."
+
+spatch -macro_file_builtins "$top"/scripts/coccinelle/tor-coccinelle.h \
+       -I "$top" -I "$top"/src -I "$top"/ext --defined COCCI "$@"
diff --git a/scripts/coccinelle/check_cocci_parse.sh b/scripts/coccinelle/check_cocci_parse.sh
new file mode 100755
index 0000000000..aaa586c093
--- /dev/null
+++ b/scripts/coccinelle/check_cocci_parse.sh
@@ -0,0 +1,98 @@
+#!/bin/sh
+
+# If we have coccinelle installed, run try_parse.sh on every filename passed
+# as an argument. If no filenames are supplied, scan a standard Tor 0.3.5 or
+# later directory layout.
+#
+# Uses the default coccinelle exceptions file, or $TOR_COCCI_EXCEPTIONS_FILE,
+# if it is set.
+#
+# Use TOR_COCCI_EXCEPTIONS_FILE=/dev/null check_cocci_parse.sh to disable
+# the default exception file.
+#
+# If spatch is not installed, remind the user to install it, but exit with
+# a success error status.
+
+scripts_cocci="$(dirname "$0")"
+top="$scripts_cocci/../.."
+try_parse="$scripts_cocci/try_parse.sh"
+
+exitcode=0
+
+export TOR_COCCI_EXCEPTIONS_FILE="${TOR_COCCI_EXCEPTIONS_FILE:-$scripts_cocci/exceptions.txt}"
+
+PURPOSE="cocci C parsing"
+
+echo "Checking spatch:"
+
+if ! command -v spatch ; then
+    echo "Install coccinelle's spatch to check $PURPOSE."
+    exit "$exitcode"
+fi
+
+# Returns true if $1 is greater than or equal to $2
+version_ge()
+{
+    if test "$1" = "$2" ; then
+        # return true
+        return 0
+    fi
+    LOWER_VERSION="$(printf '%s\n' "$1" "$2" | $SORT_V | head -n 1)"
+    # implicit return
+    test "$LOWER_VERSION" != "$1"
+}
+
+# 'sort -V' is a gnu extension
+SORT_V="sort -V"
+# Use 'sort -n' if 'sort -V' doesn't work
+if ! version_ge "1" "0" ; then
+    echo "Your 'sort -V' command appears broken. Falling back to 'sort -n'."
+    echo "Some spatch version checks may give the wrong result."
+    SORT_V="sort -n"
+fi
+
+# Print the full spatch version, for diagnostics
+spatch --version
+
+MIN_SPATCH_V="1.0.4"
+# This pattern needs to handle version strings like:
+# spatch version 1.0.0-rc19
+# spatch version 1.0.6 compiled with OCaml version 4.05.0
+SPATCH_V=$(spatch --version | head -1 | \
+               sed 's/spatch version \([0-9][^ ]*\).*/\1/')
+
+if ! version_ge "$SPATCH_V" "$MIN_SPATCH_V" ; then
+    echo "Tor requires coccinelle spatch >= $MIN_SPATCH_V to check $PURPOSE."
+    echo "But you have $SPATCH_V. Please install a newer version."
+    exit "$exitcode"
+fi
+
+if test $# -ge 1 ; then
+  "$try_parse" "$@"
+  exitcode=$?
+else
+  cd "$top" || exit 1
+  # This is the layout in 0.3.5
+  # Keep these lists consistent:
+  #   - OWNED_TOR_C_FILES in Makefile.am
+  #   - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook
+  #   - try_parse in check_cocci_parse.sh
+  "$try_parse" \
+    src/lib/*/*.[ch] \
+    src/core/*/*.[ch] \
+    src/feature/*/*.[ch] \
+    src/app/*/*.[ch] \
+    src/test/*.[ch] \
+    src/test/*/*.[ch] \
+    src/tools/*.[ch]
+  exitcode=$?
+fi
+
+if test "$exitcode" != 0 ; then
+    echo "Please fix these $PURPOSE errors in the above files"
+    echo "Set VERBOSE=1 for more details"
+    echo "Try running test-operator-cleanup or 'make autostyle-operators'"
+    echo "As a last resort, you can modify scripts/coccinelle/exceptions.txt"
+fi
+
+exit "$exitcode"
diff --git a/scripts/coccinelle/ctrl-reply-cleanup.cocci b/scripts/coccinelle/ctrl-reply-cleanup.cocci
new file mode 100644
index 0000000000..f085cd4684
--- /dev/null
+++ b/scripts/coccinelle/ctrl-reply-cleanup.cocci
@@ -0,0 +1,43 @@
+// Script to clean up after ctrl-reply.cocci -- run as a separate step
+// because cleanup_write2 (even when disabled) somehow prevents the
+// match rule in ctrl-reply.cocci from matching.
+
+// If it doesn't have to be a printf, turn it into a write
+
+@ cleanup_write @
+expression E;
+constant code, s;
+@@
+-control_printf_endreply(E, code, s)
++control_write_endreply(E, code, s)
+
+// Use send_control_done() instead of explicitly writing it out
+@ cleanup_send_done @
+type T;
+identifier f != send_control_done;
+expression E;
+@@
+ T f(...) {
+<...
+-control_write_endreply(E, 250, "OK")
++send_control_done(E)
+ ...>
+ }
+
+// Clean up more printfs that could be writes
+//
+// For some reason, including this rule, even disabled, causes the
+// match rule in ctrl-reply.cocci to fail to match some code that has
+// %s in its format strings
+
+@ cleanup_write2 @
+expression E1, E2;
+constant code;
+@@
+(
+-control_printf_endreply(E1, code, "%s", E2)
++control_write_endreply(E1, code, E2)
+|
+-control_printf_midreply(E1, code, "%s", E2)
++control_write_midreply(E1, code, E2)
+)
diff --git a/scripts/coccinelle/ctrl-reply.cocci b/scripts/coccinelle/ctrl-reply.cocci
new file mode 100644
index 0000000000..d6e9aeedd7
--- /dev/null
+++ b/scripts/coccinelle/ctrl-reply.cocci
@@ -0,0 +1,87 @@
+// Script to edit control_*.c for refactored control reply output functions
+
+@ initialize:python @
+@@
+import re
+from coccilib.report import *
+
+# reply strings "NNN-foo", "NNN+foo", "NNN foo", etc.
+r = re.compile(r'^"(\d+)([ +-])(.*)\\r\\n"$')
+
+# Generate name of function to call based on which separator character
+# comes between the numeric code and the text
+def idname(sep, base):
+    if sep == '+':
+        return base + "datareply"
+    elif sep == '-':
+        return base + "midreply"
+    else:
+        return base + "endreply"
+
+# Generate the actual replacements used by the rules
+def gen(s, base, p):
+    pos = p[0]
+    print_report(pos, "%s %s" % (base, s))
+    m = r.match(s)
+    if m is None:
+        # String not correct format, so fail match
+        cocci.include_match(False)
+        print_report(pos, "BAD STRING %s" % s)
+        return
+
+    code, sep, s1 = m.groups()
+
+    if r'\r\n' in s1:
+        # Extra CRLF in string, so fail match
+        cocci.include_match(False)
+        print_report(pos, "extra CRLF in string %s" % s)
+        return
+
+    coccinelle.code = code
+    # Need a string that is a single C token, because Coccinelle only allows
+    # "identifiers" to be output from Python scripts?
+    coccinelle.body = '"%s"' % s1
+    coccinelle.id = idname(sep, base)
+    return
+
+@ match @
+identifier f;
+position p;
+expression E;
+constant s;
+@@
+(
+ connection_printf_to_buf@f@p(E, s, ...)
+|
+ connection_write_str_to_buf@f@p(s, E)
+)
+
+@ script:python sc1 @
+s << match.s;
+p << match.p;
+f << match.f;
+id;
+body;
+code;
+@@
+if f == 'connection_printf_to_buf':
+    gen(s, 'control_printf_', p)
+elif f == 'connection_write_str_to_buf':
+    gen(s, 'control_write_', p)
+else:
+    raise(ValueError("%s: %s" % (f, s)))
+
+@ replace @
+constant match.s;
+expression match.E;
+identifier match.f;
+identifier sc1.body, sc1.id, sc1.code;
+@@
+(
+-connection_write_str_to_buf@f(s, E)
++id(E, code, body)
+|
+-connection_printf_to_buf@f(E, s
++id(E, code, body
+ , ...)
+)
diff --git a/scripts/coccinelle/debugmm.cocci b/scripts/coccinelle/debugmm.cocci
new file mode 100644
index 0000000000..dbd308df33
--- /dev/null
+++ b/scripts/coccinelle/debugmm.cocci
@@ -0,0 +1,29 @@
+// Look for use of expressions with side-effects inside of debug logs.
+//
+// This script detects expressions like ++E, --E, E++, and E-- inside of
+// calls to log_debug().
+//
+// The log_debug() macro exits early if debug logging is not enabled,
+// potentially causing problems if its arguments have side-effects.
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  --E ...+>, ... );
+
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  ++E ...+>, ... );
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  E-- ...+>, ... );
+
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  E++ ...+>, ... );
diff --git a/scripts/coccinelle/exceptions.txt b/scripts/coccinelle/exceptions.txt
new file mode 100644
index 0000000000..473f4b22c5
--- /dev/null
+++ b/scripts/coccinelle/exceptions.txt
@@ -0,0 +1,24 @@
+# A list of exception patterns for check_cocci_parse.sh
+# Passed to 'grep -f'
+src/lib/cc/compat_compiler.h
+src/lib/container/handles.h
+src/lib/container/map.c
+src/lib/container/map.h
+src/lib/container/order.c
+src/lib/crypt_ops/crypto_rand.c
+src/lib/fs/files.h
+src/lib/log/util_bug.c
+src/lib/pubsub/pubsub_macros.h
+src/lib/smartlist_core/smartlist_foreach.h
+src/lib/testsupport/testsupport.h
+src/lib/tls/tortls.h
+src/lib/tls/tortls_openssl.c
+src/lib/tls/x509.h
+src/lib/version/version.c
+src/core/mainloop/connection.c
+src/core/or/reasons.c
+src/feature/dirclient/dirclient.c
+src/feature/nodelist/networkstatus.c
+src/test/test_address.c
+src/test/test_hs_cache.c
+src/test/test_hs_descriptor.c
diff --git a/scripts/coccinelle/test-operator-cleanup b/scripts/coccinelle/test-operator-cleanup
index e7822542a4..28b4d4f588 100755
--- a/scripts/coccinelle/test-operator-cleanup
+++ b/scripts/coccinelle/test-operator-cleanup
@@ -1,4 +1,17 @@
 #!/usr/bin/perl -w -p -i
+#
+# Copyright (c) 2001 Matej Pfajfar.
+# Copyright (c) 2001-2004, Roger Dingledine.
+# Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+# Copyright (c) 2007-2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+# This script looks for instances of C comparison operators as macro arguments,
+# and replaces them with our OP_* equivalents.
+#
+# Some macros that take operators are our tt_int_op() testing macro, and the
+# standard timercmp() macro.  Coccinelle can't handle their syntax, however,
+# unless we give them their operators as a macro too.
 
 next if m#^ */\*# or m#^ *\* #;
 
diff --git a/scripts/coccinelle/tor-coccinelle.h b/scripts/coccinelle/tor-coccinelle.h
new file mode 100644
index 0000000000..44d79325eb
--- /dev/null
+++ b/scripts/coccinelle/tor-coccinelle.h
@@ -0,0 +1,63 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * This file looks like a C header, but its purpose is a bit different.
+ *
+ * We never include it from our real C files; we only tell Coccinelle
+ * about it in apply.sh.
+ *
+ * It tells the Coccinelle semantic patching tool how to understand
+ * things that would otherwise not be good C syntax, or which would
+ * otherwise not make sense to it as C.  It doesn't need to produce
+ * semantically equivalent C, or even correct C: it only has to produce
+ * syntactically valid C.
+ */
+
+#define MOCK_DECL(a, b, c) a b c
+#define MOCK_IMPL(a, b, c) a b c
+#define CHECK_PRINTF(a, b)
+#define CHECK_SCANF(a, b)
+#define STATIC static
+#define EXTERN(a,b) extern a b;
+
+#define STMT_BEGIN do {
+#define STMT_END } while (0)
+
+#define BUG(x) (x)
+#define IF_BUG_ONCE(x) if (x)
+
+#define ATTR_NORETURN
+#define ATTR_UNUSED
+#define ATTR_CONST
+#define ATTR_MALLOC
+#define ATTR_WUR
+#define DISABLE_GCC_WARNING(x)
+#define ENABLE_GCC_WARNING(x)
+
+#define HANDLE_DECL(a,b,c)
+#define HANDLE_IMPL(a,b,c)
+#define HT_ENTRY(x) void *
+#define HT_HEAD(a,b) struct ht_head
+#define HT_INITIALIZER() { }
+#define X509 struct x509_st
+#define STACK_OF(x) struct foo_stack_t
+#define TOR_TAILQ_HEAD(a,b) struct tailq_head
+#define TOR_TAILQ_ENTRY(a) struct tailq_entry
+#define TOR_SIMPLEQ_HEAD(a,b) struct simpleq_entry
+#define TOR_SIMPLEQ_ENTRY(a) struct simpleq_entry
+#define TOR_LIST_HEAD(a,b) struct list_head
+#define TOR_LIST_ENTRY(a) struct list_entry
+#define TOR_SLIST_HEAD(a,b) struct slist_head
+#define TOR_SLIST_ENTRY(a) struct slist_entry
+
+#define NS_DECL(a, b, c) a b c
+#define NS(a) a
+
+#define CONF_TEST_MEMBERS(a,b,c)
+#define DUMMY_CONF_TEST_MEMBERS
+
+#define EAT_SEMICOLON extern int dummy__;
diff --git a/scripts/coccinelle/try_parse.sh b/scripts/coccinelle/try_parse.sh
new file mode 100755
index 0000000000..a90e51b4aa
--- /dev/null
+++ b/scripts/coccinelle/try_parse.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Echo the name of every argument of this script that is not "perfect"
+# according to coccinelle's --parse-c.
+#
+# If $TOR_COCCI_EXCEPTIONS_FILE is non-empty, skip any files that match the
+# patterns in the exception file, according to "grep -f"
+#
+# If VERBOSE is non-empty, log spatch errors and skipped files.
+
+top="$(dirname "$0")/../.."
+
+exitcode=0
+
+for fn in "$@"; do
+
+    if test "${TOR_COCCI_EXCEPTIONS_FILE}" ; then
+        skip_fn=$(echo "$fn" | grep -f "${TOR_COCCI_EXCEPTIONS_FILE}")
+        if test "${skip_fn}" ; then
+            if test "${VERBOSE}" != ""; then
+                echo "Skipping '${skip_fn}'"
+            fi
+            continue
+        fi
+    fi
+
+    if spatch --macro-file-builtins \
+              "$top"/scripts/coccinelle/tor-coccinelle.h \
+              --defined COCCI \
+              --parse-c "$fn" \
+              2>/dev/null | grep "perfect = 1" > /dev/null; then
+        : # it's perfect
+    else
+        echo "$fn"
+        if test "${VERBOSE}" != ""; then
+            spatch --macro-file-builtins \
+                   "$top"/scripts/coccinelle/tor-coccinelle.h \
+                   --defined COCCI \
+                   --parse-c "$fn"
+        fi
+        exitcode=1
+    fi
+
+done
+
+exit "$exitcode"
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py
index 3c948d87cf..ae50563074 100755
--- a/scripts/codegen/fuzzing_include_am.py
+++ b/scripts/codegen/fuzzing_include_am.py
@@ -1,4 +1,9 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 FUZZERS = """
 	consensus
@@ -13,6 +18,7 @@ FUZZERS = """
 	iptsv2
 	microdesc
 	socks
+	strops
 	vrs
 """
 
@@ -23,12 +29,12 @@ FUZZING_CPPFLAGS = \
 FUZZING_CFLAGS = \
 	$(AM_CFLAGS) $(TEST_CFLAGS)
 FUZZING_LDFLAG = \
-	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
+	@TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
 FUZZING_LIBS = \
 	$(TOR_INTERNAL_TESTING_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	@TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
+	@TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
 	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
 	@TOR_SYSTEMD_LIBS@ \
 	@TOR_LZMA_LIBS@ \
diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py
index 5d326f8b9e..8c88e54a13 100755
--- a/scripts/codegen/gen_server_ciphers.py
+++ b/scripts/codegen/gen_server_ciphers.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright 2014-2019, The Tor Project, Inc
 # See LICENSE for licensing information
 
@@ -8,6 +8,11 @@
 #
 # Run it on all the files in your openssl include directory.
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import re
 import sys
 
@@ -96,7 +101,7 @@ def parse_cipher(ciph):
         fwsec, = m.groups()
         return Ciphersuite(ciph, fwsec, "CHACHA20", "256", "POLY1305", "n/a")
 
-    print "/* Couldn't parse %s ! */"%ciph
+    print("/* Couldn't parse %s ! */"%ciph)
     return None
 
 
@@ -120,12 +125,12 @@ for c in ALL_CIPHERS:
         colon = ' ":"'
 
     if c.name in MANDATORY:
-        print "%s/* Required */"%indent
-        print '%s%s%s'%(indent,c.name,colon)
+        print("%s/* Required */"%indent)
+        print('%s%s%s'%(indent,c.name,colon))
     else:
-        print "#ifdef %s"%c.name
-        print '%s%s%s'%(indent,c.name,colon)
-        print "#endif"
+        print("#ifdef %s"%c.name)
+        print('%s%s%s'%(indent,c.name,colon))
+        print("#endif")
 
-print '%s;'%indent
+print('%s;'%indent)
 
diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py
index f23f2f1e6f..ff01dd8719 100755
--- a/scripts/codegen/get_mozilla_ciphers.py
+++ b/scripts/codegen/get_mozilla_ciphers.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # coding=utf-8
 # Copyright 2011-2019, The Tor Project, Inc
 # original version by Arturo Filastò
@@ -10,12 +10,17 @@
 # It takes two arguments: the location of a firefox source directory, and the
 # location of an openssl source directory.
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import os
 import re
 import sys
 
 if len(sys.argv) != 3:
-    print >>sys.stderr, "Syntax: get_mozilla_ciphers.py <firefox-source-dir> <openssl-source-dir>"
+    print("Syntax: get_mozilla_ciphers.py <firefox-source-dir> <openssl-source-dir>", file=sys.stderr)
     sys.exit(1)
 
 ff_root = sys.argv[1]
@@ -171,13 +176,13 @@ for fl in oSSLinclude:
     fp.close()
 
 # Now generate the output.
-print """\
+print("""\
 /* This is an include file used to define the list of ciphers clients should
  * advertise.  Before including it, you should define the CIPHER and XCIPHER
  * macros.
  *
  * This file was automatically generated by get_mozilla_ciphers.py.
- */"""
+ */""")
 # Go in order by the order in CipherPrefs
 for firefox_macro in firefox_ciphers:
 
@@ -210,4 +215,4 @@ for firefox_macro in firefox_ciphers:
 #else
    XCIPHER(%(hex)s, %(macro)s)
 #endif""" % format
-    print res
+    print(res)
diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py
index efca4dda9a..48d1d31a02 100644
--- a/scripts/codegen/makedesc.py
+++ b/scripts/codegen/makedesc.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright 2014-2019, The Tor Project, Inc.
 # See LICENSE for license information
 
@@ -9,6 +9,11 @@
 # I've used this to make inputs for unit tests.  I wouldn't suggest
 # using it for anything else.
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import base64
 import binascii
 import ctypes
@@ -19,12 +24,16 @@ import os
 import re
 import struct
 import time
-import UserDict
 
 import slow_ed25519
 import slownacl_curve25519
 import ed25519_exts_ref
 
+try:
+    xrange  # Python 2
+except NameError:
+    xrange = range  # Python 3
+
 # Pull in the openssl stuff we need.
 
 crypt = ctypes.CDLL(ctypes.util.find_library('crypto'))
@@ -247,8 +256,8 @@ class OnDemandKeys(object):
 
 
 def signdesc(body, args_out=None):
-    rsa, ident_pem, id_digest = make_key()
-    _, onion_pem, _ = make_key()
+    rsa, ident_pem, id_digest = make_rsa_key()
+    _, onion_pem, _ = make_rsa_key()
 
     need_ed = '{ED25519-CERT}' in body or '{ED25519-SIGNATURE}' in body
     if need_ed:
@@ -298,10 +307,10 @@ def signdesc(body, args_out=None):
     return body.rstrip()
 
 def print_c_string(ident, body):
-    print "static const char %s[] =" % ident
+    print("static const char %s[] =" % ident)
     for line in body.split("\n"):
-        print '  "%s\\n"' %(line)
-    print "  ;"
+        print('  "%s\\n"' %(line))
+    print("  ;")
 
 def emit_ri(name, body):
     info = OnDemandKeys()
@@ -315,8 +324,8 @@ def emit_ei(name, body):
     body = info.sign_desc(body)
     print_c_string("EX_EI_%s"%name.upper(), body)
 
-    print 'const char EX_EI_{NAME}_FP[] = "{d.RSA_FINGERPRINT_NOSPACE}";'.format(
-        d=info, NAME=name.upper())
+    print('const char EX_EI_{NAME}_FP[] = "{d.RSA_FINGERPRINT_NOSPACE}";'.format(
+        d=info, NAME=name.upper()))
     print_c_string("EX_EI_%s_KEY"%name.upper(), info.RSA_IDENTITY)
 
 def analyze(s):
diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh
index 428804342a..645b3c2158 100755
--- a/scripts/codegen/run_trunnel.sh
+++ b/scripts/codegen/run_trunnel.sh
@@ -9,9 +9,7 @@ OPTIONS="--require-version=1.5.1"
 
 # Get all .trunnel files recursively from that directory so we can support
 # multiple sub-directories.
-for file in `find ./src/trunnel/ -name '*.trunnel'`; do
-  python -m trunnel ${OPTIONS} $file
-done
+find ./src/trunnel/ -name '*.trunnel' -exec python -m trunnel ${OPTIONS} {} \;
 
 python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/
 
diff --git a/scripts/git/git-install-tools.sh b/scripts/git/git-install-tools.sh
new file mode 100755
index 0000000000..ef8623a018
--- /dev/null
+++ b/scripts/git/git-install-tools.sh
@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+SCRIPTS_DIR=$(dirname "$0")
+
+TOOL_NAMES=(push-all pull-all merge-forward list-tor-branches)
+
+function usage()
+{
+    echo "$SCRIPT_NAME [-h] [-n] [-v] [-f] <all|hooks|tools|aliases>"
+    echo
+    echo "  flags:"
+    echo "    -h: show this help text"
+    echo "    -n: dry-run"
+    echo "    -v: verbose mode"
+    echo "    -f: force-install even if \$TOR_DEVTOOLS_DIR looks fishy"
+    echo
+    echo "  modes:"
+    echo "    hooks:   install git hooks in this repository."
+    echo "    tools:   install scripts in \$TOR_DEVTOOLS_DIR"
+    echo "    aliases: set up global git aliases for git tools in \$TOR_DEVTOOLS_DIR"
+    echo "    all:     all of the above."
+}
+
+INSTALL_HOOKS=0
+INSTALL_TOOLS=0
+INSTALL_ALIASES=0
+
+DRY_RUN=0
+VERBOSE=0
+FORCE=0
+
+while getopts "hnfv" opt; do
+    case "$opt" in
+        h) usage
+           exit 0
+           ;;
+        n) DRY_RUN=1
+           ;;
+        v) VERBOSE=1
+           ;;
+        f) FORCE=1
+           ;;
+        *) echo
+           usage
+           exit 1
+           ;;
+    esac
+done
+
+for item in "${@:$OPTIND}"; do
+    case "$item" in
+        hooks) INSTALL_HOOKS=1
+               ;;
+        tools) INSTALL_TOOLS=1
+               ;;
+        aliases) INSTALL_ALIASES=1
+                 ;;
+        all) INSTALL_HOOKS=1
+             INSTALL_TOOLS=1
+             INSTALL_ALIASES=1
+             ;;
+        *) echo "Unrecognized mode '$item'"
+           usage
+           exit 1
+           ;;
+    esac
+done
+
+if [[ $VERBOSE = 1 ]]; then
+    function note()
+    {
+        echo "$@"
+    }
+else
+    function note()
+    {
+        true
+    }
+fi
+
+function fail()
+{
+    echo "$@" 1>&2
+    exit 1
+}
+
+if [[ $INSTALL_HOOKS = 0 && $INSTALL_TOOLS = 0 && $INSTALL_ALIASES = 0 ]]; then
+   echo "Nothing to do. Try $SCRIPT_NAME -h for a list of commands."
+   exit 0
+fi
+
+if [[ $INSTALL_TOOLS = 1 || $INSTALL_ALIASES = 1 ]]; then
+    if [[ -z "$TOR_DEVTOOLS_DIR" ]] ; then
+        fail "\$TOR_DEVTOOLS_DIR was not set."
+    fi
+    note "Checking whether \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR) is a git repo..."
+    GITDIR=$(cd "$TOR_DEVTOOLS_DIR" && git rev-parse --git-dir 2>/dev/null)
+    note "GITDIR is $GITDIR"
+    if [[ -n "$GITDIR" ]] ; then
+        cat <<EOF
+You have asked me to install to \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR).
+That is inside a git repository, so you might not want to install there:
+depending on what you pull or push, you might find yourself giving somebody
+else write access to your scripts.  I think you should just use ~/bin or
+something.
+EOF
+
+        echo
+        if [[ "$FORCE" = 1 ]] ; then
+            echo "I will install anyway, since you said '-f'."
+        else
+            echo "I will not install. You can tell me -f if you are really sure."
+            exit 1
+        fi
+    else
+        note "It was not."
+    fi
+fi
+
+if [[ ! -d "$SCRIPTS_DIR" || ! -e "$SCRIPTS_DIR/git-push-all.sh" ]]; then
+    fail "Couldn't find scripts in '$SCRIPTS_DIR'"
+fi
+
+if [[ $DRY_RUN = 1 ]]; then
+    echo "** DRY RUN **"
+    RUN="echo >>"
+else
+    RUN=
+fi
+
+set -e
+
+# ======================================================================
+if [[ $INSTALL_HOOKS = 1 ]]; then
+       HOOKS_DIR=$(git rev-parse --git-path hooks)
+
+       note "Looking for hooks directory"
+
+       if [[ -z "$HOOKS_DIR" || ! -d "$HOOKS_DIR" ]]; then
+           fail "Couldn't find git hooks directory."
+       fi
+
+       note "Found hooks directory in $HOOKS_DIR"
+
+       note "Installing hooks"
+       for fn in "$SCRIPTS_DIR"/*.git-hook; do
+           name=$(basename "$fn")
+           $RUN install --backup "$fn" "${HOOKS_DIR}/${name%.git-hook}"
+       done
+fi
+
+
+# ======================================================================
+if [[ $INSTALL_TOOLS = 1 ]]; then
+    note "Installing tools."
+    note "Looking for \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR)"
+
+    if [[ ! -d "$TOR_DEVTOOLS_DIR" ]]; then
+        note "Creating directory"
+        $RUN mkdir -p "$TOR_DEVTOOLS_DIR"
+    fi
+
+    note "Copying scripts"
+    for tool in "${TOOL_NAMES[@]}"; do
+        $RUN install --backup "${SCRIPTS_DIR}/git-${tool}.sh" "${TOR_DEVTOOLS_DIR}/"
+    done
+fi
+
+# ======================================================================
+if [[ $INSTALL_ALIASES = 1 ]]; then
+    note "Installing aliases."
+    note "Looking for \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR)"
+
+    note "Checking for ${TOR_DEVTOOLS_DIR}/git-push-all.sh"
+    if [[ ! -x "${TOR_DEVTOOLS_DIR}/git-push-all.sh" ]]; then
+        if [[ $DRY_RUN = 0 ]]; then
+            fail "Could not find scripts in \$TOR_DEVTOOLS_DIR"
+        fi
+    fi
+
+    note "Setting aliases"
+    for tool in "${TOOL_NAMES[@]}"; do
+        $RUN git config --global "alias.$tool" \!"${TOR_DEVTOOLS_DIR}/git-${tool}.sh"
+    done
+
+fi
+
+note Done.
diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh
new file mode 100755
index 0000000000..d6b30f064f
--- /dev/null
+++ b/scripts/git/git-list-tor-branches.sh
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+
+# Script to be used by other git scripts, and provide a single place
+# that lists our supported branches.  To change which branches are
+# supported, look at the end of the file that says 'edit here'.
+
+SCRIPT_NAME=$(basename "$0")
+
+function usage()
+{
+    echo "$SCRIPT_NAME [-h] [-l|-s|-b|-m] [-R]"
+    echo
+    echo "  arguments:"
+    echo "   -h: show this help text"
+    echo
+    echo "   -l: list the active tor branches (default)"
+    echo "   -s: list the suffixes to be used with the active tor branches"
+    echo "   -b: write bash code setting WORKTREE to an array of ( branch path ) arrays"
+    echo "   -m: write bash code setting WORKTREE to an array of"
+    echo "       ( branch parent path suffix parent_suffix ) arrays"
+    echo
+    echo "   -R: omit release branches."
+}
+
+# list : just a list of branch names.
+# branch_path : For git-setup-dirs.sh and git-pull-all.sh
+# suffix: write a list of suffixes.
+# merge: branch, upstream, path, suffix, upstream suffix.
+mode="list"
+skip_release_branches="no"
+
+while getopts "hblmsR" opt ; do
+    case "$opt" in
+        h) usage
+           exit 0
+           ;;
+        b) mode="branch_path"
+           ;;
+        l) mode="list"
+           ;;
+        s) mode="suffix"
+           ;;
+        m) mode="merge"
+           ;;
+        R) skip_release_branches="yes"
+           ;;
+        *) echo "Unknown option"
+           exit 1
+           ;;
+    esac
+done
+
+all_branch_vars=()
+
+prev_maint_branch=""
+prev_maint_suffix=""
+
+branch() {
+    # The name of the branch. (Supplied by caller)  Ex: maint-0.4.3
+    brname="$1"
+
+    # The name of the branch with no dots. Ex: maint-043
+    brname_nodots="${brname//./}"
+    # The name of the branch with no dots, and _ instead of -. Ex: maint_043
+    brname_nodots_uscore="${brname_nodots//-/_}"
+    # Name to use for a variable to represent the branch. Ex: MAINT_043
+    varname="${brname_nodots_uscore^^}"
+
+    is_maint="no"
+
+    # suffix: a suffix to place at the end of branches we generate with respect
+    # to this branch.  Ex: _043
+
+    # location: where the branch can be found.
+
+    if [[ "$brname" == "master" ]]; then
+        suffix="_master"
+        location="\$GIT_PATH/\$TOR_MASTER_NAME"
+    elif [[ "$brname" =~ ^maint- ]]; then
+        suffix="_${brname_nodots#maint-}"
+        location="\$GIT_PATH/\$TOR_WKT_NAME/$brname"
+        is_maint="yes"
+    elif [[ "$brname" =~ ^release- ]]; then
+        suffix="_r${brname_nodots#release-}"
+        location="\$GIT_PATH/\$TOR_WKT_NAME/$brname"
+
+        if [[ "$skip_release_branches" = "yes" ]]; then
+            return
+        fi
+    else
+        echo "Unrecognized branch type '${brname}'" >&2
+        exit 1
+    fi
+
+    all_branch_vars+=("$varname")
+
+    # Now emit the per-branch information
+    if [[ "$mode" == "branch_path" ]]; then
+        echo "${varname}=( \"$brname\" \"$location\" )"
+    elif [[ "$mode" == "merge" ]]; then
+        echo "${varname}=( \"$brname\" \"$prev_maint_branch\" \"$location\" \"$suffix\" \"$prev_maint_suffix\" )"
+    elif [[ "$mode" == "list" ]]; then
+        echo "$brname"
+    elif [[ "$mode" == "suffix" ]]; then
+        echo "$suffix"
+    else
+        echo "unknown mode $mode" >&2
+        exit 1
+    fi
+
+    if [[ "$is_maint" == "yes" ]]; then
+        prev_maint_branch="$brname"
+        prev_maint_suffix="$suffix"
+    fi
+}
+
+finish() {
+    if [[ "$mode" == branch_path ]] || [[ "$mode" == merge ]]; then
+        echo "WORKTREE=("
+        for v in "${all_branch_vars[@]}"; do
+            echo "  ${v}[@]"
+        done
+        echo ")"
+    elif [[ "$mode" == list ]] || [[ "$mode" == suffix ]]; then
+        # nothing to do
+        :
+    else
+        echo "unknown mode $mode" >&2
+        exit 1
+    fi
+}
+
+# ==============================
+# EDIT HERE
+# ==============================
+# List of all branches.  These must be in order, from oldest to newest, with
+# maint before release.
+
+branch maint-0.3.5
+branch release-0.3.5
+
+branch maint-0.4.1
+branch release-0.4.1
+
+branch maint-0.4.2
+branch release-0.4.2
+
+branch maint-0.4.3
+branch release-0.4.3
+
+branch master
+
+finish
diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh
new file mode 100755
index 0000000000..7c72f8478d
--- /dev/null
+++ b/scripts/git/git-merge-forward.sh
@@ -0,0 +1,373 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+
+function usage()
+{
+  echo "$SCRIPT_NAME [-h] [-n] [-t <test-branch-prefix> [-u]]"
+  echo
+  echo "  arguments:"
+  echo "   -h: show this help text"
+  echo "   -n: dry run mode"
+  echo "       (default: run commands)"
+  echo "   -t: test branch mode: create new branches from the commits checked"
+  echo "       out in each maint directory. Call these branches prefix_035,"
+  echo "       prefix_040, ... , prefix_master."
+  echo "       (default: merge forward maint-*, release-*, and master)"
+  echo "   -u: in test branch mode, if a prefix_* branch already exists,"
+  echo "       skip creating that branch. Use after a merge error, to"
+  echo "       restart the merge forward at the first unmerged branch."
+  echo "       (default: if a prefix_* branch already exists, fail and exit)"
+  echo
+  echo " env vars:"
+  echo "   required:"
+  echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
+  echo "       You must set this env var, we recommend \$HOME/git/"
+  echo "       (default: fail if this env var is not set;"
+  echo "       current: $GIT_PATH)"
+  echo
+  echo "   optional:"
+  echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
+  echo "       The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
+  echo "       (default: tor; current: $TOR_MASTER_NAME)"
+  echo "   TOR_WKT_NAME: the name of the directory containing the tor"
+  echo "       worktrees. The tor worktrees are:"
+  echo "       \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
+  echo "       (default: tor-wkt; current: $TOR_WKT_NAME)"
+  echo "   we recommend that you set these env vars in your ~/.profile"
+}
+
+#################
+# Configuration #
+#################
+
+# Don't change this configuration - set the env vars in your .profile
+
+# Where are all those git repositories?
+GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
+# The tor master git repository directory from which all the worktree have
+# been created.
+TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
+# The worktrees location (directory).
+TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
+
+##########################
+# Git branches to manage #
+##########################
+
+# The branches and worktrees need to be modified when there is a new branch,
+# and when an old branch is no longer supported.
+
+# Configuration of the branches that needs merging. The values are in order:
+#   (0) current maint/release branch name
+#   (1) previous maint/release name to merge into (0)
+#         (only used in merge forward mode)
+#   (2) Full path of the git worktree
+#   (3) current branch suffix
+#         (maint branches only, only used in test branch mode)
+#   (4) previous test branch suffix to merge into (3)
+#         (maint branches only, only used in test branch mode)
+#
+# Merge forward example:
+#   $ cd <PATH/TO/WORKTREE> (2)
+#   $ git checkout maint-0.3.5 (0)
+#   $ git pull
+#   $ git merge maint-0.3.4 (1)
+#
+# Test branch example:
+#   $ cd <PATH/TO/WORKTREE> (2)
+#   $ git checkout -b ticket99999_035 (3)
+#   $ git checkout maint-0.3.5 (0)
+#   $ git pull
+#   $ git checkout ticket99999_035
+#   $ git merge maint-0.3.5
+#   $ git merge ticket99999_034 (4)
+#
+# First set of arrays are the maint-* branch and then the release-* branch.
+# New arrays need to be in the WORKTREE= array else they aren't considered.
+#
+# Only used in test branch mode
+# We create a test branch for the earliest maint branch.
+# But it's the earliest maint branch, so we don't merge forward into it.
+# Since we don't merge forward into it, the second and fifth items must be
+# blank ("").
+
+# origin that will be used to fetch the updates. All the worktrees are created
+# from that repository.
+ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
+
+#######################
+# Argument processing #
+#######################
+
+# Controlled by the -n option. The dry run option will just output the command
+# that would have been executed for each worktree.
+DRY_RUN=0
+
+# Controlled by the -t <test-branch-prefix> option. The test branch base
+# name option makes git-merge-forward.sh create new test branches:
+# <tbbn>_035, <tbbn>_040, ... , <tbbn>_master, and merge forward.
+TEST_BRANCH_PREFIX=
+
+# Controlled by the -u option. The use existing option checks for existing
+# branches with the <test-branch-prefix>, and checks them out, rather than
+# creating a new branch.
+USE_EXISTING=0
+
+while getopts "hnt:u" opt; do
+  case "$opt" in
+    h) usage
+       exit 0
+       ;;
+    n) DRY_RUN=1
+       echo "    *** DRY RUN MODE ***"
+       ;;
+    t) TEST_BRANCH_PREFIX="$OPTARG"
+       echo "    *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***"
+       ;;
+    u) USE_EXISTING=1
+       echo "    *** USE EXISTING TEST BRANCHES MODE ***"
+       ;;
+    *)
+       echo
+       usage
+       exit 1
+       ;;
+  esac
+done
+
+###########################
+# Git worktrees to manage #
+###########################
+
+set -e
+if [ -z "$TEST_BRANCH_PREFIX" ]; then
+  # maint/release merge mode
+  eval "$(git-list-tor-branches.sh -m)"
+  # Remove first element: we don't merge forward into it.
+  WORKTREE=( "${WORKTREE[@]:1}" )
+else
+  eval "$(git-list-tor-branches.sh -m -R)"
+fi
+set +e
+
+COUNT=${#WORKTREE[@]}
+
+#############
+# Constants #
+#############
+
+# Control characters
+CNRM=$'\x1b[0;0m'   # Clear color
+
+# Bright color
+BGRN=$'\x1b[1;32m'
+BBLU=$'\x1b[1;34m'
+BRED=$'\x1b[1;31m'
+BYEL=$'\x1b[1;33m'
+IWTH=$'\x1b[3;37m'
+
+# Strings for the pretty print.
+MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
+SUCCESS="${BGRN}success${CNRM}"
+FAILED="${BRED}failed${CNRM}"
+
+####################
+# Helper functions #
+####################
+
+# Validate the given returned value (error code), print success or failed. The
+# second argument is the error output in case of failure, it is printed out.
+# On failure, this function exits.
+function validate_ret
+{
+  if [ "$1" -eq 0 ]; then
+    printf "%s\\n" "$SUCCESS"
+  else
+    printf "%s\\n" "$FAILED"
+    printf "    %s" "$2"
+    exit 1
+  fi
+}
+
+# Switch to the given branch name.
+function switch_branch
+{
+  local cmd="git checkout '$1'"
+  printf "  %s Switching branch to %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Checkout a new branch with the given branch name.
+function new_branch
+{
+  local cmd="git checkout -b '$1'"
+  printf "  %s Creating new branch %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Switch to an existing branch, or checkout a new branch with the given
+# branch name.
+function switch_or_new_branch
+{
+  local cmd="git rev-parse --verify '$1'"
+  if [ $DRY_RUN -eq 0 ]; then
+    # Call switch_branch if there is a branch, or new_branch if there is not
+    msg=$( eval "$cmd" 2>&1 )
+    RET=$?
+    if [ $RET -eq 0 ]; then
+      # Branch: (commit id)
+      switch_branch "$1"
+    elif [ $RET -eq 128 ]; then
+      # Not a branch: "fatal: Needed a single revision"
+      new_branch "$1"
+    else
+      # Unexpected return value
+      validate_ret $RET "$msg"
+    fi
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:"
+    switch_branch "$1"
+    new_branch "$1"
+  fi
+}
+
+# Pull the given branch name.
+function pull_branch
+{
+  local cmd="git pull"
+  printf "  %s Pulling branch %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Merge the given branch name ($1) into the current branch ($2).
+function merge_branch
+{
+  local cmd="git merge --no-edit '$1'"
+  printf "  %s Merging branch %s into %s..." "$MARKER" "$1" "$2"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Merge origin/(branch name) into the current branch.
+function merge_branch_origin
+{
+  local cmd="git merge --ff-only 'origin/$1'"
+  printf "  %s Merging branch origin/%s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Go into the worktree repository.
+function goto_repo
+{
+  if [ ! -d "$1" ]; then
+    echo "  $1: Not found. Stopping."
+    exit 1
+  fi
+  cd "$1" || exit
+}
+
+# Fetch the origin. No arguments.
+function fetch_origin
+{
+  local cmd="git fetch origin"
+  printf "  %s Fetching origin..." "$MARKER"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+###############
+# Entry point #
+###############
+
+# First, fetch the origin.
+goto_repo "$ORIGIN_PATH"
+fetch_origin
+
+# Go over all configured worktree.
+for ((i=0; i<COUNT; i++)); do
+  current=${!WORKTREE[$i]:0:1}
+  previous=${!WORKTREE[$i]:1:1}
+  repo_path=${!WORKTREE[$i]:2:1}
+  # default to merge forward mode
+  test_current=
+  test_previous=
+  target_current="$current"
+  target_previous="$previous"
+  if [ "$TEST_BRANCH_PREFIX" ]; then
+    test_current_suffix=${!WORKTREE[$i]:3:1}
+    test_current=${TEST_BRANCH_PREFIX}${test_current_suffix}
+    # the current test branch, if present, or maint/release branch, if not
+    target_current="$test_current"
+    test_previous_suffix=${!WORKTREE[$i]:4:1}
+    if [ "$test_previous_suffix" ]; then
+      test_previous=${TEST_BRANCH_PREFIX}${test_previous_suffix}
+      # the previous test branch, if present, or maint/release branch, if not
+      target_previous="$test_previous"
+    fi
+  fi
+
+  printf "%s Handling branch \\n" "$MARKER" "${BYEL}$target_current${CNRM}"
+
+  # Go into the worktree to start merging.
+  goto_repo "$repo_path"
+  if [ "$test_current" ]; then
+    if [ $USE_EXISTING -eq 0 ]; then
+      # Create a test branch from the currently checked-out branch/commit
+      # Fail if it already exists
+      new_branch "$test_current"
+    else
+      # Switch if it exists, or create if it does not
+      switch_or_new_branch "$test_current"
+    fi
+  fi
+  # Checkout the current maint/release branch
+  switch_branch "$current"
+  # Update the current maint/release branch with an origin merge to get the
+  # latest updates
+  merge_branch_origin "$current"
+  if [ "$test_current" ]; then
+    # Checkout the test branch
+    switch_branch "$test_current"
+    # Merge the updated maint branch into the test branch
+    merge_branch "$current" "$test_current"
+  fi
+  # Merge the previous branch into the target branch
+  # Merge Forward Example:
+  #   merge maint-0.3.5 into maint-0.4.0.
+  # Test Branch Example:
+  #   merge bug99999_035 into bug99999_040.
+  # Skip the merge if the previous branch does not exist
+  # (there's nothing to merge forward into the oldest test branch)
+  if [ "$target_previous" ]; then
+    merge_branch "$target_previous" "$target_current"
+  fi
+done
diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh
new file mode 100755
index 0000000000..7f82eda296
--- /dev/null
+++ b/scripts/git/git-pull-all.sh
@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+
+usage()
+{
+  echo "$SCRIPT_NAME [-h] [-n]"
+  echo
+  echo "  arguments:"
+  echo "   -h: show this help text"
+  echo "   -n: dry run mode"
+  echo "       (default: run commands)"
+  echo
+  echo " env vars:"
+  echo "   required:"
+  echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
+  echo "       You must set this env var, we recommend \$HOME/git/"
+  echo "       (default: fail if this env var is not set;"
+  echo "       current: $GIT_PATH)"
+  echo
+  echo "   optional:"
+  echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
+  echo "       The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
+  echo "       (default: tor; current: $TOR_MASTER_NAME)"
+  echo "   TOR_WKT_NAME: the name of the directory containing the tor"
+  echo "       worktrees. The tor worktrees are:"
+  echo "       \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
+  echo "       (default: tor-wkt; current: $TOR_WKT_NAME)"
+  echo "   we recommend that you set these env vars in your ~/.profile"
+}
+
+#################
+# Configuration #
+#################
+
+# Don't change this configuration - set the env vars in your .profile
+
+# Where are all those git repositories?
+GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
+# The tor master git repository directory from which all the worktree have
+# been created.
+TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
+# The worktrees location (directory).
+TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
+
+##########################
+# Git branches to manage #
+##########################
+
+set -e
+eval "$(git-list-tor-branches.sh -b)"
+set +e
+
+# The master branch path has to be the main repository thus contains the
+# origin that will be used to fetch the updates. All the worktrees are created
+# from that repository.
+ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
+
+COUNT=${#WORKTREE[@]}
+
+#######################
+# Argument processing #
+#######################
+
+# Controlled by the -n option. The dry run option will just output the command
+# that would have been executed for each worktree.
+DRY_RUN=0
+
+while getopts "hn" opt; do
+  case "$opt" in
+    h) usage
+       exit 0
+       ;;
+    n) DRY_RUN=1
+       echo "    *** DRY DRUN MODE ***"
+       ;;
+    *)
+       echo
+       usage
+       exit 1
+       ;;
+  esac
+done
+
+#############
+# Constants #
+#############
+
+# Control characters
+CNRM=$'\x1b[0;0m'   # Clear color
+
+# Bright color
+BGRN=$'\x1b[1;32m'
+BBLU=$'\x1b[1;34m'
+BRED=$'\x1b[1;31m'
+BYEL=$'\x1b[1;33m'
+IWTH=$'\x1b[3;37m'
+
+# Strings for the pretty print.
+MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
+SUCCESS="${BGRN}ok${CNRM}"
+FAILED="${BRED}failed${CNRM}"
+
+####################
+# Helper functions #
+####################
+
+# Validate the given returned value (error code), print success or failed. The
+# second argument is the error output in case of failure, it is printed out.
+# On failure, this function exits.
+function validate_ret
+{
+  if [ "$1" -eq 0 ]; then
+    printf "%s\\n" "$SUCCESS"
+  else
+    printf "%s\\n" "$FAILED"
+    printf "    %s" "$2"
+    exit 1
+  fi
+}
+
+# Switch to the given branch name.
+function switch_branch
+{
+  local cmd="git checkout $1"
+  printf "  %s Switching branch to %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Pull the given branch name.
+function merge_branch
+{
+  local cmd="git merge --ff-only origin/$1"
+  printf "  %s Merging branch origin/%s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Go into the worktree repository.
+function goto_repo
+{
+  if [ ! -d "$1" ]; then
+    echo "  $1: Not found. Stopping."
+    exit 1
+  fi
+  cd "$1" || exit
+}
+
+# Fetch the origin. No arguments.
+function fetch_origin
+{
+  local cmd="git fetch origin"
+  printf "  %s Fetching origin..." "$MARKER"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Fetch tor-github pull requests. No arguments.
+function fetch_tor_github
+{
+  local cmd="git fetch tor-github"
+  printf "  %s Fetching tor-github..." "$MARKER"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+###############
+# Entry point #
+###############
+
+# First, fetch tor-github.
+goto_repo "$ORIGIN_PATH"
+fetch_tor_github
+
+# Then, fetch the origin.
+fetch_origin
+
+# Go over all configured worktree.
+for ((i=0; i<COUNT; i++)); do
+  current=${!WORKTREE[$i]:0:1}
+  repo_path=${!WORKTREE[$i]:1:1}
+
+  printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$current${CNRM}"
+
+  # Go into the worktree to start merging.
+  goto_repo "$repo_path"
+  # Checkout the current branch
+  switch_branch "$current"
+  # Update the current branch by merging the origin to get the latest.
+  merge_branch "$current"
+done
diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh
new file mode 100755
index 0000000000..558ea8d01c
--- /dev/null
+++ b/scripts/git/git-push-all.sh
@@ -0,0 +1,309 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+
+function usage()
+{
+  if [ "$TOR_PUSH_SAME" ]; then
+    CURRENT_PUSH_SAME="push"
+  else
+    CURRENT_PUSH_SAME="skip"
+  fi
+
+  echo "$SCRIPT_NAME [-h] [-r <remote-name> [-t <test-branch-prefix>]] [-s]"
+  # The next line looks misaligned, but it lines up in the output
+  echo "                [-- [-n] [--no-atomic] <git push options>]"
+  echo
+  echo "  arguments:"
+  echo "   -h: show this help text"
+  echo "   -n: dry run mode"
+  echo "       (default: run commands)"
+  echo "   -r: push to remote-name, rather than the default upstream remote."
+  echo "       (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)"
+  echo "   -t: test branch mode: push test branches to remote-name. Pushes"
+  echo "       branches prefix_035, prefix_040,  ... , prefix_master."
+  echo "       (default: push maint-*, release-*, and master)"
+  echo "   -s: push branches whose tips match upstream maint, release, or"
+  echo "       master branches. The default is to skip these branches,"
+  echo "       because they do not contain any new code. Use -s to test for"
+  echo "       CI environment failures, using code that previously passed CI."
+  echo "       (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
+  echo "   --: pass further arguments to git push."
+  echo "       All unrecognised arguments are passed to git push, but complex"
+  echo "       arguments before -- may be mangled by getopt."
+  echo "       (default: git push --atomic, current: $GIT_PUSH)"
+  echo
+  echo " env vars:"
+  echo "   optional:"
+  echo "   TOR_GIT_PUSH_PATH: change to this directory before pushing."
+  echo "       (default: if \$TOR_FULL_GIT_PATH is set,"
+  echo "       use \$TOR_FULL_GIT_PATH/\$TOR_MASTER;"
+  echo "       Otherwise, use the current directory for pushes;"
+  echo "       current: $TOR_GIT_PUSH_PATH)"
+  echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
+  echo "       We recommend using \$HOME/git/."
+  echo "       (default: use the current directory for pushes;"
+  echo "       current: $TOR_FULL_GIT_PATH)"
+  echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
+  echo "       The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
+  echo "       (default: tor; current: $TOR_MASTER_NAME)"
+  echo
+  echo "   TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
+  echo "       Overridden by -r."
+  echo "       (default: upstream; current: $UPSTREAM_REMOTE)"
+  echo "   TOR_GIT_PUSH: the git push command and default arguments."
+  echo "       Overridden by <git push options> after --."
+  echo "       (default: git push --atomic; current: $GIT_PUSH)"
+  echo "   TOR_PUSH_SAME: push branches whose tips match upstream maint,"
+  echo "       release, or master branches. Inverted by -s."
+  echo "       (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
+  echo "   TOR_PUSH_DELAY: pushes the master and maint branches separately,"
+  echo "       so that CI runs in a sensible order."
+  echo "       (default: push all branches immediately; current: $PUSH_DELAY)"
+  echo "   we recommend that you set these env vars in your ~/.profile"
+}
+
+set -e
+
+#################
+# Configuration #
+#################
+
+# Don't change this configuration - set the env vars in your .profile
+#
+# The tor master git repository directory from which all the worktree have
+# been created.
+TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
+# Which directory do we push from?
+if [ "$TOR_FULL_GIT_PATH" ]; then
+  TOR_GIT_PUSH_PATH=${TOR_GIT_PUSH_PATH:-"$TOR_FULL_GIT_PATH/$TOR_MASTER_NAME"}
+fi
+# git push command and default arguments
+GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"}
+# The upstream remote which git.torproject.org/tor.git points to.
+DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
+# Push to a different upstream remote using -r <remote-name>
+UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE}
+# Add a delay between pushes, so CI runs on the most important branches first
+PUSH_DELAY=${TOR_PUSH_DELAY:-0}
+# Push (1) or skip (0) test branches that are the same as an upstream
+# maint/master branch. Push if you are testing that the CI environment still
+# works on old code, skip if you are testing new code in the branch.
+# Default: skip unchanged branches.
+# Inverted by the -s option.
+PUSH_SAME=${TOR_PUSH_SAME:-0}
+
+#######################
+# Argument processing #
+#######################
+
+# Controlled by the -t <test-branch-prefix> option. The test branch prefix
+# option makes git-merge-forward.sh create new test branches:
+# <tbp>_035, <tbp>_040, ... , <tbp>_master, and merge each branch forward into
+# the next one.
+TEST_BRANCH_PREFIX=
+
+while getopts ":hr:st:" opt; do
+  case "$opt" in
+    h) usage
+       exit 0
+       ;;
+    r) UPSTREAM_REMOTE="$OPTARG"
+       echo "    *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***"
+       shift
+       shift
+       OPTIND=$((OPTIND - 2))
+       ;;
+    s) PUSH_SAME=$((! PUSH_SAME))
+       if [ "$PUSH_SAME" -eq 0 ]; then
+         echo "    *** SKIPPING UNCHANGED TEST BRANCHES ***"
+       else
+         echo "    *** PUSHING UNCHANGED TEST BRANCHES ***"
+       fi
+       shift
+       OPTIND=$((OPTIND - 1))
+       ;;
+    t) TEST_BRANCH_PREFIX="$OPTARG"
+       echo "    *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***"
+       shift
+       shift
+       OPTIND=$((OPTIND - 2))
+       ;;
+    *)
+       # Make git push handle the option
+       # This might mangle options with spaces, use -- for complex options
+       GIT_PUSH="$GIT_PUSH $1"
+       shift
+       OPTIND=$((OPTIND - 1))
+       ;;
+  esac
+done
+
+# getopts doesn't allow "-" as an option character,
+# so we have to handle -- manually
+if [ "$1" = "--" ]; then
+  shift
+fi
+
+if [ "$TEST_BRANCH_PREFIX" ]; then
+  if [ "$UPSTREAM_REMOTE" = "$DEFAULT_UPSTREAM_REMOTE" ]; then
+    echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \
+      "the default remote $DEFAULT_UPSTREAM_REMOTE is not allowed."
+    echo
+    usage
+    exit 1
+  fi
+fi
+
+if [ "$TOR_GIT_PUSH_PATH" ]; then
+  echo "Changing to $TOR_GIT_PUSH_PATH before pushing"
+  cd "$TOR_GIT_PUSH_PATH"
+else
+  echo "Pushing from the current directory"
+fi
+
+echo "Calling $GIT_PUSH" "$@" "<branches>"
+
+################################
+# Git upstream remote branches #
+################################
+
+set -e
+DEFAULT_UPSTREAM_BRANCHES=
+if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then
+    for br in $(git-list-tor-branches.sh -l); do
+        DEFAULT_UPSTREAM_BRANCHES="${DEFAULT_UPSTREAM_BRANCHES} ${DEFAULT_UPSTREAM_REMOTE}/${br}"
+    done
+fi
+
+UPSTREAM_BRANCHES=
+for br in $(git-list-tor-branches.sh -l); do
+    UPSTREAM_BRANCHES="${UPSTREAM_BRANCHES} ${UPSTREAM_REMOTE}/${br}"
+done
+
+########################
+# Git branches to push #
+########################
+
+if [ -z "$TEST_BRANCH_PREFIX" ]; then
+
+  # maint/release push mode: push all branches.
+  #
+  # List of branches to push. Ordering is not important.
+  PUSH_BRANCHES="$(git-list-tor-branches.sh -l)"
+else
+
+  # Test branch push mode: push test branches, based on each maint branch.
+  #
+  # List of branches to push. Ordering is not important.
+  PUSH_BRANCHES=""
+  for suffix in $(git-list-tor-branches.sh -s -R); do
+      PUSH_BRANCHES="${PUSH_BRANCHES} ${TEST_BRANCH_PREFIX}${suffix}"
+  done
+fi
+
+set +e
+
+###############
+# Entry point #
+###############
+
+if [ "$TEST_BRANCH_PREFIX" ]; then
+  # Skip the test branches that are the same as the default or current
+  # upstream branches (they have already been tested)
+  UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES"
+else
+  # Skip the local maint-*, release-*, master branches that are the same as the
+  # current upstream branches, but ignore the default upstream
+  # (we want to update a non-default remote, even if it matches the default)
+  UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES"
+fi
+
+# Skip branches that match the relevant upstream(s)
+if [ "$PUSH_SAME" -eq 0 ]; then
+  NEW_PUSH_BRANCHES=
+  for b in $PUSH_BRANCHES; do
+    PUSH_COMMIT=$(git rev-parse "$b")
+    SKIP_UPSTREAM=
+    for u in $UPSTREAM_SKIP_SAME_AS; do
+      # Skip the branch check on error
+      UPSTREAM_COMMIT=$(git rev-parse "$u" 2>/dev/null) || continue
+      if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then
+        SKIP_UPSTREAM="$u"
+      fi
+    done
+    if [ "$SKIP_UPSTREAM" ]; then
+      printf "Skipping unchanged: %s matching remote: %s\\n" \
+        "$b" "$SKIP_UPSTREAM"
+    else
+      if [ "$NEW_PUSH_BRANCHES" ]; then
+        NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}"
+      else
+        NEW_PUSH_BRANCHES="${b}"
+      fi
+    fi
+  done
+  PUSH_BRANCHES=${NEW_PUSH_BRANCHES}
+fi
+
+if [ ! "$PUSH_BRANCHES" ]; then
+  echo "No branches to push!"
+  # We expect the rest of the script to run without errors, even if there
+  # are no branches
+fi
+
+if [ "$PUSH_DELAY" -le 0 ]; then
+  echo "Pushing $PUSH_BRANCHES"
+  # We know that there are no spaces in any branch within $PUSH_BRANCHES, so
+  # it is safe to use it unquoted.  (This also applies to the other shellcheck
+  # exceptions below.)
+  #
+  # Push all the branches at the same time
+  # shellcheck disable=SC2086
+  $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES
+else
+  # Push the branches in optimal CI order, with a delay between each push
+  PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V)
+  MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \
+      || true # Skipped master branch
+  if [ -z "$TEST_BRANCH_PREFIX" ]; then
+    MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \
+        || true # Skipped all maint branches
+    RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \
+      tr "\\n" " ") || true # Skipped all release branches
+  else
+    # Actually test branches based on maint branches
+    MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \
+        || true # Skipped all maint test branches
+    # No release branches
+    RELEASE_BRANCHES=
+  fi
+  if [ "$MASTER_BRANCH" ] || [ "$MAINT_BRANCHES" ] \
+      || [ "$RELEASE_BRANCHES" ]; then
+    printf "Pushing with %ss delays, so CI runs in this order:\\n" \
+           "$PUSH_DELAY"
+    if [ "$MASTER_BRANCH" ]; then
+      printf "%s\\n" "$MASTER_BRANCH"
+    fi
+    if [ "$MAINT_BRANCHES" ]; then
+      printf "%s\\n" "$MAINT_BRANCHES"
+    fi
+    if [ "$RELEASE_BRANCHES" ]; then
+      printf "%s\\n" "$RELEASE_BRANCHES"
+    fi
+  fi
+  # shellcheck disable=SC2086
+  for b in $MASTER_BRANCH $MAINT_BRANCHES; do
+    $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b"
+    # If we are pushing more than one branch, delay.
+    # In the unlikely scenario where we are pushing maint without master,
+    # or maint without release, there may be an extra delay
+    if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then
+      sleep "$PUSH_DELAY"
+    fi
+  done
+  if [ "$RELEASE_BRANCHES" ]; then
+    # shellcheck disable=SC2086
+    $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES
+  fi
+fi
diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh
new file mode 100755
index 0000000000..1f61eb8b83
--- /dev/null
+++ b/scripts/git/git-setup-dirs.sh
@@ -0,0 +1,508 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+
+function usage()
+{
+  echo "$SCRIPT_NAME [-h] [-n] [-u]"
+  echo
+  echo "  arguments:"
+  echo "   -h: show this help text"
+  echo "   -n: dry run mode"
+  echo "       (default: run commands)"
+  echo "   -u: if a directory or worktree already exists, use it"
+  echo "       (default: fail and exit on existing directories)"
+  echo
+  echo " env vars:"
+  echo "   required:"
+  echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
+  echo "       You must set this env var, we recommend \$HOME/git/"
+  echo "       (default: fail if this env var is not set;"
+  echo "       current: $GIT_PATH)"
+  echo
+  echo "   optional:"
+  echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
+  echo "       The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
+  echo "       (default: tor; current: $TOR_MASTER_NAME)"
+  echo "   TOR_WKT_NAME: the name of the directory containing the tor"
+  echo "       worktrees. The tor worktrees are:"
+  echo "       \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
+  echo "       (default: tor-wkt; current: $TOR_WKT_NAME)"
+  echo "   TOR_GIT_ORIGIN_PULL: the origin remote pull URL."
+  echo "       (current: $GIT_ORIGIN_PULL)"
+  echo "   TOR_GIT_ORIGIN_PUSH: the origin remote push URL"
+  echo "       (current: $GIT_ORIGIN_PUSH)"
+  echo "   TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
+  echo "       If \$TOR_UPSTREAM_REMOTE_NAME is not 'origin', we have a"
+  echo "       separate upstream remote, and we don't push to origin."
+  echo "       (default: $DEFAULT_UPSTREAM_REMOTE)"
+  echo "   TOR_GITHUB_PULL: the tor-github remote pull URL"
+  echo "       (current: $GITHUB_PULL)"
+  echo "   TOR_GITHUB_PUSH: the tor-github remote push URL"
+  echo "       (current: $GITHUB_PUSH)"
+  echo "   TOR_EXTRA_CLONE_ARGS: extra arguments to git clone"
+  echo "       (current: $TOR_EXTRA_CLONE_ARGS)"
+  echo "   TOR_EXTRA_REMOTE_NAME: the name of an extra remote"
+  echo "       This remote is not pulled by this script or git-pull-all.sh."
+  echo "       This remote is not pushed by git-push-all.sh."
+  echo "       (current: $TOR_EXTRA_REMOTE_NAME)"
+  echo "   TOR_EXTRA_REMOTE_PULL: the extra remote pull URL."
+  echo "       (current: $TOR_EXTRA_REMOTE_PULL)"
+  echo "   TOR_EXTRA_REMOTE_PUSH: the extra remote push URL"
+  echo "       (current: $TOR_EXTRA_REMOTE_PUSH)"
+  echo "   we recommend that you set these env vars in your ~/.profile"
+}
+
+#################
+# Configuration #
+#################
+
+# Don't change this configuration - set the env vars in your .profile
+
+# Where are all those git repositories?
+GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
+# The tor master git repository directory from which all the worktree have
+# been created.
+TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
+# The worktrees location (directory).
+TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
+
+# Origin repositories
+GIT_ORIGIN_PULL=${TOR_GIT_ORIGIN_PULL:-"https://git.torproject.org/tor.git"}
+GIT_ORIGIN_PUSH=${TOR_GIT_ORIGIN_PUSH:-"git@git-rw.torproject.org:tor.git"}
+# The upstream remote which git.torproject.org/tor.git points to.
+DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
+# Copy the URLs from origin
+GIT_UPSTREAM_PULL="$GIT_ORIGIN_PULL"
+GIT_UPSTREAM_PUSH="$GIT_ORIGIN_PUSH"
+# And avoid pushing to origin if we have an upstream
+if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
+  GIT_ORIGIN_PUSH="No pushes to origin, if there is an upstream"
+fi
+# GitHub repositories
+GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"}
+GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"}
+
+##########################
+# Git branches to manage #
+##########################
+
+# The branches and worktrees need to be modified when there is a new branch,
+# and when an old branch is no longer supported.
+
+set -e
+eval "$(git-list-tor-branches.sh -b)"
+set +e
+
+# The master branch path has to be the main repository thus contains the
+# origin that will be used to fetch the updates. All the worktrees are created
+# from that repository.
+ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
+
+#######################
+# Argument processing #
+#######################
+
+# Controlled by the -n option. The dry run option will just output the command
+# that would have been executed for each worktree.
+DRY_RUN=0
+
+# Controlled by the -s option. The use existing option checks for existing
+# directories, and re-uses them, rather than creating a new directory.
+USE_EXISTING=0
+USE_EXISTING_HINT="Use existing: '$SCRIPT_NAME -u'."
+
+while getopts "hnu" opt; do
+  case "$opt" in
+    h) usage
+       exit 0
+       ;;
+    n) DRY_RUN=1
+       echo "    *** DRY RUN MODE ***"
+       ;;
+    u) USE_EXISTING=1
+       echo "    *** USE EXISTING DIRECTORIES MODE ***"
+       ;;
+    *)
+       echo
+       usage
+       exit 1
+       ;;
+  esac
+done
+
+###########################
+# Git worktrees to manage #
+###########################
+
+COUNT=${#WORKTREE[@]}
+
+#############
+# Constants #
+#############
+
+# Control characters
+CNRM=$'\x1b[0;0m'   # Clear color
+
+# Bright color
+BGRN=$'\x1b[1;32m'
+BBLU=$'\x1b[1;34m'
+BRED=$'\x1b[1;31m'
+BYEL=$'\x1b[1;33m'
+IWTH=$'\x1b[3;37m'
+
+# Strings for the pretty print.
+MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
+SUCCESS="${BGRN}success${CNRM}"
+SKIPPED="${BYEL}skipped${CNRM}"
+FAILED="${BRED}failed${CNRM}"
+
+####################
+# Helper functions #
+####################
+
+# Validate the given returned value (error code), print success or failed. The
+# second argument is the error output in case of failure, it is printed out.
+# On failure, this function exits.
+function validate_ret
+{
+  if [ "$1" -eq 0 ]; then
+    printf "%s\\n" "$SUCCESS"
+  else
+    printf "%s\\n" "$FAILED"
+    printf "      %s\\n" "$2"
+    exit 1
+  fi
+}
+
+# Validate the given returned value (error code), print success, skipped, or
+# failed. If $USE_EXISTING is 0, fail on error, otherwise, skip on error.
+# The second argument is the error output in case of failure, it is printed
+# out. On failure, this function exits.
+function validate_ret_skip
+{
+  if [ "$1" -ne 0 ]; then
+    if [ "$USE_EXISTING" -eq "0" ]; then
+      # Fail and exit with error
+      validate_ret "$1" "$2 $USE_EXISTING_HINT"
+    else
+      printf "%s\\n" "$SKIPPED"
+      printf "      %s\\n" "${IWTH}$2${CNRM}"
+      # Tell the caller to skip the rest of the function
+      return 0
+    fi
+  fi
+  # Tell the caller to continue
+  return 1
+}
+
+# Create a directory, and any missing enclosing directories.
+# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
+function make_directory
+{
+  local cmd="mkdir -p '$1'"
+  printf "  %s Creating directory %s..." "$MARKER" "$1"
+  local check_cmd="[ ! -d '$1' ]"
+  msg=$( eval "$check_cmd" 2>&1 )
+  if validate_ret_skip $? "Directory already exists."; then
+    return
+  fi
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Create a symlink from the first argument to the second argument
+# If the link already exists: fail if $USE_EXISTING is 0, otherwise skip.
+function make_symlink
+{
+  local cmd="ln -s '$1' '$2'"
+  printf "  %s Creating symlink from %s to %s..." "$MARKER" "$1" "$2"
+  local check_cmd="[ ! -e '$2' ]"
+  msg=$( eval "$check_cmd" 2>&1 )
+  if validate_ret_skip $? "File already exists."; then
+    return
+  fi
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Go into the directory or repository, even if $DRY_RUN is non-zero.
+# If the directory does not exist, fail and log an error.
+# Otherwise, silently succeed.
+function goto_dir
+{
+  if ! cd "$1" 1>/dev/null 2>/dev/null ; then
+    printf "  %s Changing to directory %s..." "$MARKER" "$1"
+    validate_ret 1 "$1: Not found. Stopping."
+  fi
+}
+
+# Clone a repository into a directory.
+# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
+function clone_repo
+{
+  local cmd="git clone $TOR_EXTRA_CLONE_ARGS '$1' '$2'"
+  printf "  %s Cloning %s into %s..." "$MARKER" "$1" "$2"
+  local check_cmd="[ ! -d '$2' ]"
+  msg=$( eval "$check_cmd" 2>&1 )
+  if validate_ret_skip $? "Directory already exists."; then
+    # If we skip the clone, we need to do a fetch
+    goto_dir "$ORIGIN_PATH"
+    fetch_remote "origin"
+    return
+  fi
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Add a remote by name and URL.
+# If the remote already exists: fail if $USE_EXISTING is 0, otherwise skip.
+function add_remote
+{
+  local cmd="git remote add '$1' '$2'"
+  printf "  %s Adding remote %s at %s..." "$MARKER" "$1" "$2"
+  local check_cmd="git remote get-url '$1'"
+  msg=$( eval "$check_cmd" 2>&1 )
+  ret=$?
+  # We don't want a remote, so we invert the exit status
+  if validate_ret_skip $(( ! ret )) \
+                       "Remote already exists for $1 at $msg."; then
+    return
+  fi
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Set a remote's push URL by name and URL.
+function set_remote_push
+{
+  local cmd="git remote set-url --push '$1' '$2'"
+  printf "  %s Setting remote %s push URL to '%s'..." "$MARKER" "$1" "$2"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Fetch a remote by name.
+function fetch_remote
+{
+  local cmd="git fetch '$1'"
+  printf "  %s Fetching %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Replace the fetch configs for a remote with config if they match a pattern.
+function replace_fetch_config
+{
+  local cmd="git config --replace-all remote.'$1'.fetch '$2' '$3'"
+  printf "  %s Replacing %s fetch configs for '%s'..." \
+    "$MARKER" "$1" "$3"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Set up the tor-github PR config, so tor-github/pr/NNNN/head points to GitHub
+# PR NNNN. In some repositories, "/head" is optional.
+function set_tor_github_pr_fetch_config
+{
+  # Standard branches
+  replace_fetch_config tor-github \
+    "+refs/heads/*:refs/remotes/tor-github/*" \
+    "refs/heads"
+  # PRs
+  replace_fetch_config "tor-github" \
+    "+refs/pull/*:refs/remotes/tor-github/pr/*" \
+    "refs/pull.*pr"
+}
+
+# Add a new worktree for branch at path.
+# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
+function add_worktree
+{
+  local cmd="git worktree add '$2' '$1'"
+  printf "  %s Adding worktree for %s at %s..." "$MARKER" "$1" "$2"
+  local check_cmd="[ ! -d '$2' ]"
+  msg=$( eval "$check_cmd" 2>&1 )
+  if validate_ret_skip $? "Directory already exists."; then
+    return
+  fi
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Switch to the given branch name.
+# If the branch does not exist: fail.
+function switch_branch
+{
+  local cmd="git checkout '$1'"
+  printf "  %s Switching branch to %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Checkout a new branch with the given branch name.
+# If the branch already exists: fail if $USE_EXISTING is 0, otherwise skip.
+function new_branch
+{
+  local cmd="git checkout -b '$1'"
+  printf "  %s Creating new branch %s..." "$MARKER" "$1"
+  local check_cmd="git branch --list '$1'"
+  msg=$( eval "$check_cmd" 2>&1 )
+  if validate_ret_skip $? "Branch already exists."; then
+    return
+  fi
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Switch to an existing branch, or checkout a new branch with the given
+# branch name.
+function switch_or_new_branch
+{
+  local cmd="git rev-parse --verify '$1'"
+  if [ $DRY_RUN -eq 0 ]; then
+    # Call switch_branch if there is a branch, or new_branch if there is not
+    msg=$( eval "$cmd" 2>&1 )
+    RET=$?
+    if [ $RET -eq 0 ]; then
+      # Branch: (commit id)
+      switch_branch "$1"
+    elif [ $RET -eq 128 ]; then
+      # Not a branch: "fatal: Needed a single revision"
+      new_branch "$1"
+    else
+      # Unexpected return value
+      validate_ret $RET "$msg"
+    fi
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:"
+    switch_branch "$1"
+    new_branch "$1"
+  fi
+}
+
+# Set the upstream for branch to upstream.
+function set_upstream
+{
+  # Note the argument order is swapped
+  local cmd="git branch --set-upstream-to='$2' '$1'"
+  printf "  %s Setting upstream for %s to %s..." "$MARKER" "$1" "$2"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+###############
+# Entry point #
+###############
+
+printf "%s Setting up the repository and remote %s\\n" "$MARKER" \
+  "${BYEL}origin${CNRM}"
+# First, fetch the origin.
+ORIGIN_PARENT=$(dirname "$ORIGIN_PATH")
+make_directory "$ORIGIN_PARENT"
+# This is just cd with an error check
+goto_dir "$ORIGIN_PARENT"
+
+# clone repository / origin remote
+clone_repo "$GIT_ORIGIN_PULL" "$TOR_MASTER_NAME"
+goto_dir "$ORIGIN_PATH"
+set_remote_push "origin" "$GIT_ORIGIN_PUSH"
+
+# upstream remote, if different to origin
+if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
+  printf "%s Setting up remote %s\\n" "$MARKER" \
+    "${BYEL}$DEFAULT_UPSTREAM_REMOTE${CNRM}"
+  add_remote "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PULL"
+  set_remote_push "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PUSH"
+  fetch_remote "$DEFAULT_UPSTREAM_REMOTE"
+fi
+
+# GitHub remote
+printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-github${CNRM}"
+# Add remote
+add_remote "tor-github" "$GITHUB_PULL"
+set_remote_push "tor-github" "$GITHUB_PUSH"
+# Add custom fetch for PRs
+set_tor_github_pr_fetch_config
+# Now fetch them all
+fetch_remote "tor-github"
+
+# Extra remote
+if [ "$TOR_EXTRA_REMOTE_NAME" ]; then
+  printf "%s Setting up remote %s\\n" "$MARKER" \
+    "${BYEL}$TOR_EXTRA_REMOTE_NAME${CNRM}"
+  # Add remote
+  add_remote "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PULL"
+  set_remote_push "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PUSH"
+  # But leave it to the user to decide if they want to fetch it
+  #fetch_remote "$TOR_EXTRA_REMOTE_NAME"
+fi
+
+# Go over all configured worktree.
+for ((i=0; i<COUNT; i++)); do
+  branch=${!WORKTREE[$i]:0:1}
+  repo_path=${!WORKTREE[$i]:1:1}
+
+  printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}"
+  # We cloned the repository, and master is the default branch
+  if [ "$branch" = "master" ]; then
+    if [ "$TOR_MASTER_NAME" != "master" ]; then
+      # Set up a master link in the worktree directory
+      make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master"
+    fi
+  else
+    # git makes worktree directories if they don't exist
+    add_worktree "origin/$branch" "$repo_path"
+  fi
+  goto_dir "$repo_path"
+  switch_or_new_branch "$branch"
+  set_upstream "$branch" "origin/$branch"
+done
+
+echo
+echo "Remember to copy the git hooks from tor/scripts/git/*.git-hook to"
+echo "$ORIGIN_PATH/.git/hooks/*"
diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook
new file mode 100755
index 0000000000..eae4f999e7
--- /dev/null
+++ b/scripts/git/post-merge.git-hook
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# This is post-merge git hook script to check for changes in:
+# * git hook scripts
+# * helper scripts for using git efficiently.
+# If any changes are detected, a diff of them is printed.
+#
+# To install this script, copy it to .git/hooks/post-merge in local copy of
+# tor git repo and make sure it has permission to execute.
+
+git_toplevel=$(git rev-parse --show-toplevel)
+
+check_for_diffs() {
+        installed="$git_toplevel/.git/hooks/$1"
+        latest="$git_toplevel/scripts/git/$1.git-hook"
+
+        if [ -e "$installed" ]
+        then
+               if ! cmp "$installed" "$latest" >/dev/null 2>&1
+               then
+                        echo "ATTENTION: $1 hook has changed:"
+                        echo "==============================="
+                        diff -u "$installed" "$latest"
+               fi
+        fi
+}
+
+check_for_script_update() {
+        fullpath="$1"
+
+        if ! git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" >/dev/null
+        then
+                echo "ATTENTION: $1 has changed:"
+                git --no-pager diff ORIG_HEAD HEAD -- "$fullpath"
+        fi
+}
+
+cur_branch=$(git rev-parse --abbrev-ref HEAD)
+if [ "$cur_branch" != "master" ]; then
+  echo "post-merge: Not a master branch. Skipping."
+  exit 0
+fi
+
+check_for_diffs "pre-push"
+check_for_diffs "pre-commit"
+check_for_diffs "post-merge"
+
+for file in "$git_toplevel"/scripts/git/* ; do
+        check_for_script_update "$file"
+done
+
diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook
new file mode 100755
index 0000000000..f630a242bd
--- /dev/null
+++ b/scripts/git/pre-commit.git-hook
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+#
+# To install this script, copy it to .git/hooks/pre-commit in local copy of
+# tor git repo and make sure it has permission to execute.
+#
+# This is pre-commit git hook script that prevents commiting your changeset if
+# it fails our code formatting, changelog entry formatting, module include
+# rules, or best practices tracker.
+
+workdir=$(git rev-parse --show-toplevel)
+
+cd "$workdir" || exit 1
+
+set -e
+
+if [ $# -eq 0 ]; then
+    # When called in pre-commit, check the files modified in this commit
+    CHECK_FILTER="git diff --cached --name-only --diff-filter=ACMR"
+    # Use the appropriate owned tor source list to filter the changed files
+
+    # This is the layout in 0.3.5 and later.
+
+    # Keep these lists consistent:
+    #   - OWNED_TOR_C_FILES in Makefile.am
+    #   - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook
+    #   - try_parse in check_cocci_parse.sh
+    CHECK_FILES="$($CHECK_FILTER \
+                           src/lib/*/*.[ch] \
+                           src/core/*/*.[ch] \
+                           src/feature/*/*.[ch] \
+                           src/app/*/*.[ch] \
+                           src/test/*.[ch] \
+                           src/test/*/*.[ch] \
+                           src/tools/*.[ch] \
+                  )"
+else
+    # When called in pre-push, concatenate the argument array
+    # Fails on special characters in file names
+    CHECK_FILES="$*"
+fi
+
+## General File Checks
+
+if [ -n "$(ls ./changes/)" ]; then
+    python scripts/maint/lintChanges.py ./changes/*
+fi
+
+if [ -e scripts/maint/checkShellScripts.sh ]; then
+    scripts/maint/checkShellScripts.sh
+fi
+
+# Always run the practracker unit tests
+PT_DIR=scripts/maint/practracker
+
+if [ -e "${PT_DIR}/test_practracker.sh" ]; then
+    "${PT_DIR}/test_practracker.sh"
+fi
+
+if [ -e scripts/maint/checkSpaceTest.sh ]; then
+    scripts/maint/checkSpaceTest.sh
+fi
+
+if [ ! "$CHECK_FILES" ]; then
+    echo "No modified tor-owned source files, skipping further checks"
+    exit 0
+fi
+
+## Owned Source File Checks
+
+printf "Modified tor-owned source files:\\n%s\\n" "$CHECK_FILES"
+
+# We want word splitting here, because file names are space separated
+# shellcheck disable=SC2086
+perl scripts/maint/checkSpace.pl -C \
+     $CHECK_FILES
+
+if test -e scripts/maint/practracker/includes.py; then
+    python scripts/maint/practracker/includes.py
+fi
+
+# Only call practracker if ${PT_DIR}/.enable_practracker_in_hooks exists
+# We do this check so that we can enable practracker in hooks in master, and
+# disable it on maint branches
+if [ -e "${PT_DIR}/practracker.py" ]; then
+    if [ -e "${PT_DIR}/.enable_practracker_in_hooks" ]; then
+        python3 "${PT_DIR}/practracker.py" "$workdir"
+    fi
+fi
+
+if [ -e scripts/coccinelle/check_cocci_parse.sh ]; then
+
+    # Run a verbose cocci parse check on the changed files
+    # (spatch is slow, so we don't want to check all the files.)
+    #
+    # We want word splitting here, because file names are space separated
+    # shellcheck disable=SC2086
+    VERBOSE=1 scripts/coccinelle/check_cocci_parse.sh \
+              $CHECK_FILES
+fi
diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook
new file mode 100755
index 0000000000..efa45b9860
--- /dev/null
+++ b/scripts/git/pre-push.git-hook
@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+
+# git pre-push hook script to:
+# 0) Call the pre-commit hook, if it is available
+# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-*
+#    or maint-*
+# 2) Disallow pushing branches other than master, release-*
+#    and maint-* to origin (e.g. gitweb.torproject.org)
+#
+# To install this script, copy it into .git/hooks/pre-push path in your
+# local copy of git repository. Make sure it has permission to execute.
+# Furthermore, make sure that TOR_UPSTREAM_REMOTE_NAME environment
+# variable is set to local name of git remote that corresponds to upstream
+# repository on e.g. git.torproject.org.
+#
+# The following sample script was used as starting point:
+# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample
+
+# Are you adding a new check to the git hooks?
+#  - Common checks belong in the pre-commit hook
+#  - Push-only checks belong in the pre-push hook
+
+echo "Running pre-push hook"
+
+z40=0000000000000000000000000000000000000000
+
+upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
+
+# The working directory
+workdir=$(git rev-parse --show-toplevel)
+# The .git directory
+# If $workdir is a worktree, then $gitdir is not $workdir/.git
+gitdir=$(git rev-parse --git-dir)
+
+cd "$workdir" || exit 1
+
+remote="$1"
+remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1)
+
+
+ref_is_upstream_branch() {
+    if [ "$1" == "refs/heads/master" ] ||
+        [[ "$1" == refs/heads/release-* ]] ||
+        [[ "$1" == refs/heads/maint-* ]]; then
+        return 1
+    fi
+}
+
+# shellcheck disable=SC2034
+while read -r local_ref local_sha remote_ref remote_sha
+do
+    if [ "$local_sha" = $z40 ]; then
+        # Handle delete
+        :
+    else
+        if [ "$remote_sha" = $z40 ]; then
+            # New branch, examine commits not in master
+            range="master...$local_sha"
+        else
+            # Update to existing branch, examine new commits
+            range="$remote_sha..$local_sha"
+        fi
+
+        # Call the pre-commit hook for the common checks, if it is executable
+        pre_commit=${gitdir}/hooks/pre-commit
+        if [ -x "$pre_commit" ]; then
+            # Only check the files newly modified in this branch
+            CHECK_FILTER="git diff --name-only --diff-filter=ACMR $range"
+            # Use the appropriate owned tor source list to filter the changed
+            # files
+            # This is the layout in 0.3.5
+            # Keep these lists consistent:
+            #   - OWNED_TOR_C_FILES in Makefile.am
+            #   - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook
+            #   - try_parse in check_cocci_parse.sh
+            CHECK_FILES="$($CHECK_FILTER \
+                                   src/lib/*/*.[ch] \
+                                   src/core/*/*.[ch] \
+                                   src/feature/*/*.[ch] \
+                                   src/app/*/*.[ch] \
+                                   src/test/*.[ch] \
+                                   src/test/*/*.[ch] \
+                                   src/tools/*.[ch] \
+                          )"
+
+            # We want word splitting here, because file names are space
+            # separated
+            # shellcheck disable=SC2086
+            if ! "$pre_commit" $CHECK_FILES ; then
+                exit 1
+            fi
+        fi
+
+        if [[ "$remote_name" != "$upstream_name" ]]; then
+            echo "Not pushing to upstream - refraining from further checks"
+            continue
+        fi
+
+        if (ref_is_upstream_branch "$local_ref" == 0 ||
+                ref_is_upstream_branch "$remote_ref"  == 0) &&
+            [ "$local_ref" != "$remote_ref" ]; then
+            if [ "$remote" == "origin" ]; then
+                echo >&2 "Not pushing: $local_ref to $remote_ref"
+                echo >&2 "If you really want to push this, use --no-verify."
+                exit 1
+            else
+                continue
+            fi
+        fi
+
+        # Check for fixup! commit
+        commit=$(git rev-list -n 1 --grep '^fixup!' "$range")
+        if [ -n "$commit" ]; then
+            echo >&2 "Found fixup! commit in $local_ref, not pushing"
+            echo >&2 "If you really want to push this, use --no-verify."
+            exit 1
+        fi
+
+        # Check for squash! commit
+        commit=$(git rev-list -n 1 --grep '^squash!' "$range")
+        if [ -n "$commit" ]; then
+            echo >&2 "Found squash! commit in $local_ref, not pushing"
+            echo >&2 "If you really want to push this, use --no-verify."
+            exit 1
+        fi
+    fi
+done
+
+exit 0
diff --git a/scripts/maint/add_c_file.py b/scripts/maint/add_c_file.py
new file mode 100755
index 0000000000..e1e224d8d5
--- /dev/null
+++ b/scripts/maint/add_c_file.py
@@ -0,0 +1,333 @@
+#!/usr/bin/env python3
+
+"""
+   Add a C file with matching header to the Tor codebase.  Creates
+   both files from templates, and adds them to the right include.am file.
+
+   This script takes paths relative to the top-level tor directory. It
+   expects to be run from that directory.
+
+   This script creates files, and inserts them into include.am, also
+   relative to the top-level tor directory.
+
+   But the template content in those files is relative to tor's src
+   directory. (This script strips "src" from the paths used to create
+   templated comments and macros.)
+
+   This script expects posix paths, so it should be run with a python
+   where os.path is posixpath. (Rather than ntpath.) This probably means
+   Linux, macOS, or BSD, although it might work on Windows if your python
+   was compiled with mingw, MSYS, or cygwin.
+
+   Example usage:
+
+   % add_c_file.py ./src/feature/dirauth/ocelot.c
+"""
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import time
+
+def tordir_file(fname):
+    """Make fname relative to the current directory, which should be the
+       top-level tor directory. Also performs basic path simplifications."""
+    return os.path.normpath(os.path.relpath(fname))
+
+def srcdir_file(tor_fname):
+    """Make tor_fname relative to tor's "src" directory.
+       Also performs basic path simplifications.
+       (This function takes paths relative to the top-level tor directory,
+       but outputs a path that is relative to tor's src directory.)"""
+    return os.path.normpath(os.path.relpath(tor_fname, 'src'))
+
+def guard_macro(src_fname):
+    """Return the guard macro that should be used for the header file
+       'src_fname'. This function takes paths relative to tor's src directory.
+    """
+    td = src_fname.replace(".", "_").replace("/", "_").upper()
+    return "TOR_{}".format(td)
+
+def makeext(fname, new_extension):
+    """Replace the extension for the file called 'fname' with 'new_extension'.
+       This function takes and returns paths relative to either the top-level
+       tor directory, or tor's src directory, and returns the same kind
+       of path.
+    """
+    base = os.path.splitext(fname)[0]
+    return base + "." + new_extension
+
+def instantiate_template(template, tor_fname):
+    """
+    Fill in a template with string using the fields that should be used
+    for 'tor_fname'.
+
+    This function takes paths relative to the top-level tor directory,
+    but the paths in the completed template are relative to tor's src
+    directory. (Except for one of the fields, which is just a basename).
+    """
+    src_fname = srcdir_file(tor_fname)
+    names = {
+        # The relative location of the header file.
+        'header_path' : makeext(src_fname, "h"),
+        # The relative location of the C file file.
+        'c_file_path' : makeext(src_fname, "c"),
+        # The truncated name of the file.
+        'short_name' : os.path.basename(src_fname),
+        # The current year, for the copyright notice
+        'this_year' : time.localtime().tm_year,
+        # An appropriate guard macro, for the header.
+        'guard_macro' : guard_macro(src_fname),
+    }
+
+    return template.format(**names)
+
+# This template operates on paths relative to tor's src directory
+HEADER_TEMPLATE = """\
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-{this_year}, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file {short_name}
+ * @brief Header for {c_file_path}
+ **/
+
+#ifndef {guard_macro}
+#define {guard_macro}
+
+#endif /* !defined({guard_macro}) */
+"""
+
+# This template operates on paths relative to the tor's src directory
+C_FILE_TEMPLATE = """\
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-{this_year}, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file {short_name}
+ * @brief DOCDOC
+ **/
+
+#include "orconfig.h"
+#include "{header_path}"
+"""
+
+class AutomakeChunk:
+    """
+    Represents part of an automake file.  If it is decorated with
+    an ADD_C_FILE comment, it has a "kind" based on what to add to it.
+    Otherwise, it only has a bunch of lines in it.
+
+    This class operates on paths relative to the top-level tor directory.
+    """
+    pat = re.compile(r'# ADD_C_FILE: INSERT (\S*) HERE', re.I)
+
+    def __init__(self):
+        self.lines = []
+        self.kind = ""
+        self.hasBlank = False # true if we end with a blank line.
+
+    def addLine(self, line):
+        """
+        Insert a line into this chunk while parsing the automake file.
+
+        Return True if we have just read the last line in the chunk, and
+        False otherwise.
+        """
+        m = self.pat.match(line)
+        if m:
+            if self.lines:
+                raise ValueError("control line not preceded by a blank line")
+            self.kind = m.group(1)
+
+        if line.strip() == "":
+            self.hasBlank = True
+            return True
+
+        self.lines.append(line)
+
+        return False
+
+    def insertMember(self, new_tor_fname):
+        """
+        Add a new file name new_tor_fname to this chunk.  Try to insert it in
+        alphabetical order with matching indentation, but don't freak out too
+        much if the source isn't consistent.
+
+        Assumes that this chunk is of the form:
+           FOOBAR = \
+              X     \
+              Y     \
+              Z
+
+        This function operates on paths relative to the top-level tor
+        directory.
+        """
+        prespace = "\t"
+        postspace = "\t\t"
+        for lineno, line in enumerate(self.lines):
+            m = re.match(r'(\s+)(\S+)(\s+)\\', line)
+            if not m:
+                continue
+            prespace, cur_tor_fname, postspace = m.groups()
+            if cur_tor_fname > new_tor_fname:
+                self.insert_before(lineno, new_tor_fname, prespace, postspace)
+                return
+        self.insert_at_end(new_tor_fname, prespace, postspace)
+
+    def insert_before(self, lineno, new_tor_fname, prespace, postspace):
+        self.lines.insert(lineno,
+                          "{}{}{}\\\n".format(prespace, new_tor_fname,
+                                              postspace))
+
+    def insert_at_end(self, new_tor_fname, prespace, postspace):
+        lastline = self.lines[-1].strip()
+        self.lines[-1] = '{}{}{}\\\n'.format(prespace, lastline, postspace)
+        self.lines.append("{}{}\n".format(prespace, new_tor_fname))
+
+    def dump(self, f):
+        """Write all the lines in this chunk to the file 'f'."""
+        for line in self.lines:
+            f.write(line)
+            if not line.endswith("\n"):
+                f.write("\n")
+
+        if self.hasBlank:
+            f.write("\n")
+
+class ParsedAutomake:
+    """A sort-of-parsed automake file, with identified chunks into which
+       headers and c files can be inserted.
+
+       This class operates on paths relative to the top-level tor directory.
+    """
+    def __init__(self):
+        self.chunks = []
+        self.by_type = {}
+
+    def addChunk(self, chunk):
+        """Add a newly parsed AutomakeChunk to this file."""
+        self.chunks.append(chunk)
+        self.by_type[chunk.kind.lower()] = chunk
+
+    def add_file(self, tor_fname, kind):
+        """Insert a file tor_fname of kind 'kind' to the appropriate
+           section of this file. Return True if we added it.
+
+           This function operates on paths relative to the top-level tor
+           directory.
+        """
+        if kind.lower() in self.by_type:
+            self.by_type[kind.lower()].insertMember(tor_fname)
+            return True
+        else:
+            return False
+
+    def dump(self, f):
+        """Write this file into a file 'f'."""
+        for chunk in self.chunks:
+            chunk.dump(f)
+
+def get_include_am_location(tor_fname):
+    """Find the right include.am file for introducing a new file
+       tor_fname.  Return None if we can't guess one.
+
+       Note that this function is imperfect because our include.am layout is
+       not (yet) consistent.
+
+       This function operates on paths relative to the top-level tor directory.
+    """
+    # Strip src for pattern matching, but add it back when returning the path
+    src_fname = srcdir_file(tor_fname)
+    m = re.match(r'^(lib|core|feature|app)/([a-z0-9_]*)/', src_fname)
+    if m:
+        return "src/{}/{}/include.am".format(m.group(1),m.group(2))
+
+    if re.match(r'^test/', src_fname):
+        return "src/test/include.am"
+
+    return None
+
+def run(fname):
+    """
+    Create a new C file and H file corresponding to the filename "fname",
+    and add them to the corresponding include.am.
+
+    This function operates on paths relative to the top-level tor directory.
+    """
+
+    # Make sure we're in the top-level tor directory,
+    # which contains the src directory
+    if not os.path.isdir("src"):
+        raise RuntimeError("Could not find './src/'. "
+                           "Run this script from the top-level tor source "
+                           "directory.")
+
+    # And it looks like a tor/src directory
+    if not os.path.isfile("src/include.am"):
+        raise RuntimeError("Could not find './src/include.am'. "
+                           "Run this script from the top-level tor source "
+                           "directory.")
+
+    # Make the file name relative to the top-level tor directory
+    tor_fname = tordir_file(fname)
+    # And check that we're adding files to the "src" directory,
+    # with canonical paths
+    if tor_fname[:4] != "src/":
+        raise ValueError("Requested file path '{}' canonicalized to '{}', "
+                         "but the canonical path did not start with 'src/'. "
+                         "Please add files to the src directory."
+                         .format(fname, tor_fname))
+
+    c_tor_fname = makeext(tor_fname, "c")
+    h_tor_fname = makeext(tor_fname, "h")
+
+    if os.path.exists(c_tor_fname):
+        print("{} already exists".format(c_tor_fname))
+        return 1
+    if os.path.exists(h_tor_fname):
+        print("{} already exists".format(h_tor_fname))
+        return 1
+
+    with open(c_tor_fname, 'w') as f:
+        f.write(instantiate_template(C_FILE_TEMPLATE, c_tor_fname))
+
+    with open(h_tor_fname, 'w') as f:
+        f.write(instantiate_template(HEADER_TEMPLATE, h_tor_fname))
+
+    iam = get_include_am_location(c_tor_fname)
+    if iam is None or not os.path.exists(iam):
+        print("Made files successfully but couldn't identify include.am for {}"
+              .format(c_tor_fname))
+        return 1
+
+    amfile = ParsedAutomake()
+    cur_chunk = AutomakeChunk()
+    with open(iam) as f:
+        for line in f:
+            if cur_chunk.addLine(line):
+                amfile.addChunk(cur_chunk)
+                cur_chunk = AutomakeChunk()
+        amfile.addChunk(cur_chunk)
+
+    amfile.add_file(c_tor_fname, "sources")
+    amfile.add_file(h_tor_fname, "headers")
+
+    with open(iam+".tmp", 'w') as f:
+        amfile.dump(f)
+
+    os.rename(iam+".tmp", iam)
+
+if __name__ == '__main__':
+    import sys
+    sys.exit(run(sys.argv[1]))
diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives
deleted file mode 100755
index ca267a865e..0000000000
--- a/scripts/maint/annotate_ifdef_directives
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2017-2019, The Tor Project, Inc.
-# See LICENSE for licensing information
-
-import re
-
-LINE_OBVIOUSNESS_LIMIT = 4
-
-class Problem(Exception):
-    pass
-
-def uncomment(s):
-    s = re.sub(r'//.*','',s)
-    s = re.sub(r'/\*.*','',s)
-    return s.strip()
-
-def translate(f_in, f_out):
-    whole_file = []
-    stack = []
-    cur_level = whole_file
-    lineno = 0
-    for line in f_in:
-        lineno += 1
-        m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)',
-                     line)
-        if not m:
-            f_out.write(line)
-            continue
-        command,rest = m.groups()
-        if command in ("if", "ifdef", "ifndef"):
-            # The #if directive pushes us one level lower on the stack.
-            if command == 'ifdef':
-                rest = "defined(%s)"%uncomment(rest)
-            elif command == 'ifndef':
-                rest = "!defined(%s)"%uncomment(rest)
-            elif rest.endswith("\\"):
-                rest = rest[:-1]+"..."
-
-            rest = uncomment(rest)
-
-            new_level = [ (command, rest, lineno) ]
-            stack.append(cur_level)
-            cur_level = new_level
-            f_out.write(line)
-        elif command in ("else", "elif"):
-            if len(cur_level) == 0 or cur_level[-1][0] == 'else':
-                raise Problem("Unexpected #%s on %d"% (command,lineno))
-            if (len(cur_level) == 1 and command == 'else' and
-                lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT):
-                f_out.write("#else /* !(%s) */\n"%cur_level[0][1])
-            else:
-                f_out.write(line)
-            cur_level.append((command, rest, lineno))
-        else:
-            assert command == 'endif'
-            if len(stack) == 0:
-                raise Problem("Unmatched #%s on %s"% (command,lineno))
-            if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT:
-                f_out.write(line)
-            elif len(cur_level) == 1 or (
-                    len(cur_level) == 2 and cur_level[1][0] == 'else'):
-                f_out.write("#endif /* %s */\n"%cur_level[0][1])
-            else:
-                f_out.write("#endif /* %s || ... */\n"%cur_level[0][1])
-            cur_level = stack.pop()
-    if len(stack) or cur_level != whole_file:
-        raise Problem("Missing #endif")
-
-import sys,os
-for fn in sys.argv[1:]:
-    with open(fn+"_OUT", 'w') as output_file:
-        translate(open(fn, 'r'), output_file)
-    os.rename(fn+"_OUT", fn)
-
diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py
new file mode 100755
index 0000000000..9ca090d595
--- /dev/null
+++ b/scripts/maint/annotate_ifdef_directives.py
@@ -0,0 +1,322 @@
+#!/usr/bin/env python
+# Copyright (c) 2017-2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+r"""
+This script iterates over a list of C files. For each file, it looks at the
+#if/#else C macros, and annotates them with comments explaining what they
+match.
+
+For example, it replaces this kind of input...
+
+>>> INPUT = '''
+... #ifdef HAVE_OCELOT
+...   C code here
+... #if MIMSY == BOROGROVE
+...   block 1
+...   block 1
+...   block 1
+...   block 1
+... #else
+...   block 2
+...   block 2
+...   block 2
+...   block 2
+... #endif
+... #endif
+... '''
+
+With this kind of output:
+>>> EXPECTED_OUTPUT = '''
+... #ifdef HAVE_OCELOT
+...   C code here
+... #if MIMSY == BOROGROVE
+...   block 1
+...   block 1
+...   block 1
+...   block 1
+... #else /* !(MIMSY == BOROGROVE) */
+...   block 2
+...   block 2
+...   block 2
+...   block 2
+... #endif /* MIMSY == BOROGROVE */
+... #endif /* defined(HAVE_OCELOT) */
+... '''
+
+Here's how to use it:
+>>> import sys
+>>> if sys.version_info.major < 3: from cStringIO import StringIO
+>>> if sys.version_info.major >= 3: from io import StringIO
+
+>>> OUTPUT = StringIO()
+>>> translate(StringIO(INPUT), OUTPUT)
+>>> assert OUTPUT.getvalue() == EXPECTED_OUTPUT
+
+Note that only #else and #endif lines are annotated.  Existing comments
+on those lines are removed.
+"""
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import re
+
+# Any block with fewer than this many lines does not need annotations.
+LINE_OBVIOUSNESS_LIMIT = 4
+
+# Maximum line width.  This includes a terminating newline character.
+#
+# (This is the maximum before encoding, so that if the the operating system
+# uses multiple characers to encode newline, that's still okay.)
+LINE_WIDTH=80
+
+class Problem(Exception):
+    pass
+
+def close_parens_needed(expr):
+    """Return the number of left-parentheses needed to make 'expr'
+       balanced.
+
+    >>> close_parens_needed("1+2")
+    0
+    >>> close_parens_needed("(1 + 2)")
+    0
+    >>> close_parens_needed("(1 + 2")
+    1
+    >>> close_parens_needed("(1 + (2 *")
+    2
+    >>> close_parens_needed("(1 + (2 * 3) + (4")
+    2
+    """
+    return expr.count("(") - expr.count(")")
+
+def truncate_expression(expr, new_width):
+    """Given a parenthesized C expression in 'expr', try to return a new
+       expression that is similar to 'expr', but no more than 'new_width'
+       characters long.
+
+       Try to return an expression with balanced parentheses.
+
+    >>> truncate_expression("1+2+3", 8)
+    '1+2+3'
+    >>> truncate_expression("1+2+3+4+5", 8)
+    '1+2+3...'
+    >>> truncate_expression("(1+2+3+4)", 8)
+    '(1+2...)'
+    >>> truncate_expression("(1+(2+3+4))", 8)
+    '(1+...)'
+    >>> truncate_expression("(((((((((", 8)
+    '((...))'
+    """
+    if len(expr) <= new_width:
+        # The expression is already short enough.
+        return expr
+
+    ellipsis = "..."
+
+    # Start this at the minimum that we might truncate.
+    n_to_remove = len(expr) + len(ellipsis) - new_width
+
+    # Try removing characters, one by one, until we get something where
+    # re-balancing the parentheses still fits within the limit.
+    while n_to_remove < len(expr):
+        truncated = expr[:-n_to_remove] + ellipsis
+        truncated += ")" * close_parens_needed(truncated)
+        if len(truncated) <= new_width:
+            return truncated
+        n_to_remove += 1
+
+    return ellipsis
+
+def commented_line(fmt, argument, maxwidth=LINE_WIDTH):
+    # (This is a raw docstring so that our doctests can use \.)
+    r"""
+    Return fmt%argument, for use as a commented line.  If the line would
+    be longer than maxwidth, truncate argument but try to keep its
+    parentheses balanced.
+
+    Requires that fmt%"..." will fit into maxwidth characters.
+
+    Requires that fmt ends with a newline.
+
+    >>> commented_line("/* %s */\n", "hello world", 32)
+    '/* hello world */\n'
+    >>> commented_line("/* %s */\n", "hello world", 15)
+    '/* hello... */\n'
+    >>> commented_line("#endif /* %s */\n", "((1+2) && defined(FOO))", 32)
+    '#endif /* ((1+2) && defi...) */\n'
+
+
+    The default line limit is 80 characters including the newline:
+
+    >>> long_argument = "long " * 100
+    >>> long_line = commented_line("#endif /* %s */\n", long_argument)
+    >>> len(long_line)
+    80
+
+    >>> long_line[:40]
+    '#endif /* long long long long long long '
+    >>> long_line[40:]
+    'long long long long long long lon... */\n'
+
+    If a line works out to being 80 characters naturally, it isn't truncated,
+    and no ellipsis is added.
+
+    >>> medium_argument = "a"*66
+    >>> medium_line = commented_line("#endif /* %s */\n", medium_argument)
+    >>> len(medium_line)
+    80
+    >>> "..." in medium_line
+    False
+    >>> medium_line[:40]
+    '#endif /* aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+    >>> medium_line[40:]
+    'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa */\n'
+
+
+    """
+    assert fmt.endswith("\n")
+    result = fmt % argument
+    if len(result) <= maxwidth:
+        return result
+    else:
+        # How long can we let the argument be?  Try filling in the
+        # format with an empty argument to find out.
+        max_arg_width = maxwidth - len(fmt % "")
+        result = fmt % truncate_expression(argument, max_arg_width)
+        assert len(result) <= maxwidth
+        return result
+
+def negate(expr):
+    """Return a negated version of expr; try to avoid double-negation.
+
+    We usually wrap expressions in parentheses and add a "!".
+    >>> negate("A && B")
+    '!(A && B)'
+
+    But if we recognize the expression as negated, we can restore it.
+    >>> negate(negate("A && B"))
+    'A && B'
+
+    The same applies for defined(FOO).
+    >>> negate("defined(FOO)")
+    '!defined(FOO)'
+    >>> negate(negate("defined(FOO)"))
+    'defined(FOO)'
+
+    Internal parentheses don't confuse us:
+    >>> negate("!(FOO) && !(BAR)")
+    '!(!(FOO) && !(BAR))'
+
+    """
+    expr = expr.strip()
+    # See whether we match !(...), with no intervening close-parens.
+    m = re.match(r'^!\s*\(([^\)]*)\)$', expr)
+    if m:
+        return m.group(1)
+
+
+    # See whether we match !?defined(...), with no intervening close-parens.
+    m = re.match(r'^(!?)\s*(defined\([^\)]*\))$', expr)
+    if m:
+        if m.group(1) == "!":
+            prefix = ""
+        else:
+            prefix = "!"
+        return prefix + m.group(2)
+
+    return "!(%s)" % expr
+
+def uncomment(s):
+    """
+    Remove existing trailing comments from an #else or #endif line.
+    """
+    s = re.sub(r'//.*','',s)
+    s = re.sub(r'/\*.*','',s)
+    return s.strip()
+
+def translate(f_in, f_out):
+    """
+    Read a file from f_in, and write its annotated version to f_out.
+    """
+    # A stack listing our current if/else state.  Each member of the stack
+    # is a list of directives.  Each directive is a 3-tuple of
+    #    (command, rest, lineno)
+    # where "command" is one of if/ifdef/ifndef/else/elif, and where
+    # "rest" is an expression in a format suitable for use with #if, and where
+    # lineno is the line number where the directive occurred.
+    stack = []
+    # the stack element corresponding to the top level of the file.
+    whole_file = []
+    cur_level = whole_file
+    lineno = 0
+    for line in f_in:
+        lineno += 1
+        m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)',
+                     line)
+        if not m:
+            # no directive, so we can just write it out.
+            f_out.write(line)
+            continue
+        command,rest = m.groups()
+        if command in ("if", "ifdef", "ifndef"):
+            # The #if directive pushes us one level lower on the stack.
+            if command == 'ifdef':
+                rest = "defined(%s)"%uncomment(rest)
+            elif command == 'ifndef':
+                rest = "!defined(%s)"%uncomment(rest)
+            elif rest.endswith("\\"):
+                rest = rest[:-1]+"..."
+
+            rest = uncomment(rest)
+
+            new_level = [ (command, rest, lineno) ]
+            stack.append(cur_level)
+            cur_level = new_level
+            f_out.write(line)
+        elif command in ("else", "elif"):
+            # We stay at the same level on the stack.  If we have an #else,
+            # we comment it.
+            if len(cur_level) == 0 or cur_level[-1][0] == 'else':
+                raise Problem("Unexpected #%s on %d"% (command,lineno))
+            if (len(cur_level) == 1 and command == 'else' and
+                lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT):
+                f_out.write(commented_line("#else /* %s */\n",
+                                           negate(cur_level[0][1])))
+            else:
+                f_out.write(line)
+            cur_level.append((command, rest, lineno))
+        else:
+            # We pop one element on the stack, and comment an endif.
+            assert command == 'endif'
+            if len(stack) == 0:
+                raise Problem("Unmatched #%s on %s"% (command,lineno))
+            if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT:
+                f_out.write(line)
+            elif len(cur_level) == 1 or (
+                    len(cur_level) == 2 and cur_level[1][0] == 'else'):
+                f_out.write(commented_line("#endif /* %s */\n",
+                                           cur_level[0][1]))
+            else:
+                f_out.write(commented_line("#endif /* %s || ... */\n",
+                                           cur_level[0][1]))
+            cur_level = stack.pop()
+    if len(stack) or cur_level != whole_file:
+        raise Problem("Missing #endif")
+
+if __name__ == '__main__':
+
+    import sys,os
+
+    if sys.argv[1] == "--self-test":
+        import doctest
+        doctest.testmod()
+        sys.exit(0)
+
+    for fn in sys.argv[1:]:
+        with open(fn+"_OUT", 'w') as output_file:
+            translate(open(fn, 'r'), output_file)
+        os.rename(fn+"_OUT", fn)
diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py
index 46a3f39638..ae0ccb9e12 100755
--- a/scripts/maint/checkIncludes.py
+++ b/scripts/maint/checkIncludes.py
@@ -1,115 +1,19 @@
-#!/usr/bin/python3
+#!/usr/bin/env python
 # Copyright 2018 The Tor Project, Inc.  See LICENSE file for licensing info.
 
-"""This script looks through all the directories for files matching *.c or
-   *.h, and checks their #include directives to make sure that only "permitted"
-   headers are included.
-
-   Any #include directives with angle brackets (like #include <stdio.h>) are
-   ignored -- only directives with quotes (like #include "foo.h") are
-   considered.
-
-   To decide what includes are permitted, this script looks at a .may_include
-   file in each directory.  This file contains empty lines, #-prefixed
-   comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h)
-   for files that are permitted.
-"""
-
+# This file is no longer here; see practracker/includes.py for this
+# functionality.  This is a stub file that exists so that older git
+# hooks will know where to look.
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import fnmatch
-import os
-import re
-import sys
-
-# Global: Have there been any errors?
-trouble = False
-
-if sys.version_info[0] <= 2:
-    def open_file(fname):
-        return open(fname, 'r')
-else:
-    def open_file(fname):
-        return open(fname, 'r', encoding='utf-8')
-
-def err(msg):
-    """ Declare that an error has happened, and remember that there has
-        been an error. """
-    global trouble
-    trouble = True
-    print(msg, file=sys.stderr)
-
-def fname_is_c(fname):
-    """ Return true iff 'fname' is the name of a file that we should
-        search for possibly disallowed #include directives. """
-    return fname.endswith(".h") or fname.endswith(".c")
-
-INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"')
-RULES_FNAME = ".may_include"
-
-class Rules(object):
-    """ A 'Rules' object is the parsed version of a .may_include file. """
-    def __init__(self, dirpath):
-        self.dirpath = dirpath
-        self.patterns = []
-        self.usedPatterns = set()
-
-    def addPattern(self, pattern):
-        self.patterns.append(pattern)
-
-    def includeOk(self, path):
-        for pattern in self.patterns:
-            if fnmatch.fnmatchcase(path, pattern):
-                self.usedPatterns.add(pattern)
-                return True
-        return False
-
-    def applyToLines(self, lines, context=""):
-        lineno = 0
-        for line in lines:
-            lineno += 1
-            m = INCLUDE_PATTERN.match(line)
-            if m:
-                include = m.group(1)
-                if not self.includeOk(include):
-                    err("Forbidden include of {} on line {}{}".format(
-                        include, lineno, context))
-
-    def applyToFile(self, fname):
-        with open_file(fname) as f:
-            #print(fname)
-            self.applyToLines(iter(f), " of {}".format(fname))
-
-    def noteUnusedRules(self):
-        for p in self.patterns:
-            if p not in self.usedPatterns:
-                print("Pattern {} in {} was never used.".format(p, self.dirpath))
-
-def load_include_rules(fname):
-    """ Read a rules file from 'fname', and return it as a Rules object. """
-    result = Rules(os.path.split(fname)[0])
-    with open_file(fname) as f:
-        for line in f:
-            line = line.strip()
-            if line.startswith("#") or not line:
-                continue
-            result.addPattern(line)
-    return result
-
-list_unused = False
+import sys, os
 
-for dirpath, dirnames, fnames in os.walk("src"):
-    if ".may_include" in fnames:
-        rules = load_include_rules(os.path.join(dirpath, RULES_FNAME))
-        for fname in fnames:
-            if fname_is_c(fname):
-                rules.applyToFile(os.path.join(dirpath,fname))
-        if list_unused:
-            rules.noteUnusedRules()
+dirname = os.path.split(sys.argv[0])[0]
+new_location = os.path.join(dirname, "practracker", "includes.py")
+python = sys.executable
 
-if trouble:
-    err(
-"""To change which includes are allowed in a C file, edit the {}
-files in its enclosing directory.""".format(RULES_FNAME))
-    sys.exit(1)
+os.execl(python, python, new_location, *sys.argv[1:])
diff --git a/scripts/maint/checkManpageAlpha.py b/scripts/maint/checkManpageAlpha.py
new file mode 100755
index 0000000000..70421c2fd1
--- /dev/null
+++ b/scripts/maint/checkManpageAlpha.py
@@ -0,0 +1,72 @@
+#!/usr/bin/python
+
+import difflib
+import re
+import sys
+
+# Assume we only use the "== Section Name" section title syntax
+sectionheader_re = re.compile(r'^==+\s(.*)\s*$')
+
+# Assume we only use the "[[ItemName]]" anchor syntax
+anchor_re = re.compile(r'^\[\[([^]]+)\]\]')
+
+class Reader(object):
+    def __init__(self):
+        self.d = {}
+        # Initial state is to gather section headers
+        self.getline = self._getsec
+        self.section = None
+
+    def _getsec(self, line):
+        """Read a section header
+
+        Prepare to gather anchors from subsequent lines.  Don't change
+        state if the line isn't a section header.
+        """
+        m = sectionheader_re.match(line)
+        if not m:
+            return
+        self.anchors = anchors = []
+        self.d[m.group(1)] = anchors
+        self.getline = self._getanchor
+
+    def _getanchor(self, line):
+        """Read an anchor for an item definition
+
+        Append the anchor names to the list of items in the current
+        section.
+        """
+        m = anchor_re.match(line)
+        if not m:
+            return self._getsec(line)
+        self.anchors.append(m.group(1))
+
+    def diffsort(self, key):
+        """Unified diff of unsorted and sorted item lists
+        """
+        # Append newlines because difflib works better with them
+        a = [s + '\n' for s in self.d[key]]
+        b = sorted(a, key=str.lower)
+        return difflib.unified_diff(a, b, fromfile=key+' unsorted',
+                                    tofile=key+' sorted')
+
+def main():
+    """Diff unsorted and sorted lists of option names in a manpage
+
+    Use the file named by the first argument, or standard input if
+    there is none.
+    """
+    try:
+        fname = sys.argv[1]
+        f = open(fname, 'r')
+    except IndexError:
+        f = sys.stdin
+
+    reader = Reader()
+    for line in f:
+        reader.getline(line)
+    for key in sorted(reader.d.keys(), key=str.lower):
+        sys.stdout.writelines(reader.diffsort(key))
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/maint/checkShellScripts.sh b/scripts/maint/checkShellScripts.sh
new file mode 100755
index 0000000000..0a423be29e
--- /dev/null
+++ b/scripts/maint/checkShellScripts.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Tor Project, Inc.
+# See LICENSE for license information
+#
+# checkShellScripts.sh
+# --------------------
+# If shellcheck is installed, check all the shell scripts that we can fix.
+
+set -e
+
+# Only run this script if shellcheck is installed
+# command echoes the path to shellcheck, which is a useful diagnostic log
+if ! command -v shellcheck; then
+    printf "%s: Install shellcheck to check shell scripts.\\n" "$0"
+    exit 0
+fi
+
+# Some platforms don't have realpath
+if command -v realpath ; then
+    HERE=$(dirname "$(realpath "$0")")
+else
+    HERE=$(dirname "$0")
+    if [ ! -d "$HERE" ] || [ "$HERE" = "." ]; then
+        HERE=$(dirname "$PWD/$0")
+    fi
+fi
+TOPLEVEL=$(dirname "$(dirname "$HERE")")
+
+# Check we actually have a tor/src directory
+if [ ! -d "$TOPLEVEL/src" ]; then
+    printf "Error: Couldn't find src directory in expected location: %s\\n" \
+        "$TOPLEVEL/src"
+    exit 1
+fi
+
+# Remove obsolete scripts generated from older versions of Tor
+rm -f "$TOPLEVEL/contrib/dist/suse/tor.sh" "$TOPLEVEL/contrib/dist/tor.sh"
+
+# Check *.sh scripts, but ignore the ones that we can't fix
+find "$TOPLEVEL/contrib" "$TOPLEVEL/doc" "$TOPLEVEL/scripts" "$TOPLEVEL/src" \
+    -name "*.sh" \
+    -not -path "$TOPLEVEL/src/ext/*" \
+    -not -path "$TOPLEVEL/src/rust/registry/*" \
+    -exec shellcheck {} +
+
+# Check scripts that aren't named *.sh
+if [ -d "$TOPLEVEL/scripts/test" ]; then
+    shellcheck \
+        "$TOPLEVEL/scripts/test/cov-diff" \
+        "$TOPLEVEL/scripts/test/coverage"
+fi
+if [ -e \
+    "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert" \
+   ]; then
+    shellcheck \
+        "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert"
+fi
+if [ -e "$TOPLEVEL/contrib/client-tools/torify" ]; then
+    shellcheck "$TOPLEVEL/contrib/client-tools/torify"
+fi
+if [ -d "$TOPLEVEL/scripts/git" ]; then
+    shellcheck "$TOPLEVEL/scripts/git/"*.git-hook
+fi
diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl
index 633b47e314..857ce6f6f1 100755
--- a/scripts/maint/checkSpace.pl
+++ b/scripts/maint/checkSpace.pl
@@ -4,9 +4,16 @@ use strict;
 use warnings;
 
 my $found = 0;
+my $COLON_POS = 10;
+
 sub msg {
   $found = 1;
-  print "$_[0]";
+  my $v = shift;
+  $v =~ /^\s*([^:]+):(.*)$/;
+  chomp(my $errtype = $1);
+  my $rest = $2;
+  my $padding = ' ' x ($COLON_POS - length $errtype);
+  print "$padding$errtype:$rest\n";
 }
 
 my $C = 0;
@@ -16,8 +23,29 @@ if ($ARGV[0] =~ /^-/) {
     $C = ($lang eq '-C');
 }
 
+# hashmap of things where we allow spaces between them and (.
+our %allow_space_after= map {$_, 1} qw{
+    if while for switch return int unsigned elsif WINAPI
+    void __attribute__ op size_t double uint64_t
+    bool ssize_t
+    workqueue_reply_t hs_desc_decode_status_t
+    PRStatus
+    SMARTLIST_FOREACH_BEGIN SMARTLIST_FOREACH_END
+    HT_FOREACH
+    DIGESTMAP_FOREACH_MODIFY DIGESTMAP_FOREACH
+    DIGEST256MAP_FOREACH_MODIFY DIGEST256MAP_FOREACH
+    STRMAP_FOREACH_MODIFY STRMAP_FOREACH
+    SDMAP_FOREACH EIMAP_FOREACH RIMAP_FOREACH
+    MAP_FOREACH_MODIFY MAP_FOREACH
+    TOR_SIMPLEQ_FOREACH TOR_SIMPLEQ_FOREACH_SAFE
+    TOR_LIST_FOREACH TOR_LIST_FOREACH_SAFE
+    TOR_SLIST_FOREACH TOR_SLIST_FOREACH_SAFE
+};
+
 our %basenames = ();
 
+our %guardnames = ();
+
 for my $fn (@ARGV) {
     open(F, "$fn");
     my $lastnil = 0;
@@ -27,27 +55,31 @@ for my $fn (@ARGV) {
     my $basename = $fn;
     $basename =~ s#.*/##;
     if ($basenames{$basename}) {
-        msg "Duplicate fnames: $fn and $basenames{$basename}.\n";
+        msg "dup fname:$fn (same as $basenames{$basename}).\n";
     } else {
         $basenames{$basename} = $fn;
     }
+    my $isheader = ($fn =~ /\.h/);
+    my $seenguard = 0;
+    my $guardname = "<none>";
+
     while (<F>) {
         ## Warn about windows-style newlines.
         #    (We insist on lines that end with a single LF character, not
         #    CR LF.)
         if (/\r/) {
-            msg "       CR:$fn:$.\n";
+            msg "CR:$fn:$.\n";
         }
         ## Warn about tabs.
         #    (We only use spaces)
         if (/\t/) {
-            msg "      TAB:$fn:$.\n";
+            msg "TAB:$fn:$.\n";
         }
         ## Warn about labels that don't have a space in front of them
         #    (We indent every label at least one space)
-        if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) {
-            msg "nosplabel:$fn:$.\n";
-        }
+        #if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) {
+        #    msg "nosplabel:$fn:$.\n";
+        #}
         ## Warn about trailing whitespace.
         #    (We don't allow whitespace at the end of the line; make your
         #    editor highlight it for you so you can stop adding it in.)
@@ -57,12 +89,12 @@ for my $fn (@ARGV) {
         ## Warn about control keywords without following space.
         #    (We put a space after every 'if', 'while', 'for', 'switch', etc)
         if ($C && /\s(?:if|while|for|switch)\(/) {
-            msg "      KW(:$fn:$.\n";
+            msg "KW(:$fn:$.\n";
         }
         ## Warn about #else #if instead of #elif.
         #    (We only allow #elif)
         if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) {
-            msg " #else#if:$fn:$.\n";
+            msg "#else#if:$fn:$.\n";
         }
         ## Warn about some K&R violations
         #    (We use K&R-style C, where open braces go on the same line as
@@ -77,19 +109,19 @@ for my $fn (@ARGV) {
             msg "non-K&R {:$fn:$.\n";
         }
         if (/^\s*else/ and $lastline =~ /\}$/) {
-            msg "  }\\nelse:$fn:$.\n";
+            msg "}\\nelse:$fn:$.\n";
         }
         $lastline = $_;
         ## Warn about unnecessary empty lines.
         #   (Don't put an empty line before a line that contains nothing
         #   but a closing brace.)
         if ($lastnil && /^\s*}\n/) {
-            msg "  UnnecNL:$fn:$.\n";
+            msg "UnnecNL:$fn:$.\n";
         }
         ## Warn about multiple empty lines.
         #   (At most one blank line in a row.)
         if ($lastnil && /^$/) {
-            msg " DoubleNL:$fn:$.\n";
+            msg "DoubleNL:$fn:$.\n";
         } elsif (/^$/) {
             $lastnil = 1;
         } else {
@@ -98,8 +130,8 @@ for my $fn (@ARGV) {
         ## Terminals are still 80 columns wide in my world.  I refuse to
         ## accept double-line lines.
         #   (Don't make lines wider than 80 characters, including newline.)
-        if (/^.{80}/) {
-            msg "     Wide:$fn:$.\n";
+        if (/^.{80}/ and not /LCOV_EXCL/) {
+            msg "Wide:$fn:$.\n";
         }
         ### Juju to skip over comments and strings, since the tests
         ### we're about to do are okay there.
@@ -112,6 +144,23 @@ for my $fn (@ARGV) {
                     next;
                 }
             }
+
+            if ($isheader) {
+                if ($seenguard == 0) {
+                    if (/^\s*\#\s*ifndef\s+(\S+)/) {
+                        ++$seenguard;
+                        $guardname = $1;
+                    }
+                } elsif ($seenguard == 1) {
+                    if (/^\s*\#\s*define (\S+)/) {
+                        ++$seenguard;
+                        if ($1 ne $guardname) {
+                            msg "GUARD:$fn:$.: Header guard macro mismatch.\n";
+                        }
+                    }
+                }
+            }
+
             if (m!/\*.*?\*/!) {
                 s!\s*/\*.*?\*/!!;
             } elsif (m!/\*!) {
@@ -121,41 +170,34 @@ for my $fn (@ARGV) {
             }
             s!"(?:[^\"]+|\\.)*"!"X"!g;
             next if /^\#/;
-            ## Warn about C++-style comments.
-            #   (Use C style comments only.)
+            ## Skip C++-style comments.
             if (m!//!) {
-                #    msg "       //:$fn:$.\n";
+                #    msg "//:$fn:$.\n";
                 s!//.*!!;
             }
-            ## Warn about unquoted braces preceded by non-space.
-            #   (No character except a space should come before a {)
-            if (/([^\s'])\{/) {
-                msg "       $1\{:$fn:$.\n";
+            ## Warn about unquoted braces preceded by unexpected character.
+            if (/([^\s'\)\(\{])\{/) {
+                msg "$1\{:$fn:$.\n";
             }
             ## Warn about double semi-colons at the end of a line.
             if (/;;$/) {
-                msg "       double semi-colons at the end of $. in $fn\n"
+                msg ";;:$fn:$.\n"
             }
             ## Warn about multiple internal spaces.
             #if (/[^\s,:]\s{2,}[^\s\\=]/) {
-            #    msg "     X  X:$fn:$.\n";
+            #    msg "X  X:$fn:$.\n";
             #}
             ## Warn about { with stuff after.
             #s/\s+$//;
             #if (/\{[^\}\\]+$/) {
-            #    msg "     {X:$fn:$.\n";
+            #    msg "{X:$fn:$.\n";
             #}
             ## Warn about function calls with space before parens.
             #   (Don't put a space between the name of a function and its
             #   arguments.)
             if (/(\w+)\s\(([A-Z]*)/) {
-                if ($1 ne "if" and $1 ne "while" and $1 ne "for" and
-                    $1 ne "switch" and $1 ne "return" and $1 ne "int" and
-                    $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and
-                    $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and
-                    $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and
-                    $1 ne "workqueue_reply_t") {
-                    msg "     fn ():$fn:$.\n";
+                if (! $allow_space_after{$1} && $2 ne 'WINAPI') {
+                    msg "fn ():$fn:$.\n";
                 }
             }
             ## Warn about functions not declared at start of line.
@@ -165,8 +207,8 @@ for my $fn (@ARGV) {
             if ($in_func_head ||
                 ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ &&
                  ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ &&
-                 ! /= *\{$/ && ! /;$/)) {
-                if (/.\{$/){
+                 ! /= *\{$/ && ! /;$/) && ! /^[a-zA-Z0-9_]+\s*:/) {
+                if (/[^,\s]\s*\{$/){
                     msg "fn() {:$fn:$.\n";
                     $in_func_head = 0;
                 } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) {
@@ -184,23 +226,32 @@ for my $fn (@ARGV) {
             ## Check for forbidden functions except when they are
             # explicitly permitted
             if (/\bassert\(/ && not /assert OK/) {
-                msg "assert :$fn:$.   (use tor_assert)\n";
+                msg "assert:$fn:$.   (use tor_assert)\n";
             }
             if (/\bmemcmp\(/ && not /memcmp OK/) {
-                msg "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
+                msg "memcmp:$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
             }
             # always forbidden.
             if (not /\ OVERRIDE\ /) {
                 if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
-                    msg "$& :$fn:$.\n";
+                    msg "$&:$fn:$.\n";
                 }
                 if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
                     /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
-                    msg "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
+                    msg "$&:$fn:$.    (use tor_malloc, tor_free, etc)\n";
                 }
             }
         }
     }
+    if ($isheader && $C) {
+        if ($seenguard < 2) {
+            msg "noguard:$fn (No #ifndef/#define header guard pair found)\n";
+        } elsif ($guardnames{$guardname}) {
+            msg "dupguard:$fn (Guard macro $guardname also used in $guardnames{$guardname})\n";
+        } else {
+            $guardnames{$guardname} = $fn;
+        }
+    }
     close(F);
 }
 
diff --git a/scripts/maint/checkSpaceTest.sh b/scripts/maint/checkSpaceTest.sh
new file mode 100755
index 0000000000..e1d207a1a8
--- /dev/null
+++ b/scripts/maint/checkSpaceTest.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+# Copyright 2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+# Integration test for checkSpace.pl, which we want to rewrite.
+
+umask 077
+set -e
+
+# Skip this test if we're running on Windows; we expect line-ending
+# issues in that case.
+case "$(uname -s)" in
+    CYGWIN*) WINDOWS=1;;
+    MINGW*) WINDOWS=1;;
+    MSYS*) WINDOWS=1;;
+    *) WINDOWS=0;;
+esac
+if test "$WINDOWS" = 1; then
+    # This magic value tells automake that the test has been skipped.
+    exit 77
+fi
+
+# make a safe space for temporary files
+DATA_DIR=$(mktemp -d -t tor_checkspace_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+RECEIVED_FNAME="${DATA_DIR}/got.txt"
+
+cd "$(dirname "$0")/checkspace_tests"
+
+# we expect this to give an error code.
+../checkSpace.pl -C ./*.[ch] ./*/*.[ch] > "${RECEIVED_FNAME}" && exit 1
+
+diff -u expected.txt "${RECEIVED_FNAME}" || exit 1
+
+echo "OK"
diff --git a/scripts/maint/checkspace_tests/dubious.c b/scripts/maint/checkspace_tests/dubious.c
new file mode 100644
index 0000000000..59c5f8e4fe
--- /dev/null
+++ b/scripts/maint/checkspace_tests/dubious.c
@@ -0,0 +1,83 @@
+
+// The { coming up should be on its own line.
+int
+foo(void) {
+  // There should be a space before (1)
+  if(1) x += 1;
+
+  // The following empty line is unnecessary.
+
+}
+
+
+// There should be a newline between void and bar.
+void bar(void)
+{
+  // too wide:
+  testing("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+}
+
+long
+bad_spacing()
+{
+  // here comes a tab
+	return 2;
+  // here comes a label without space:
+foo:
+  ;
+}
+
+// Here comes a CR:

+
+// Trailing space:         
+
+int
+non_k_and_r(void)
+{
+  // non-k&r
+  if (foo)
+    {
+      // double-semi
+      return 1;;
+    }
+  else
+    {
+      return 2;
+    }
+}
+
+// #else #if causes a warning.
+#if 1
+#else
+#if 2
+#else
+#endif
+#endif
+
+// always space before a brace.
+foo{
+}
+
+void
+unexpected_space(void)
+{
+  // This space gives a warning.
+  foobar (77);
+}
+
+void
+bad_function_calls(long)
+{
+  // These are forbidden:
+  assert(1);
+  memcmp("a","b",1);
+  strcat(foo,x);
+  strcpy(foo,y);
+  sprintf(foo,"x");
+  malloc(7);
+  free(p);
+  realloc(p);
+  strdup(s);
+  strndup(s,10);
+  calloc(a,b);
+}
diff --git a/scripts/maint/checkspace_tests/dubious.h b/scripts/maint/checkspace_tests/dubious.h
new file mode 100644
index 0000000000..744ec33955
--- /dev/null
+++ b/scripts/maint/checkspace_tests/dubious.h
@@ -0,0 +1,4 @@
+
+// no guards.
+
+int foo(int);
diff --git a/scripts/maint/checkspace_tests/expected.txt b/scripts/maint/checkspace_tests/expected.txt
new file mode 100644
index 0000000000..38595ed373
--- /dev/null
+++ b/scripts/maint/checkspace_tests/expected.txt
@@ -0,0 +1,30 @@
+    fn() {:./dubious.c:4
+       KW(:./dubious.c:6
+   UnnecNL:./dubious.c:10
+  DoubleNL:./dubious.c:12
+   tp fn():./dubious.c:15
+      Wide:./dubious.c:17
+       TAB:./dubious.c:24
+        CR:./dubious.c:30
+ Space@EOL:./dubious.c:32
+ non-K&R {:./dubious.c:39
+        ;;:./dubious.c:41
+   }\nelse:./dubious.c:43
+  #else#if:./dubious.c:52
+        o{:./dubious.c:58
+    fn() {:./dubious.c:58
+     fn ():./dubious.c:65
+    assert:./dubious.c:72   (use tor_assert)
+    memcmp:./dubious.c:73   (use {tor,fast}_mem{eq,neq,cmp}
+   strcat(:./dubious.c:74
+   strcpy(:./dubious.c:75
+  sprintf(:./dubious.c:76
+   malloc(:./dubious.c:77    (use tor_malloc, tor_free, etc)
+     free(:./dubious.c:78    (use tor_malloc, tor_free, etc)
+  realloc(:./dubious.c:79    (use tor_malloc, tor_free, etc)
+   strdup(:./dubious.c:80    (use tor_malloc, tor_free, etc)
+  strndup(:./dubious.c:81    (use tor_malloc, tor_free, etc)
+   calloc(:./dubious.c:82    (use tor_malloc, tor_free, etc)
+   noguard:./dubious.h (No #ifndef/#define header guard pair found)
+  dupguard:./same_guard.h (Guard macro GUARD_MACRO_H also used in ./good_guard.h)
+ dup fname:./subdir/dubious.c (same as ./dubious.c).
diff --git a/scripts/maint/checkspace_tests/good_guard.h b/scripts/maint/checkspace_tests/good_guard.h
new file mode 100644
index 0000000000..b792912d90
--- /dev/null
+++ b/scripts/maint/checkspace_tests/good_guard.h
@@ -0,0 +1,6 @@
+#ifndef GUARD_MACRO_H
+#define GUARD_MACRO_H
+
+int bar(void);
+
+#endif
diff --git a/scripts/maint/checkspace_tests/same_guard.h b/scripts/maint/checkspace_tests/same_guard.h
new file mode 100644
index 0000000000..b792912d90
--- /dev/null
+++ b/scripts/maint/checkspace_tests/same_guard.h
@@ -0,0 +1,6 @@
+#ifndef GUARD_MACRO_H
+#define GUARD_MACRO_H
+
+int bar(void);
+
+#endif
diff --git a/scripts/maint/checkspace_tests/subdir/dubious.c b/scripts/maint/checkspace_tests/subdir/dubious.c
new file mode 100644
index 0000000000..7f22bf79bf
--- /dev/null
+++ b/scripts/maint/checkspace_tests/subdir/dubious.c
@@ -0,0 +1 @@
+// Nothing wrong with this file, but the name is a duplicate.
diff --git a/scripts/maint/clang-format.sh b/scripts/maint/clang-format.sh
new file mode 100755
index 0000000000..59832117b4
--- /dev/null
+++ b/scripts/maint/clang-format.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+# Copyright 2020, The Tor Project, Inc.
+# See LICENSE for licensing information.
+
+#
+# DO NOT COMMIT OR MERGE CODE THAT IS RUN THROUGH THIS TOOL YET.
+#
+# WE ARE STILL DISCUSSING OUR DESIRED STYLE AND ITERATING ON IT.
+#     (12 Feb 2020)
+#
+
+# This script runs "clang-format" and "codetool" in sequence over each of
+# our source files, and replaces the original file only if it has changed.
+#
+# We can't just use clang-format -i, since we also want to use codetool to
+# reformat a few things back to how we want them, and we want avoid changing
+# the mtime on files that didn't actually change.
+
+set -e
+
+cd "$(dirname "$0")/../../src/"
+
+# Shellcheck complains that a for loop over find's output is unreliable,
+# since there might be special characters in the output.  But we happen
+# to know that none of our C files have special characters or spaces in
+# their names, so this is safe.
+#
+# shellcheck disable=SC2044
+for fname in $(find lib core feature app test tools -name '[^.]*.[ch]'); do
+    tmpfname="${fname}.clang_fmt.tmp"
+    rm -f "${tmpfname}"
+    clang-format --style=file "${fname}" > "${tmpfname}"
+    ../scripts/maint/codetool.py "${tmpfname}"
+    if cmp "${fname}" "${tmpfname}" >/dev/null 2>&1; then
+        echo "No change in ${fname}"
+        rm -f "${tmpfname}"
+    else
+        echo "Change in ${fname}"
+        mv "${tmpfname}" "${fname}"
+    fi
+done
diff --git a/scripts/maint/codetool.py b/scripts/maint/codetool.py
new file mode 100755
index 0000000000..725712c0cc
--- /dev/null
+++ b/scripts/maint/codetool.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020, The Tor Project, Inc.
+# See LICENSE for licensing information.
+
+#
+# DO NOT COMMIT OR MERGE CODE THAT IS RUN THROUGH THIS TOOL YET.
+#
+# WE ARE STILL DISCUSSING OUR DESIRED STYLE AND ITERATING ON IT,
+# ALONG WITH THE TOOLS THAT ACHIEVE IT.
+#     (12 Feb 2020)
+#
+
+"""
+   This program uses a set of plugable filters to inspect and transform
+   our C code.
+"""
+
+import os
+import re
+import sys
+
+class Filter:
+    """A Filter transforms a string containing a C program."""
+    def __init__(self):
+        pass
+
+    def transform(self, s):
+        return s
+
+class CompoundFilt(Filter):
+    """A CompoundFilt runs another set of filters, in sequence."""
+    def __init__(self, items=()):
+        super().__init__()
+        self._filters = list(items)
+
+    def add(self, filt):
+        self._filters.append(filt)
+        return self
+
+    def transform(self, s):
+        for f in self._filters:
+            s = f.transform(s)
+
+        return s
+
+class SplitError(Exception):
+    """Exception: raised if split_comments() can't understand a C file."""
+    pass
+
+def split_comments(s):
+    r"""Iterate over the C code in 's', and yield a sequence of (code,
+       comment) pairs.  Each pair will contain either a nonempty piece
+       of code, a nonempty comment, or both.
+
+       >>> list(split_comments("hello // world\n"))
+       [('hello ', '// world'), ('\n', '')]
+
+       >>> list(split_comments("a /* b cd */ efg // hi"))
+       [('a ', '/* b cd */'), (' efg ', '// hi')]
+    """
+
+    # Matches a block of code without any comments.
+    PAT_CODE = re.compile(r'''^(?: [^/"']+ |
+                                   "(?:[^\\"]+|\\.)*" |
+                                   '(?:[^\\']+|\\.)*' |
+                                   /[^/*]
+                               )*''', re.VERBOSE|re.DOTALL)
+
+    # Matches a C99 "//" comment.
+    PAT_C99_COMMENT = re.compile(r'^//.*$', re.MULTILINE)
+
+    # Matches a C "/*  */" comment.
+    PAT_C_COMMENT = re.compile(r'^/\*(?:[^*]|\*+[^*/])*\*+/', re.DOTALL)
+
+    while True:
+        # Find some non-comment code at the start of the string.
+        m = PAT_CODE.match(s)
+
+        # If we found some code here, save it and advance the string.
+        # Otherwise set 'code' to "".
+        if m:
+            code = m.group(0)
+            s = s[m.end():]
+        else:
+            code = ""
+
+        # Now we have a comment, or the end of the string.  Find out which
+        # one, and how long it is.
+        if s.startswith("//"):
+            m = PAT_C99_COMMENT.match(s)
+        else:
+            m = PAT_C_COMMENT.match(s)
+
+        # If we got a comment, save it and advance the string.  Otherwise
+        # set 'comment' to "".
+        if m:
+            comment = m.group(0)
+            s = s[m.end():]
+        else:
+            comment = ""
+
+        # If we found no code and no comment, we should be at the end of
+        # the string...
+        if code == "" and comment == "":
+            if s:
+                # But in case we *aren't* at the end of the string, raise
+                # an error.
+                raise SplitError()
+            # ... all is well, we're done scanning the code.
+            return
+
+        yield (code, comment)
+
+class IgnoreCommentsFilt(Filter):
+    """Wrapper: applies another filter to C code only, excluding comments.
+    """
+    def __init__(self, filt):
+        super().__init__()
+        self._filt = filt
+
+    def transform(self, s):
+        result = []
+        for code, comment in split_comments(s):
+            result.append(self._filt.transform(code))
+            result.append(comment)
+        return "".join(result)
+
+
+class RegexFilt(Filter):
+    """A regex filter applies a regular expression to some C code."""
+    def __init__(self, pat, replacement, flags=0):
+        super().__init__()
+        self._pat = re.compile(pat, flags)
+        self._replacement = replacement
+
+    def transform(self, s):
+        s, _ = self._pat.subn(self._replacement, s)
+        return s
+
+def revise(fname, filt):
+    """Run 'filt' on the contents of the file in 'fname'.  If any
+       changes are made, then replace the file with its new contents.
+       Otherwise, leave the file alone.
+    """
+    contents = open(fname, 'r').read()
+    result = filt.transform(contents)
+    if result == contents:
+        return
+
+    tmpname = "{}_codetool_tmp".format(fname)
+    try:
+        with open(tmpname, 'w') as f:
+            f.write(result)
+            os.rename(tmpname, fname)
+    except:
+        os.unlink(tmpname)
+        raise
+
+##############################
+# Filtering rules.
+##############################
+
+# Make sure that there is a newline after the first comma in a MOCK_IMPL()
+BREAK_MOCK_IMPL = RegexFilt(
+    r'^MOCK_IMPL\(([^,]+),\s*(\S+)',
+    r'MOCK_IMPL(\1,\n\2',
+    re.MULTILINE)
+
+# Make sure there is no newline between } and a loop iteration terminator.
+RESTORE_SMARTLIST_END = RegexFilt(
+    r'}\s*(SMARTLIST|DIGESTMAP|DIGEST256MAP|STRMAP|MAP)_FOREACH_END\s*\(',
+    r'} \1_FOREACH_END (',
+    re.MULTILINE)
+
+F = CompoundFilt()
+F.add(IgnoreCommentsFilt(CompoundFilt([
+    RESTORE_SMARTLIST_END,
+    BREAK_MOCK_IMPL])))
+
+if __name__ == '__main__':
+    for fname in sys.argv[1:]:
+        revise(fname, F)
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
deleted file mode 100644
index 79551948c6..0000000000
--- a/scripts/maint/fallback.whitelist
+++ /dev/null
@@ -1,997 +0,0 @@
-# updateFallbackDirs.py directory mirror whitelist
-#
-# Format:
-# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
-# or use:
-# scripts/maint/generateFallbackDirLine.py fingerprint ...
-#
-# All attributes must match for the directory mirror to be included.
-# If the fallback has an ipv6 key, the whitelist line must also have
-# it, and vice versa, otherwise they don't match.
-# (The blacklist overrides the whitelist.)
-
-# To replace this list with the hard-coded fallback list (for testing), use
-# a command similar to:
-#   cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \
-#   tr -d '\n' | \
-#   sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \
-#   > scripts/maint/fallback.whitelist
-#
-# When testing before a release, exclusions due to changed details will result
-# in a warning, unless the IPv4 address or port change happened recently.
-# Then it is only logged at info level, as part of the eligibility check.
-# Exclusions due to stability also are only shown at info level.
-#
-# Add the number of selected, slow, and excluded relays, and compare that to
-# the number of hard-coded relays. If it's less, use info-level logs to find
-# out why each of the missing relays was excluded.
-
-# If a relay operator wants their relay to be a FallbackDir,
-# enter the following information here:
-# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html
-# https://trac.torproject.org/projects/tor/ticket/22321#comment:22
-78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE ipv6=[2a01:4f8:120:4023::110]:80 # fluxe3
-131.188.40.188:1443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 ipv6=[2001:638:a000:4140::ffff:188]:80 # fluxe4
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html
-5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
-212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6
-212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008372.html
-# IPv6 tunnel available on request (is this a good idea?)
-108.53.208.157:80 orport=443 id=4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html
-167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html
-144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
-# Email sent directly to teor, verified using relay contact info
-91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
-91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html
-# Sent additional emails to teor with updated relays
-81.7.11.96:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 # Doedel22
-# 9F5068310818ED7C70B0BC4087AB55CB12CB4377 not found in current consensus
-178.254.19.101:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE # Freebird31
-178.254.19.101:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 # Freebird32
-81.7.14.253:9001 orport=443 id=1AE039EE0B11DB79E4B4B29CBA9F752864A0259E # Ichotolot60
-81.7.11.186:1080 orport=443 id=B86137AE9681701901C6720E55C16805B46BD8E3 # BeastieJoy60
-85.25.213.211:465 orport=80 id=CE47F0356D86CF0A1A2008D97623216D560FB0A8 # BeastieJoy61
-85.25.159.65:995 orport=80 id=52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9 # BeastieJoy63
-81.7.3.67:993 orport=443 id=A2E6BB5C391CD46B38C55B4329C35304540771F1 # BeastieJoy62
-81.7.14.31:9001 orport=443 id=7600680249A22080ECC6173FBBF64D6FCF330A61 # Ichotolot62
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html
-51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008383.html
-81.7.14.246:80 orport=443 id=CE75BF0972ADD52AF8807602374E495C815DB304 ipv6=[2a02:180:a:51::dead]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html
-# Sent additional email to teor with fingerprint change
-149.202.98.161:80 orport=443 id=FC64CD763F8C1A319BFBBF62551684F4E1E42332 ipv6=[2001:41d0:8:4528::161]:443
-193.111.136.162:80 orport=443 id=C79552275DFCD486B942510EF663ED36ACA1A84B ipv6=[2001:4ba0:cafe:10d0::1]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008416.html
-185.100.84.212:80 orport=443 id=330CD3DB6AD266DC70CDB512B036957D03D9BC59 ipv6=[2a06:1700:0:7::1]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008417.html
-178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443
-178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008513.html
-178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10 ipv6=[2a03:b0c0:0:1010::a4:b001]:9001
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008534.html
-5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB ipv6=[2a01:4f8:162:51e2::2]:9001
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
-178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
-
-# Email sent directly to teor, verified using relay contact info
-94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
-
-# Email sent directly to teor, verified using relay contact info
-171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
-171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
-171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443
-# same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
-171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
-
-# Email sent directly to teor, verified using relay contact info
-212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001
-93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D
-46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE # TykRelay01
-185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE # TykRelay051
-# This fallback opted-in in previous releases, then changed its details,
-# and so we blacklisted it. Now we want to whitelist changes.
-# Assume details update is permanent
-185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 # TykRelay06
-
-# Email sent directly to teor, verified using relay contact info
-185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778
-
-# Email sent directly to teor, verified using relay contact info
-178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443
-46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7
-178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443
-178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443
-178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443
-81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443
-209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443
-81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443
-46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1
-178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443"
-46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7
-178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443
-178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443
-178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443
-81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443
-209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443
-81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443
-46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1
-178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443
-178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029 ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443
-178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443
-178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443
-
-# Email sent directly to teor, verified using relay contact info
-5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651
-
-# Email sent directly to teor, verified using relay contact info
-109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52
-109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C
-109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24
-109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18
-109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281
-109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9
-62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
-# Assume details update is permanent
-77.247.181.166:80 orport=443 id=77131D7E2EC1CA9B8D737502256DA9103599CE51 # CriticalMass
-77.247.181.164:80 orport=443 id=204DFD2A2C6A0DC1FA0EACB495218E0B661704FD # HaveHeart
-77.247.181.162:80 orport=443 id=7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55 # sofia
-
-# https://twitter.com/biotimylated/status/718994247500718080
-212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0
-
-# Email sent directly to teor, verified using relay contact info
-46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E
-
-# Email sent directly to teor, verified using relay contact info
-94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004
-176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003
-176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81
-94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003
-85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004
-85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003
-
-# Email sent directly to teor, verified using relay contact info
-148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8 ipv6=[2a01:4f8:211:c68::2]:9010
-
-# Email sent directly to teor, verified using relay contact info
-5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E ipv6=[2001:1af8:4700:a012:1::1]:443
-5.79.68.161:9030 orport=9001 id=B7EC0C02D7D9F1E31B0C251A6B058880778A0CD1 ipv6=[2001:1af8:4700:a012:1::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B ipv6=[2001:bc8:338c::1]:9001
-62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF ipv6=[2001:bc8:338c::1]:9101
-
-# Email sent directly to teor, verified using relay contact info
-188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF
-
-# Message sent directly to teor, verified using relay contact info
-95.215.44.110:80 orport=443 id=D56AA4A1AA71961F5279FB70A6DCF7AD7B993EB5
-95.215.44.122:80 orport=443 id=998D8FE06B867AA3F8D257A7D28FFF16964D53E2
-95.215.44.111:80 orport=443 id=A7C7FD510B20BC8BE8F2A1D911364E1A23FBD09F
-
-# Email sent directly to teor, verified using relay contact info
-86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0
-86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B
-
-# Email sent directly to teor, verified using relay contact info
-193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001
-109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F ipv6=[2001:948:7:2::163]:5001
-
-# Email sent directly to teor, verified using relay contact info
-146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E
-
-# Email sent directly to teor, verified using relay contact info
-46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D
-
-# Email sent directly to teor, verified using relay contact info
-89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C
-
-# Email sent directly to teor
-# Assume details update is permanent
-212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 # Jans
-
-# Email sent directly to teor, verified using relay contact info
-46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB
-
-# Email sent directly to teor, verified using relay contact info
-193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-197.231.221.211:9030 orport=443 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 # IPredator
-
-# Email sent directly to teor, verified using relay contact info
-185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C
-
-# Email sent directly to teor, verified using relay contact info
-193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB
-193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89 ipv6=[2001:6b0:30:1000::99]:9050
-193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E
-
-# Email sent directly to teor, verified using relay contact info
-138.201.250.33:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
-
-# Email sent directly to teor, verified using relay contact info
-37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811
-
-# Email sent directly to teor, verified using relay contact info
-91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5
-
-# Email sent directly to teor, verified using relay contact info
-185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0 ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443
-
-# Email sent directly to teor, verified using relay contact info
-193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4
-
-# Email sent directly to teor, verified using relay contact info
-134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE
-
-# Email sent directly to teor, verified using relay contact info
-173.212.254.192:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
-
-# Email sent directly to teor, verified using relay contact info
-178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050
-188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
-198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050
-159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050
-
-# Email sent directly to teor, verified using relay contact info
-5.196.31.80:9030 orport=9900 id=DFB2EB472643FAFCD5E73D2E37D51DB67203A695 ipv6=[2001:41d0:52:400::a65]:9900
-
-# Email sent directly to teor, verified using relay contact info
-188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788
-
-# Email sent directly to teor, verified using relay contact info
-213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22
-213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4
-213.61.66.115:9034 orport=9004 id=480CCC94CEA04D2DEABC0D7373868E245D4C2AE2
-213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690
-
-# Email sent directly to teor, verified using relay contact info
-136.243.187.165:9030 orport=443 id=1AC65257D7BFDE7341046625470809693A8ED83E
-
-# Email sent directly to teor, verified using relay contact info
-212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97
-
-# Email sent directly to teor, verified using relay contact info
-192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023
-192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244
-51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A
-51.254.215.129:80 orport=443 id=7B4491D05144B20AE8519AE784B94F0525A8BB79
-192.99.59.139:80 orport=443 id=82EC878ADA7C205146B9F5193A7310867FAA0D7B
-51.254.215.124:80 orport=443 id=98999EBE89B5FA9AA0C58421F0B46C3D0AF51CBA
-51.254.214.208:80 orport=443 id=C3F0D1417848EAFC41277A73DEB4A9F2AEC23DDF
-192.99.59.141:80 orport=443 id=F45426551795B9DA78BEDB05CD5F2EACED8132E4
-192.99.59.14:80 orport=443 id=161A1B29A37EBF096D2F8A9B1E176D6487FE42AE
-
-# Email sent directly to teor, verified using relay contact info
-151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443
-
-# Email sent directly to teor, verified using relay contact info
-176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1
-176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683
-195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F
-195.154.9.161:80 orport=443 id=B6295A9960F89BD0C743EEBC5670450EA6A34685
-46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7
-
-# Email sent directly to teor, verified using relay contact info
-37.187.102.108:80 orport=443 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 ipv6=[2001:41d0:a:266c::1]:443 # EvilMoe
-212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C # EvilMoe
-
-# Email sent directly to teor, verified using relay contact info
-212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
-
-# Email sent directly to teor
-37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 # thanatosCZ
-5.189.169.190:8030 orport=8080 id=8D79F73DCD91FC4F5017422FAC70074D6DB8DD81 # thanatosDE
-
-# Email sent directly to teor, verified using relay contact info
-37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443
-
-# Email sent directly to teor, verified using relay contact info
-185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529
-
-# Email sent directly to teor, verified using relay contact info
-185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443
-185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956
-185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835
-185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21
-
-# Email sent directly to teor, verified using relay contact info
-37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
-
-# Email sent directly to teor, verified using relay contact info
-212.83.154.33:8888 orport=443 id=3C79699D4FBC37DE1A212D5033B56DAE079AC0EF
-212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B
-
-# Email sent directly to teor, verified using relay contact info
-51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A
-
-# Email sent directly to teor, verified using relay contact info
-78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74
-
-# Email sent directly to teor, verified using relay contact info
-195.154.97.91:80 orport=443 id=BD33C50D50DCA2A46AAED54CA319A1EFEBF5D714
-
-# Email sent directly to teor, verified using relay contact info
-62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8
-
-# Email sent directly to teor, verified using relay contact info
-5.196.74.215:9030 orport=9001 id=5818055DFBAF0FA7F67E8125FD63E3E7F88E28F6
-
-# Email sent directly to teor, verified using relay contact info
-212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20
-
-# Email sent directly to teor, verified using relay contact info
-85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0
-
-# Email sent directly to teor, verified using relay contact info
-46.166.170.4:80 orport=443 id=19F42DB047B72C7507F939F5AEA5CD1FA4656205
-46.166.170.5:80 orport=443 id=DA705AD4591E7B4708FA2CAC3D53E81962F3E6F6
-
-# Email sent directly to teor, verified using relay contact info
-5.189.157.56:80 orport=443 id=77F6D6A6B6EAFB8F5DADDC07A918BBF378ED6725
-
-# Email sent directly to teor, verified using relay contact info
-46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E
-185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D
-95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF
-
-# Email sent directly to teor, verified using relay contact info
-212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7
-
-# Email sent directly to teor, verified using relay contact info
-91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7
-
-# Email sent directly to teor, verified using relay contact info
-46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8
-178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794
-
-# Email sent directly to teor, verified using relay contact info
-178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163
-
-# Email sent directly to teor, verified using relay contact info
-82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98 ipv6=[2001:470:53e0::cafe]:9050
-
-# Email sent directly to teor, verified using relay contact info
-146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D
-
-# Email sent directly to teor, verified using relay contact info
-37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400 ipv6=[2001:41d0:a:1657::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7
-
-# Email sent directly to teor, verified using relay contact info
-212.47.250.243:9030 orport=9001 id=5B33EDBAEA92F446768B3753549F3B813836D477
-# Confirm with operator before adding these
-#163.172.133.36:9030 orport=9001 id=D8C2BD36F01FA86F4401848A0928C4CB7E5FDFF9
-#158.69.216.70:9030 orport=9001 id=0ACE25A978D4422C742D6BC6345896719BF6A7EB
-
-# Email sent directly to teor, verified using relay contact info
-5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265
-
-# Email sent directly to teor
-188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy
-
-# Email sent directly to teor, verified using relay contact info
-46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D
-
-# Email sent directly to teor, verified using relay contact info
-144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC
-
-# Email sent directly to teor, verified using relay contact info
-46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
-46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
-46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
-163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001
-
-# Email sent directly to teor, verified using relay contact info
-176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
-
-# Email sent directly to teor, verified using relay contact info
-46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
-46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB
-46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76
-95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA
-
-# Email sent directly to teor, verified using relay contact info
-93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F
-
-# Email sent directly to teor, verified using relay contact info
-37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A
-
-# Email sent directly to teor, verified using relay contact info
-5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8
-
-# Email sent directly to teor, verified using relay contact info
-51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5
-
-# Email sent directly to teor, verified using relay contact info
-5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
-
-# Email sent directly to teor, verified using relay contact info
-# Updated details from atlas based on ticket #20010
-163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
-163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
-163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2
-
-# Email sent directly to teor, verified using relay contact info
-204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70
-
-# Email sent directly to teor, verified using relay contact info
-151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54
-
-# Email sent directly to teor, verified using relay contact info
-185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223
-
-# Email sent directly to teor, verified using relay contact info
-89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
-
-# Email sent directly to teor, verified using relay contact info
-92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C
-
-# Email sent directly to teor, verified using relay contact info
-80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001
-
-# Email sent directly to teor, verified using relay contact info
-83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443
-
-# Email sent directly to teor, verified using relay contact info
-95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A
-
-# Email sent directly to teor, verified using relay contact info
-128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001
-
-# Email sent directly to teor, verified using relay contact info
-178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2
-
-# Email sent directly to teor, verified using relay contact info
-212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887
-
-# Email sent directly to teor, verified using relay contact info
-204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28
-
-# Email sent directly to teor, verified using relay contact info
-80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B
-
-# Email sent directly to teor, verified using relay contact info
-185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088
-
-# Email sent directly to teor
-149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 ipv6=[2607:5300:201:3000::17d3]:9002 # PiotrTorpotkinOne
-
-# Email sent directly to teor, verified using relay contact info
-81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80
-
-# Email sent directly to teor, verified using relay contact info
-# IPv6 address unreliable
-195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993
-138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F
-81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993
-134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993
-46.228.199.19:80 orport=443 id=E26AFC5F718E21AC502899B20C653AEFF688B0D2 ipv6=[2001:4ba0:cafe:4a::1]:993
-37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996 ipv6=[2a00:1158:3::11a]:993
-46.23.70.195:80 orport=443 id=C9933B3725239B6FAB5227BA33B30BE7B48BB485
-185.15.244.124:80 orport=443 id=935BABE2564F82016C19AEF63C0C40B5753BA3D2 ipv6=[2001:4ba0:cafe:e35::1]:993
-195.154.116.232:80 orport=443 id=B35C5739C8C5AB72094EB2B05738FD1F8EEF6EBD ipv6=[2001:bc8:399f:200::1]:993
-195.154.121.198:80 orport=443 id=0C77421C890D16B6D201283A2244F43DF5BC89DD ipv6=[2001:bc8:399f:100::1]:993
-37.187.20.59:80 orport=443 id=91D23D8A539B83D2FB56AA67ECD4D75CC093AC55 ipv6=[2001:41d0:a:143b::1]:993
-217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993
-81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993
-46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995
-91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993
-163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F
-188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F
-81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993
-213.246.56.95:80 orport=443 id=27E6E8E19C46751E7312420723C6162FF3356A4C ipv6=[2a00:c70:1:213:246:56:95:1]:993
-94.198.100.18:80 orport=443 id=BAACCB29197DB833F107E410E2BFAE5009EE7583
-217.12.203.46:80 orport=443 id=6A29FD8C00D573E6C1D47852345B0E5275BA3307
-212.117.180.107:80 orport=443 id=0B454C7EBA58657B91133A587C1BDAEDC6E23142
-217.12.199.190:80 orport=443 id=A37C47B03FF31CA6937D3D68366B157997FE7BCD ipv6=[2a02:27a8:0:2::486]:993
-216.230.230.247:80 orport=443 id=4C7BF55B1BFF47993DFF995A2926C89C81E4F04A
-69.30.215.42:80 orport=443 id=510176C07005D47B23E6796F02C93241A29AA0E9 ipv6=[2604:4300:a:2e::2]:993
-89.46.100.162:80 orport=443 id=6B7191639E179965FD694612C9B2C8FB4267B27D
-107.181.174.22:80 orport=443 id=5A551BF2E46BF26CC50A983F7435CB749C752553 ipv6=[2607:f7a0:3:4::4e]:993
-
-# Email sent directly to teor, verified using relay contact info
-212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001
-
-# Email sent directly to teor
-176.158.236.102:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 # Underworld
-
-# Email sent directly to teor, verified using relay contact info
-91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F
-
-# Email sent directly to teor, verified using relay contact info
-80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443
-
-# Email sent directly to teor
-163.172.138.22:80 orport=443 id=16102E458460349EE45C0901DAA6C30094A9BBEA ipv6=[2001:bc8:4400:2100::1:3]:443 # mkultra
-
-# Email sent directly to teor, verified using relay contact info
-97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94
-
-# Email sent directly to teor, verified using relay contact info
-192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC
-
-# Email sent directly to teor, verified using relay contact info
-178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A
-
-# Email sent directly to teor, verified using relay contact info
-163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6
-
-# Email sent directly to teor, verified using relay contact info
-185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
-185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E
-
-# Email sent directly to teor, verified using relay contact info
-164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
-78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123
-
-# Email sent directly to teor, verified using relay contact info
-46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C ipv6=[2a03:b0c0:3:d0::208:5001]:9050
-178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050
-
-# Email sent directly to teor, verified using relay contact info
-# Very low bandwidth, stale consensues, excluded to cut down on warnings
-#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
-
-# Email sent directly to teor, verified using relay contact info
-167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-213.141.138.174:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 # Schakalium
-
-# Email sent directly to teor, verified using relay contact info
-95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443
-
-# Email sent directly to teor, verified using relay contact info
-166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB
-
-# Email sent directly to teor, verified using relay contact info
-91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC
-
-# Email sent directly to teor, verified using relay contact info
-178.217.184.32:8080 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D
-
-# Email sent directly to teor, verified using relay contact info
-85.10.201.47:9030 orport=9001 id=D8B7A3A6542AA54D0946B9DC0257C53B6C376679 ipv6=[2a01:4f8:a0:43eb::beef]:9001
-
-# Email sent directly to teor, verified using relay contact info
-120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62
-
-# Email sent directly to teor, verified using relay contact info
-37.153.1.10:9030 orport=9001 id=9772EFB535397C942C3AB8804FB35CFFAD012438
-
-# Email sent directly to teor, verified using relay contact info
-92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C
-
-# Email sent directly to teor, verified using relay contact info
-31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798
-
-# Email sent directly to teor
-192.160.102.169:80 orport=9001 id=C0192FF43E777250084175F4E59AC1BA2290CE38 ipv6=[2620:132:300c:c01d::9]:9002 # manipogo
-192.160.102.166:80 orport=9001 id=547DA56F6B88B6C596B3E3086803CDA4F0EF8F21 ipv6=[2620:132:300c:c01d::6]:9002 # chaucer
-192.160.102.170:80 orport=9001 id=557ACEC850F54EEE65839F83CACE2B0825BE811E ipv6=[2620:132:300c:c01d::a]:9002 # ogopogo
-192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2620:132:300c:c01d::4]:9002 # snowfall
-192.160.102.165:80 orport=9001 id=C90CA3B7FE01A146B8268D56977DC4A2C024B9EA ipv6=[2620:132:300c:c01d::5]:9002 # cowcat
-192.160.102.168:80 orport=9001 id=F6A358DD367B3282D6EF5824C9D45E1A19C7E815 ipv6=[2620:132:300c:c01d::8]:9002 # prawksi
-
-# Email sent directly to teor, verified using relay contact info
-136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664
-
-# Email sent directly to teor, verified using relay contact info
-192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
-192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
-
-# Email sent directly to teor, verified using relay contact info
-192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
-# same machine as ED2338CAC2711B3E331392E1ED2831219B794024
-192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
-
-# https://twitter.com/kosjoli/status/719507270904758272
-85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A
-176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911
-46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
-
-# Email sent directly to teor, verified using relay contact info
-185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
-
-# Email sent directly to teor, verified using relay contact info
-108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
-
-# Email sent directly to teor, verified using relay contact info
-91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6
-
-# Email sent directly to teor, verified using relay contact info
-185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260
-192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8
-128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D
-
-# Email sent directly to teor, verified using relay contact info
-185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361
-
-# Email sent directly to teor, verified using relay contact info
-128.204.39.106:9030 orport=9001 id=6F0F3C09AF9580F7606B34A7678238B3AF7A57B7
-
-# Email sent directly to teor, verified using relay contact info
-198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92
-
-# Email sent directly to teor, verified using relay contact info
-167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65 ipv6=[2607:5300:100::78d]:443
-
-# Email sent directly to teor, verified using relay contact info
-66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142
-66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A
-
-# Email sent directly to teor, verified using relay contact info
-92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82
-
-# Email sent directly to teor, verified using relay contact info
-212.47.228.115:9030 orport=443 id=BCA017ACDA48330D02BB70716639ED565493E36E
-
-# Email sent directly to teor, verified using relay contact info
-185.100.84.175:80 orport=443 id=39B59AF4FE54FAD8C5085FA9C15FDF23087250DB
-
-# Email sent directly to teor, verified using relay contact info
-166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812
-
-# Emails sent directly to teor, verified using relay contact info
-69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
-
-# Email sent directly to teor, verified using relay contact info
-213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik
-
-# Email sent directly to teor, verified using relay contact info
-88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001
-
-# Email sent directly to teor, verified using relay contact info
-185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
-46.36.36.127:9030 orport=9001 id=C80DF89B21FF932DEC0D7821F679B6C79E1449C3
-
-# Email sent directly to teor, verified using relay contact info
-176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80
-176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88
-176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C
-176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F
-
-# Email sent directly to teor, verified using relay contact info
-107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA
-
-# Email sent directly to teor
-193.70.112.165:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC # ParkBenchInd001
-
-# Email sent directly to teor
-185.220.101.6:10006 orport=20006 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A # niftyvolcanorabbit
-185.220.101.13:10013 orport=20013 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 # niftycottontail
-185.220.101.5:10005 orport=20005 id=1084200B44021D308EA4253F256794671B1D099A # niftyhedgehog
-185.220.101.9:10009 orport=20009 id=14877C6384A9E793F422C8D1DDA447CACA4F7C4B # niftywoodmouse
-185.220.101.8:10008 orport=20008 id=24E91955D969AEA1D80413C64FE106FAE7FD2EA9 # niftymouse
-185.220.101.1:10001 orport=20001 id=28F4F392F8F19E3FBDE09616D9DB8143A1E2DDD3 # niftycottonmouse
-185.220.101.21:10021 orport=20021 id=348B89013EDDD99E4755951D1EC284D9FED71226 # niftysquirrel
-185.220.101.10:10010 orport=20010 id=4031460683AE9E0512D3620C2758D98758AC6C93 # niftyeuropeanrabbit
-185.220.101.34:10034 orport=20034 id=47C42E2094EE482E7C9B586B10BABFB67557030B # niftyquokka
-185.220.101.18:10018 orport=20018 id=5D5006E4992F2F97DF4F8B926C3688870EB52BD8 # niftyplagiodontia
-185.220.101.28:10028 orport=20028 id=609E598FB6A00BCF7872906B602B705B64541C50 # niftychipmunk
-185.220.101.20:10020 orport=20020 id=619349D82424C601CAEB94161A4CF778993DAEE7 # niftytucotuco
-185.220.101.17:10017 orport=20017 id=644DECC5A1879C0FE23DE927DD7049F58BBDF349 # niftyhutia
-185.220.101.0:10000 orport=20000 id=6E94866ED8CA098BACDFD36D4E8E2B459B8A734E # niftybeaver
-185.220.101.30:10030 orport=20030 id=71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36 # niftypedetidae
-185.220.101.29:10029 orport=20029 id=7DC52AE6667A30536BA2383CD102CFC24F20AD71 # niftyllipika
-185.220.101.41:10041 orport=20041 id=7E281CD2C315C4F7A84BC7C8721C3BC974DDBFA3 # niftyporcupine
-185.220.101.25:10025 orport=20025 id=8EE0534532EA31AA5172B1892F53B2F25C76EB02 # niftyjerboa
-185.220.101.33:10033 orport=20033 id=906DCB390F2BA987AE258D745E60BAAABAD31DE8 # niftyquokka
-185.220.101.26:10026 orport=20026 id=92A6085EABAADD928B6F8E871540A1A41CBC08BA # niftypedetes
-185.220.101.40:10040 orport=20040 id=9A857254F379194D1CD76F4A79A20D2051BEDA3F # niftynutria
-185.220.101.42:10042 orport=20042 id=9B816A5B3EB20B8E4E9B9D1FBA299BD3F40F0320 # niftypygmyjerboa
-185.220.101.2:10002 orport=20002 id=B740BCECC4A9569232CDD45C0E1330BA0D030D33 # niftybunny
-185.220.101.32:10032 orport=20032 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 # niftypika
-185.220.101.12:10012 orport=20012 id=BC82F2190DE2E97DE65F49B4A95572374BDC0789 # niftycapybara
-185.220.101.22:10022 orport=20022 id=CA37CD46799449D83B6B98B8C22C649906307888 # niftyjackrabbit
-185.220.101.4:10004 orport=20004 id=CDA2EA326E2272C57ACB26773D7252C211795B78 # niftygerbil
-185.220.101.14:10014 orport=20014 id=E7EBA5D8A4E09684D11A1DF24F75362817333768 # niftyhare
-185.220.101.16:10016 orport=20016 id=EC1997D51892E4607C68E800549A1E7E4694005A # niftyguineapig
-185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat
-
-# Email sent directly to teor, verified using relay contact info
-64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2
-
-# Emails sent directly to teor, verified using relay contact info
-51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF
-
-# Emails sent directly to teor, verified using relay contact info
-85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911
-
-# Email sent directly to teor, verified using relay contact info
-173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7
-
-# Email sent directly to teor, verified using relay contact info
-62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE
-
-# Email sent directly to teor, verified using relay contact info
-51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63
-
-# Email sent directly to teor, verified using relay contact info
-163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443
-
-# Email sent directly to teor, verified using relay contact info
-46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001
-
-# Email sent directly to teor, verified using relay contact info
-199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3
-199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6
-199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29
-199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813
-199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338
-199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060
-199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C
-199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616
-199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971
-199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241
-199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0
-199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD
-199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887
-199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B
-199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A
-199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F
-199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2
-199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0
-199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51
-199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
-86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022
-
-# Email sent directly to teor, verified using relay contact info
-178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443
-
-# Email sent directly to teor, verified using relay contact info
-31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
-# same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
-31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D
-31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1
-31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148
-
-# Email sent directly to teor, verified using relay contact info
-185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013939.html
-94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443 # rejozenger
-
-# Email sent directly to teor, verified using relay contact info
-185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001
-
-# Email sent directly to teor, verified using relay contact info
-# The e84 part of the IPv6 address does not have a leading 0 in the consensus
-81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html
-5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
-
-# Email sent directly to teor, verified using relay contact info
-45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A
-
-# Email sent directly to teor, verified using relay contact info
-217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001
-
-# Email sent directly to teor, verified using relay contact info
-212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20
-163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24
-163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7
-
-# Email sent directly to teor, verified using relay contact info
-163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438
-195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6
-
-# Email sent directly to teor, verified using relay contact info
-185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
-185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270
-31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F
-
-# Email sent directly to teor, verified using relay contact info
-89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001
-
-# Email sent directly to teor, verified using relay contact info
-95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423
-
-# Email sent directly to teor, verified using relay contact info
-85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812
-
-# email sent directly to teor
-72.52.75.27:9030 orport=9001 id=8567AD0A6369ED08527A8A8533A5162AC00F7678 # piecoopdotnet
-
-# Email sent directly to teor, verified using relay contact info
-5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
-5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 ipv6=[2a01:4f8:190:30e1::2]:9001 # zwiubel
-
-# https://trac.torproject.org/projects/tor/ticket/22527#comment:1
-199.184.246.250:80 orport=443 id=1F6ABD086F40B890A33C93CC4606EE68B31C9556 ipv6=[2620:124:1009:1::171]:443
-
-# https://trac.torproject.org/projects/tor/ticket/24695
-163.172.53.84:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:24f8::]:21 # Multivac
-
-# Email sent directly to teor
-54.36.237.163:80 orport=443 id=DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E # GermanCraft2
-
-# Email sent directly to teor
-62.138.7.171:9030 orport=9001 id=9844B981A80B3E4B50897098E2D65167E6AEF127 # 0x3d004
-62.138.7.171:8030 orport=8001 id=9285B22F7953D7874604EEE2B470609AD81C74E9 # 0x3d005
-91.121.23.100:9030 orport=9001 id=3711E80B5B04494C971FB0459D4209AB7F2EA799 # 0x3d002
-91.121.23.100:8030 orport=8001 id=CFBBA0D858F02E40B1432A65F6D13C9BDFE7A46B # 0x3d001
-51.15.13.245:9030 orport=9001 id=CED527EAC230E7B56E5B363F839671829C3BA01B # 0x3d006
-51.15.13.245:8030 orport=8001 id=8EBB8D1CF48FE2AB95C451DA8F10DB6235F40F8A # 0x3d007
-
-# Email sent directly to teor
-104.192.5.248:9030 orport=9001 id=BF735F669481EE1CCC348F0731551C933D1E2278 # Freeway11
-
-# Email sent directly to teor
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013961.html
-178.17.174.14:9030 orport=9001 id=B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1 # TorExitMoldova
-178.17.170.156:9030 orport=9001 id=41C59606AFE1D1AA6EC6EF6719690B856F0B6587 # TorExitMoldova2
-
-# Email sent directly to teor
-163.172.221.44:59030 orport=59001 id=164604F5C86FC8CC9C0288BD9C02311958427597 # altego
-
-# Email sent directly to teor
-46.38.237.221:9030 orport=9001 id=D30E9D4D639068611D6D96861C95C2099140B805 # mine
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013911.html
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013912.html
-199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 # Quintex13
-199.249.224.45:80 orport=443 id=041646640AB306EA74B001966E86169B04CC88D2 # QuintexAirVPN26
-199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 # Quintex18
-199.249.223.45:80 orport=443 id=1AE949967F82BBE7534A3D6BA77A7EBE1CED4369 # Quintex36
-199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B # Quintex14
-199.249.224.63:80 orport=443 id=1E5136DDC52FAE1219208F0A6BADB0BA62587EE6 # Quintex43
-199.249.224.46:80 orport=443 id=2ED4D25766973713EB8C56A290BF07E06B85BF12 # QuintexAirVPN27
-199.249.223.42:80 orport=443 id=3687FEC7E73F61AC66F7AE251E7DEE6BBD8C0252 # Quintex33
-199.249.223.49:80 orport=443 id=36D68478366CB8627866757EBCE7FB3C17FC1CB8 # Quintex40
-199.249.224.49:80 orport=443 id=3CA0D15567024D2E0B557DC0CF3E962B37999A79 # QuintexAirVPN30
-199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD # Quintex12
-199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 # QuintexAirVPN5
-199.249.224.41:80 orport=443 id=54A4820B46E65509BF3E2B892E66930A41759DE9 # QuintexAirVPN22
-199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 # QuintexAirVPN8
-199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C # QuintexAirVPN7
-199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 # QuintexAirVPN6
-199.249.223.46:80 orport=443 id=66E19E8C4773086F669A1E06A3F8C23B6C079129 # Quintex37
-199.249.224.65:80 orport=443 id=764BF8A03868F84C8F323C1A676AA254B80DC3BF # Quintex45
-199.249.223.48:80 orport=443 id=7A3DD280EA4CD4DD16EF8C67B93D9BDE184D1A81 # Quintex39
-199.249.224.68:80 orport=443 id=7E6E9A6FDDB8DC7C92F0CFCC3CBE76C29F061799 # Quintex48
-199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 # Quintex20
-199.249.223.44:80 orport=443 id=8B80169BEF71450FC4069A190853523B7AEA45E1 # Quintex35
-199.249.224.60:80 orport=443 id=9314BD9503B9014261A65C221D77E57389DBCCC1 # Quintex50
-199.249.224.40:80 orport=443 id=9C1E7D92115D431385B8CAEA6A7C15FB89CE236B # QuintexAirVPN21
-199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F # Quintex16
-199.249.224.67:80 orport=443 id=9E2D7C6981269404AA1970B53891701A20424EF8 # Quintex47
-199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A # Quintex15
-199.249.224.48:80 orport=443 id=A0DB820FEC87C0405F7BF05DEE5E4ADED2BB9904 # QuintexAirVPN29
-199.249.224.64:80 orport=443 id=A4A393FEF48640961AACE92D041934B55348CEF9 # Quintex44
-199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 # QuintexAirVPN9
-199.249.223.40:80 orport=443 id=B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249 # Quintex31
-199.249.224.61:80 orport=443 id=B2197C23A4FF5D1C49EE45BA7688BA8BCCD89A0B # Quintex41
-199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 # QuintexAirVPN10
-199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 # Quintex11
-199.249.224.66:80 orport=443 id=C78AFFEEE320EA0F860961763E613FD2FAC855F5 # Quintex46
-199.249.224.44:80 orport=443 id=CB7C0D841FE376EF43F7845FF201B0290C0A239E # QuintexAirVPN25
-199.249.223.47:80 orport=443 id=CC14C97F1D23EE97766828FC8ED8582E21E11665 # Quintex38
-199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 # QuintexAirVPN4
-199.249.223.41:80 orport=443 id=D25210CE07C49F2A4F2BC7A506EB0F5EA7F5E2C2 # Quintex32
-199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 # QuintexAirVPN2
-199.249.224.47:80 orport=443 id=D6FF2697CEA5C0C7DA84797C2E71163814FC2466 # QuintexAirVPN28
-199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 # Quintex19
-199.249.223.43:80 orport=443 id=E480D577F58E782A5BC4FA6F49A6650E9389302F # Quintex34
-199.249.224.69:80 orport=443 id=EABC2DD0D47B5DB11F2D37EB3C60C2A4D91C10F2 # Quintex49
-199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 # QuintexAirVPN3
-199.249.224.42:80 orport=443 id=F21DE9C7DE31601D9716781E17E24380887883D1 # QuintexAirVPN23
-199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 # QuintexAirVPN1
-199.249.224.43:80 orport=443 id=FDD700C791CC6BB0AC1C2099A82CBC367AD4B764 # QuintexAirVPN24
-199.249.224.62:80 orport=443 id=FE00A3A835680E67FBBC895A724E2657BB253E97 # Quintex42
-199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html
-5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01
-217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02
-193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03
-149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html
-104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013929.html
-139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 # coffswifi2
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013946.html
-172.98.193.43:80 orport=443 id=5E56738E7F97AA81DEEF59AF28494293DFBFCCDF # Backplane
-
-# Email sent directly to teor
-62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine
-
-# Email sent directly to teor
-80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html
-51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1
-51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2
-
-# Email sent directly to teor
-185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html
-24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014012.html
-128.31.0.13:80 orport=443 id=A53C46F5B157DD83366D45A8E99A244934A14C46 # csailmitexit
-
-# Email sent directly to teor
-82.247.103.117:110 orport=995 id=C9B3C1661A9577BA24C1C2C6123918921A495509 # Casper01
-109.238.2.79:110 orport=995 id=7520892E3DD133D0B0464D01A158B54B8E2A8B75 # Casper02
-51.15.179.153:110 orport=995 id=BB60F5BA113A0B8B44B7B37DE3567FE561E92F78 # Casper04
-
-# Email sent directly to teor
-80.127.107.179:80 orport=443 id=BC6B2E2F62ACC5EDECBABE64DA1E48F84DD98B78 ipv6=[2001:981:4a22:c::6]:443 # TVISION02
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014020.html
-37.120.174.249:80 orport=443 id=11DF0017A43AF1F08825CD5D973297F81AB00FF3 ipv6=[2a03:4000:6:724c:df98:15f9:b34d:443]:443 # gGDHjdcC6zAlM8k08lX
-
-# These fallbacks opted-in in previous releases, then changed their details,
-# and so we blacklisted them. Now we want to whitelist changes.
-# Assume details update is permanent
-85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # Logforme
-176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 # armbrust
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html
-82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo
diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py
index 8dce4b6e51..93ab56e257 100755
--- a/scripts/maint/format_changelog.py
+++ b/scripts/maint/format_changelog.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright (c) 2014-2019, The Tor Project, Inc.
 # See LICENSE for licensing information
 #
@@ -9,6 +9,11 @@
 # To run it, pipe a section of the changelog (starting with "Changes
 # in Tor 0.x.y.z-alpha" through the script.)
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import os
 import re
 import sys
@@ -190,7 +195,7 @@ def body_parser(line):
     elif re.match(r'^\s+\S', line):
         return TP_HEADTEXT
     else:
-        print "Weird line %r"%line
+        print("Weird line %r"%line, file=sys.stderr)
 
 def clean_head(head):
     return head
@@ -198,7 +203,7 @@ def clean_head(head):
 def head_score(s):
     m = re.match(r'^ +o (.*)', s)
     if not m:
-        print >>sys.stderr, "Can't score %r"%s
+        print("Can't score %r"%s, file=sys.stderr)
         return 99999
     lw = m.group(1).lower()
     if lw.startswith("security") and "feature" not in lw:
@@ -286,12 +291,12 @@ class ChangeLog(object):
             self.curgraf.append(line)
 
         else:
-            assert "This" is "unreachable"
+            assert False  # This should be unreachable.
 
     def lint_head(self, line, head):
         m = re.match(r'^ *o ([^\(]+)((?:\([^\)]+\))?):', head)
         if not m:
-            print >>sys.stderr, "Weird header format on line %s"%line
+            print("Weird header format on line %s"%line, file=sys.stderr)
 
     def lint_item(self, line, grafs, head_type):
         pass
@@ -306,7 +311,7 @@ class ChangeLog(object):
     def dumpGraf(self,par,indent1,indent2=-1):
         if not self.wrapText:
             for line in par:
-                print line
+                print(line)
             return
 
         if indent2 == -1:
@@ -320,17 +325,17 @@ class ChangeLog(object):
 
     def dumpPreheader(self, graf):
         self.dumpGraf(graf, 0)
-        print
+        print()
 
     def dumpMainhead(self, head):
-        print head
+        print(head)
 
     def dumpHeadGraf(self, graf):
         self.dumpGraf(graf, 2)
-        print
+        print()
 
     def dumpSectionHeader(self, header):
-        print header
+        print(header)
 
     def dumpStartOfSections(self):
         pass
@@ -339,10 +344,10 @@ class ChangeLog(object):
         pass
 
     def dumpEndOfSection(self):
-        print
+        print()
 
     def dumpEndOfChangelog(self):
-        print
+        print()
 
     def dumpDrupalBreak(self):
         pass
@@ -350,7 +355,7 @@ class ChangeLog(object):
     def dumpItem(self, grafs):
         self.dumpGraf(grafs[0],4,6)
         for par in grafs[1:]:
-            print
+            print()
             self.dumpGraf(par,6,6)
 
     def collateAndSortSections(self):
@@ -389,7 +394,7 @@ class ChangeLog(object):
         self.dumpStartOfSections()
         for _,head,items in self.sections:
             if not head.endswith(':'):
-                print >>sys.stderr, "adding : to %r"%head
+                print("adding : to %r"%head, file=sys.stderr)
                 head = head + ":"
             self.dumpSectionHeader(head)
             for _,grafs in items:
@@ -400,10 +405,31 @@ class ChangeLog(object):
         self.dumpEndOfSections()
         self.dumpEndOfChangelog()
 
+# Map from issue prefix to pair of (visible prefix, url prefix)
+ISSUE_PREFIX_MAP = {
+    "" : ( "", "tpo/core/tor" ),
+    "tor#" : ( "", "tpo/core/tor" ),
+    "chutney#" : ( "chutney#", "tpo/core/chutney" ),
+    "torspec#" : ( "torspec#", "tpo/core/torspec" ),
+    "trunnel#" : ( "trunnel#", "tpo/core/trunnel" ),
+    "torsocks#" : ( "torsocks#", "tpo/core/torsocks"),
+}
+
 # Let's turn bugs to html.
-BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I)
+BUG_PAT = re.compile('(bug|ticket|issue|feature)\s+([\w/]+#)?(\d{4,6})', re.I)
 def bug_html(m):
-    return "%s <a href='https://bugs.torproject.org/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2))
+    kind = m.group(1)
+    prefix = m.group(2) or ""
+    bugno = m.group(3)
+    try:
+        disp_prefix, url_prefix = ISSUE_PREFIX_MAP[prefix]
+    except KeyError:
+        print("Can't figure out URL for {}{}".formt(prefix,bugno),
+              file=sys.stderr)
+        return "{} {}{}".format(kind, prefix, bugno)
+
+    return "{} <a href='https://bugs.torproject.org/{}/{}'>{}{}</a>".format(
+        kind, url_prefix, bugno, disp_prefix, bugno)
 
 class HTMLChangeLog(ChangeLog):
     def __init__(self, *args, **kwargs):
@@ -445,16 +471,16 @@ class HTMLChangeLog(ChangeLog):
         pass
 
     def dumpStartOfSections(self):
-        print "<ul>\n"
+        print("<ul>\n")
 
     def dumpEndOfSections(self):
-        print "</ul>\n"
+        print("</ul>\n")
 
     def dumpDrupalBreak(self):
-        print "\n</ul>\n"
-        print "<p>&nbsp;</p>"
-        print "\n<!--break-->\n\n"
-        print "<ul>"
+        print("\n</ul>\n")
+        print("<p>&nbsp;</p>")
+        print("\n<!--break-->\n\n")
+        print("<ul>")
 
     def dumpItem(self, grafs):
         grafs[0][0] = grafs[0][0].replace(" - ", "", 1).lstrip()
@@ -464,7 +490,7 @@ class HTMLChangeLog(ChangeLog):
                 self.htmlPar(par)
         else:
             self.htmlText(grafs[0])
-        print
+        print()
 
 op = optparse.OptionParser(usage="usage: %prog [options] [filename]")
 op.add_option('-W', '--no-wrap', action='store_false',
@@ -560,7 +586,7 @@ if options.firstOnly:
     sys.exit(0)
 
 if nextline is not None:
-    print nextline
+    print(nextline)
 
 for line in sys.stdin:
     sys.stdout.write(line)
diff --git a/scripts/maint/generateFallbackDirLine.py b/scripts/maint/generateFallbackDirLine.py
deleted file mode 100755
index b856c938bf..0000000000
--- a/scripts/maint/generateFallbackDirLine.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-# Generate a fallback directory whitelist/blacklist line for every fingerprint
-# passed as an argument.
-#
-# Usage:
-# generateFallbackDirLine.py fingerprint ...
-
-import sys
-import urllib2
-
-import stem.descriptor.remote
-import stem.util.tor_tools
-
-if len(sys.argv) <= 1:
-  print('Usage: %s fingerprint ...' % sys.argv[0])
-  sys.exit(1)
-
-for fingerprint in sys.argv[1:]:
-  if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
-    print("'%s' isn't a valid relay fingerprint" % fingerprint)
-    sys.exit(1)
-
-  try:
-    desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0]
-  except urllib2.HTTPError as exc:
-    if exc.code == 404:
-      print('# %s not found in recent descriptors' % fingerprint)
-      continue
-    else:
-      raise
-
-  if not desc.dir_port:
-    print("# %s needs a DirPort" % fingerprint)
-  else:
-    ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6]
-    ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else ''
-    print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname))
diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py
index d5b8fcae5c..cf7b09fcc3 100755
--- a/scripts/maint/lintChanges.py
+++ b/scripts/maint/lintChanges.py
@@ -1,7 +1,10 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
 from __future__ import print_function
-from __future__ import with_statement
+from __future__ import unicode_literals
+
 import sys
 import re
 import os
@@ -22,7 +25,12 @@ KNOWN_GROUPS = set([
     "Code simplification and refactoring",
     "Removed features",
     "Deprecated features",
-    "Directory authority changes"])
+    "Directory authority changes",
+
+    # These aren't preferred, but sortChanges knows how to clean them up.
+    "Code simplifications and refactoring",
+    "Code simplification and refactorings",
+    "Code simplifications and refactorings"])
 
 NEEDS_SUBCATEGORIES = set([
     "Minor bugfix",
@@ -35,6 +43,36 @@ NEEDS_SUBCATEGORIES = set([
     "Major features",
     ])
 
+def split_tor_version(version):
+    '''
+    Return the initial numeric components of the Tor version as a list of ints.
+    For versions earlier than 0.1.0, returns MAJOR, MINOR, and MICRO.
+    For versions 0.1.0 and later, returns MAJOR, MINOR, MICRO, and PATCHLEVEL if present.
+
+    If the version is malformed, returns None.
+    '''
+    version_match = re.match('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version)
+    if version_match is None:
+        return None
+
+    version_groups = version_match.groups()
+    if version_groups is None:
+        return None
+    if len(version_groups) < 3:
+        return None
+
+    if len(version_groups) != 5:
+        return None
+    version_components = version_groups[0:3]
+    version_components += version_groups[4:5]
+
+    try:
+        version_list = [int(v) for v in version_components if v is not None]
+    except ValueError:
+        return None
+
+    return version_list
+
 def lintfile(fname):
     have_warned = []
 
@@ -87,6 +125,32 @@ def lintfile(fname):
             warn("Bugfix does not say 'Fixes bug X; bugfix on Y'")
         elif re.search('tor-([0-9]+)', contents):
             warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)")
+        else:
+            bugfix_match = re.search('bugfix on ([0-9]+\.[0-9]+\.[0-9]+)', contents)
+            if bugfix_match is None:
+                warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)")
+            elif bugfix_match.group(0) is None:
+                warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)")
+            else:
+                bugfix_match = re.search('bugfix on ([0-9a-z][-.0-9a-z]+[0-9a-z])', contents)
+                bugfix_group = bugfix_match.groups() if bugfix_match is not None else None
+                bugfix_version = bugfix_group[0] if bugfix_group is not None else None
+                package_version = os.environ.get('PACKAGE_VERSION', None)
+                if bugfix_version is None:
+                    # This should be unreachable, unless the patterns are out of sync
+                    warn("Malformed bugfix version.")
+                elif package_version is not None:
+                    # If $PACKAGE_VERSION isn't set, skip this check
+                    bugfix_split = split_tor_version(bugfix_version)
+                    package_split = split_tor_version(package_version)
+                    if bugfix_split is None:
+                        # This should be unreachable, unless the patterns are out of sync
+                        warn("Malformed bugfix version: '{}'.".format(bugfix_version))
+                    elif package_split is None:
+                        # This should be unreachable, unless the patterns are out of sync, or the package versioning scheme has changed
+                        warn("Malformed $PACKAGE_VERSION: '{}'.".format(package_version))
+                    elif bugfix_split > package_split:
+                        warn("Bugfixes must be made on earlier versions (or this version). (Bugfix on version: '{}', current tor package version: '{}'.)".format(bugfix_version, package_version))
 
     return have_warned != []
 
diff --git a/scripts/maint/locatemissingdoxygen.py b/scripts/maint/locatemissingdoxygen.py
index 797bf8176f..a2844346d6 100755
--- a/scripts/maint/locatemissingdoxygen.py
+++ b/scripts/maint/locatemissingdoxygen.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """
   This script parses the stderr output of doxygen and looks for undocumented
@@ -7,6 +7,11 @@
   to highlight the undocumented stuff.
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import os
 import re
 import shutil
@@ -29,10 +34,10 @@ def buildWarnings():
 
 def count(fn):
     if os.path.abspath(fn) not in warnings:
-        print "0\t%s"%fn
+        print("0\t%s"%fn)
     else:
         n = len(warnings[os.path.abspath(fn)])
-        print "%d\t%s"%(n,fn)
+        print("%d\t%s"%(n,fn))
 
 def getIndentation(line):
     s = line.lstrip()
@@ -62,7 +67,7 @@ def annotate(filename):
 
 if __name__ == '__main__':
     if len(sys.argv) == 1:
-        print "Usage: locatemissingdoxygen.py [-A] filename... <doxygen_log"
+        print("Usage: locatemissingdoxygen.py [-A] filename... <doxygen_log")
         sys.exit(1)
     buildWarnings()
     if sys.argv[1] == '-A':
diff --git a/scripts/maint/lookupFallbackDirContact.py b/scripts/maint/lookupFallbackDirContact.py
deleted file mode 100755
index 14c53d1282..0000000000
--- a/scripts/maint/lookupFallbackDirContact.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-
-# Lookup fallback directory contact lines for every fingerprint passed as an
-# argument.
-#
-# Usage:
-# lookupFallbackDirContact.py fingerprint ...
-
-import sys
-
-import stem.descriptor.remote as remote
-
-if len(sys.argv) <= 1:
-  print "Usage: {} fingerprint ...".format(sys.argv[0])
-  sys.exit(-1)
-
-# we need descriptors, because the consensus does not have contact infos
-descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run()
-
-descriptor_list_fingerprints = []
-for d in descriptor_list:
-  assert d.fingerprint in sys.argv[1:]
-  descriptor_list_fingerprints.append(d.fingerprint)
-  print "{} {}".format(d.fingerprint, d.contact)
-
-for fingerprint in sys.argv[1:]:
-  if fingerprint not in descriptor_list_fingerprints:
-    print "{} not found in current descriptors".format(fingerprint)
diff --git a/scripts/maint/practracker/README b/scripts/maint/practracker/README
new file mode 100644
index 0000000000..d978b39806
--- /dev/null
+++ b/scripts/maint/practracker/README
@@ -0,0 +1,21 @@
+Practracker is a simple python tool that keeps track of places where
+our code is ugly, and tries to warn us about new ones or ones that
+get worse.
+
+Right now, practracker looks for the following kinds of
+best-practices violations:
+
+  .c files greater than 3000 lines long
+  .h files greater than 500 lines long
+  .c files with more than 50 includes
+  .h files with more than 15 includes
+
+  All files that include a local header not listed in a .may_include
+  file in the same directory, when that .may_include file has an
+  "!advisory" marker.
+
+The list of current violations is tracked in exceptions.txt; slight
+deviations of the current exceptions cause warnings, whereas large
+ones cause practracker to fail.
+
+For usage information, run "practracker.py --help".
diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt
new file mode 100644
index 0000000000..711ef67d36
--- /dev/null
+++ b/scripts/maint/practracker/exceptions.txt
@@ -0,0 +1,325 @@
+# Welcome to the exceptions file for Tor's best-practices tracker!
+#
+# Each line of this file represents a single violation of Tor's best
+# practices -- typically, a violation that we had before practracker.py
+# first existed.
+#
+# There are three kinds of problems that we recognize right now:
+#   function-size -- a function of more than 100 lines.
+#   file-size -- a .c file of more than 3000 lines, or a .h
+#      file with more than 500 lines.
+#   include-count -- a .c file with more than 50 #includes,
+#      or a .h file with more than 15 #includes.
+#   dependency-violation -- a file includes a header that it should
+#      not, according to an advisory .may_include file.
+#
+# Each line below represents a single exception that practracker should
+# _ignore_. Each line has four parts:
+#  1. The word "problem".
+#  2. The kind of problem.
+#  3. The location of the problem: either a filename, or a
+#     filename:functionname pair.
+#  4. The magnitude of the problem to ignore.
+#
+# So for example, consider this line:
+#    problem file-size /src/core/or/connection_or.c 3200
+#
+# It tells practracker to allow the mentioned file to be up to 3200 lines
+# long, even though ordinarily it would warn about any file with more than
+# 3000 lines.
+#
+# You can either edit this file by hand, or regenerate it completely by
+# running `make practracker-regen`.
+#
+# Remember: It is better to fix the problem than to add a new exception!
+
+problem file-size /src/app/config/config.c 7525
+problem include-count /src/app/config/config.c 80
+problem function-size /src/app/config/config.c:options_act() 381
+problem function-size /src/app/config/config.c:options_validate_cb() 794
+problem function-size /src/app/config/config.c:options_init_from_torrc() 192
+problem function-size /src/app/config/config.c:options_init_from_string() 103
+problem function-size /src/app/config/config.c:options_init_logs() 125
+problem function-size /src/app/config/config.c:parse_bridge_line() 104
+problem function-size /src/app/config/config.c:pt_parse_transport_line() 190
+problem function-size /src/app/config/config.c:parse_dir_authority_line() 150
+problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101
+problem function-size /src/app/config/config.c:port_parse_config() 435
+problem function-size /src/app/config/config.c:parse_ports() 132
+problem function-size /src/app/config/resolve_addr.c:resolve_my_address() 191
+problem file-size /src/app/config/or_options_st.h 1050
+problem include-count /src/app/main/main.c 68
+problem function-size /src/app/main/main.c:dumpstats() 102
+problem function-size /src/app/main/main.c:tor_init() 101
+problem function-size /src/app/main/main.c:sandbox_init_filter() 291
+problem function-size /src/app/main/main.c:run_tor_main_loop() 105
+problem function-size /src/app/main/ntmain.c:nt_service_install() 126
+problem dependency-violation /src/core/crypto/hs_ntor.c 1
+problem dependency-violation /src/core/crypto/hs_ntor.h 1
+problem dependency-violation /src/core/crypto/onion_crypto.c 5
+problem dependency-violation /src/core/crypto/onion_fast.c 1
+problem dependency-violation /src/core/crypto/onion_tap.c 3
+problem dependency-violation /src/core/crypto/relay_crypto.c 9
+problem file-size /src/core/mainloop/connection.c 5700
+problem include-count /src/core/mainloop/connection.c 65
+problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 181
+problem function-size /src/core/mainloop/connection.c:connection_listener_new() 325
+problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161
+problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153
+problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 112
+problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111
+problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 186
+problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241
+problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143
+problem dependency-violation /src/core/mainloop/connection.c 47
+problem dependency-violation /src/core/mainloop/cpuworker.c 12
+problem include-count /src/core/mainloop/mainloop.c 64
+problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 107
+problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123
+problem dependency-violation /src/core/mainloop/mainloop.c 50
+problem dependency-violation /src/core/mainloop/mainloop_pubsub.c 1
+problem dependency-violation /src/core/mainloop/mainloop_sys.c 1
+problem dependency-violation /src/core/mainloop/netstatus.c 4
+problem dependency-violation /src/core/mainloop/periodic.c 2
+problem dependency-violation /src/core/or/address_set.c 1
+problem dependency-violation /src/core/or/cell_queue_st.h 1
+problem file-size /src/core/or/channel.c 3500
+problem dependency-violation /src/core/or/channel.c 9
+problem file-size /src/core/or/channel.h 800
+problem dependency-violation /src/core/or/channel.h 1
+problem dependency-violation /src/core/or/channelpadding.c 6
+problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160
+problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170
+problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214
+problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246
+problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202
+problem dependency-violation /src/core/or/channeltls.c 11
+problem include-count /src/core/or/circuitbuild.c 53
+problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128
+problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206
+problem dependency-violation /src/core/or/circuitbuild.c 25
+problem include-count /src/core/or/circuitlist.c 55
+problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109
+problem function-size /src/core/or/circuitlist.c:circuit_free_() 146
+problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 101
+problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117
+problem dependency-violation /src/core/or/circuitlist.c 19
+problem dependency-violation /src/core/or/circuitlist.h 1
+problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109
+problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113
+problem dependency-violation /src/core/or/circuitmux_ewma.c 2
+problem file-size /src/core/or/circuitpadding.c 3101
+problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113
+problem dependency-violation /src/core/or/circuitpadding.c 6
+problem file-size /src/core/or/circuitpadding.h 813
+problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 103
+problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112
+problem dependency-violation /src/core/or/circuitpadding_machines.c 1
+problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 123
+problem dependency-violation /src/core/or/circuitstats.c 11
+problem file-size /src/core/or/circuituse.c 3195
+problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 128
+problem function-size /src/core/or/circuituse.c:circuit_expire_building() 389
+problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126
+problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149
+problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 108
+problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 351
+problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244
+problem dependency-violation /src/core/or/circuituse.c 24
+problem function-size /src/core/or/command.c:command_process_create_cell() 156
+problem function-size /src/core/or/command.c:command_process_relay_cell() 132
+problem dependency-violation /src/core/or/command.c 9
+problem file-size /src/core/or/connection_edge.c 4655
+problem include-count /src/core/or/connection_edge.c 65
+problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 193
+problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 185
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 420
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 101
+problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 185
+problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 130
+problem dependency-violation /src/core/or/connection_edge.c 27
+problem dependency-violation /src/core/or/connection_edge.h 1
+problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105
+problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142
+problem dependency-violation /src/core/or/connection_or.c 21
+problem dependency-violation /src/core/or/dos.c 6
+problem dependency-violation /src/core/or/onion.c 2
+problem file-size /src/core/or/or.h 1105
+problem include-count /src/core/or/or.h 48
+problem dependency-violation /src/core/or/or.h 1
+problem dependency-violation /src/core/or/or_periodic.c 1
+problem file-size /src/core/or/policies.c 3182
+problem function-size /src/core/or/policies.c:policy_summarize() 107
+problem dependency-violation /src/core/or/policies.c 14
+problem function-size /src/core/or/protover.c:protover_all_supported() 117
+problem dependency-violation /src/core/or/reasons.c 2
+problem file-size /src/core/or/relay.c 3300
+problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127
+problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 109
+problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 192
+problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 137
+problem function-size /src/core/or/relay.c:handle_relay_cell_command() 369
+problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 128
+problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 146
+problem dependency-violation /src/core/or/relay.c 17
+problem dependency-violation /src/core/or/scheduler.c 1
+problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171
+problem dependency-violation /src/core/or/scheduler_kist.c 2
+problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109
+problem dependency-violation /src/core/or/scheduler_vanilla.c 1
+problem dependency-violation /src/core/or/sendme.c 2
+problem dependency-violation /src/core/or/status.c 13
+problem function-size /src/core/or/versions.c:tor_version_parse() 104
+problem dependency-violation /src/core/proto/proto_cell.c 3
+problem dependency-violation /src/core/proto/proto_control0.c 1
+problem dependency-violation /src/core/proto/proto_ext_or.c 2
+problem dependency-violation /src/core/proto/proto_http.c 1
+problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 110
+problem dependency-violation /src/core/proto/proto_socks.c 8
+problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 109
+problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 125
+problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108
+problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153
+problem file-size /src/feature/client/entrynodes.c 4000
+problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 155
+problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246
+problem file-size /src/feature/client/entrynodes.h 700
+problem function-size /src/feature/client/transports.c:handle_proxy_line() 108
+problem function-size /src/feature/client/transports.c:parse_method_line_helper() 110
+problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 111
+problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 113
+problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186
+problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150
+problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 256
+problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 118
+problem function-size /src/feature/control/control_events.c:control_event_stream_status() 124
+problem include-count /src/feature/control/control_getinfo.c 56
+problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 108
+problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 297
+problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234
+problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121
+problem file-size /src/feature/dirauth/dirvote.c 4734
+problem include-count /src/feature/dirauth/dirvote.c 55
+problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 230
+problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 233
+problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 952
+problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 119
+problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 161
+problem function-size /src/feature/dirauth/dirvote.c:dirvote_compute_consensuses() 164
+problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 281
+problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196
+problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 109
+problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125
+problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 109
+problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 175
+problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115
+problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111
+problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132
+problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 165
+problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 124
+problem file-size /src/feature/dirclient/dirclient.c 3204
+problem include-count /src/feature/dirclient/dirclient.c 54
+problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 126
+problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201
+problem function-size /src/feature/dirclient/dirclient.c:directory_send_command() 239
+problem function-size /src/feature/dirclient/dirclient.c:dir_client_decompress_response_body() 111
+problem function-size /src/feature/dirclient/dirclient.c:connection_dir_client_reached_eof() 199
+problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_consensus() 104
+problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 203
+problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 158
+problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 181
+problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 280
+problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389
+problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 635
+problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 101
+problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 165
+problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554
+problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 208
+problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109
+problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115
+problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 134
+problem function-size /src/feature/hs/hs_client.c:send_introduce1() 108
+problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 102
+problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 111
+problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122
+problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107
+problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109
+problem file-size /src/feature/hs/hs_service.c 4300
+problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326
+problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123
+problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295
+problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 158
+problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134
+problem include-count /src/feature/nodelist/networkstatus.c 65
+problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 175
+problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 289
+problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 122
+problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 204
+problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112
+problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190
+problem file-size /src/feature/nodelist/routerlist.c 3247
+problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148
+problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168
+problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121
+problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 142
+problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103
+problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 131
+problem function-size /src/feature/relay/dns.c:configure_nameservers() 161
+problem function-size /src/feature/relay/dns.c:evdns_callback() 108
+problem function-size /src/feature/relay/relay_handshake.c:connection_or_compute_authenticate_cell_body() 231
+problem file-size /src/feature/relay/router.c 3600
+problem include-count /src/feature/relay/router.c 57
+problem function-size /src/feature/relay/router.c:init_keys() 254
+problem function-size /src/feature/relay/router.c:get_my_declared_family() 114
+problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 113
+problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 372
+problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294
+problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 190
+problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 219
+problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 221
+problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 105
+problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 181
+problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 129
+problem file-size /src/feature/rend/rendservice.c 4504
+problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107
+problem function-size /src/feature/rend/rendservice.c:rend_config_service() 143
+problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178
+problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 334
+problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 111
+problem function-size /src/feature/rend/rendservice.c:rend_service_decrypt_intro() 112
+problem function-size /src/feature/rend/rendservice.c:rend_service_intro_has_opened() 126
+problem function-size /src/feature/rend/rendservice.c:rend_service_rendezvous_has_opened() 117
+problem function-size /src/feature/rend/rendservice.c:directory_post_to_hs_dir() 106
+problem function-size /src/feature/rend/rendservice.c:upload_service_descriptor() 111
+problem function-size /src/feature/rend/rendservice.c:rend_consider_services_intro_points() 170
+problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185
+problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148
+problem function-size /src/lib/compress/compress.c:tor_compress_impl() 127
+problem function-size /src/lib/compress/compress_zstd.c:tor_zstd_compress_process() 123
+problem function-size /src/lib/container/smartlist.c:smartlist_bsearch_idx() 107
+problem function-size /src/lib/crypt_ops/crypto_rand.c:crypto_strongest_rand_syscall() 102
+problem function-size /src/lib/encoding/binascii.c:base64_encode() 106
+problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_verbose() 117
+problem function-size /src/lib/encoding/cstring.c:unescape_string() 108
+problem function-size /src/lib/fs/dir.c:check_private_dir() 230
+problem function-size /src/lib/math/prob_distr.c:sample_uniform_interval() 145
+problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 195
+problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 110
+problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107
+problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102
+problem function-size /src/lib/process/process_unix.c:process_unix_exec() 213
+problem function-size /src/lib/process/process_win32.c:process_win32_exec() 151
+problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 109
+problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102
+problem function-size /src/lib/process/setuid.c:switch_id() 156
+problem function-size /src/lib/sandbox/sandbox.c:prot_strings() 104
+problem function-size /src/lib/string/scanf.c:tor_vsscanf() 112
+problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 152
+problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 170
+problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 121
+problem function-size /src/tools/tor-gencert.c:parse_commandline() 111
+problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 102
+problem function-size /src/tools/tor-resolve.c:do_resolve() 171
+problem function-size /src/tools/tor-resolve.c:main() 112
diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py
new file mode 100755
index 0000000000..a5ee728824
--- /dev/null
+++ b/scripts/maint/practracker/includes.py
@@ -0,0 +1,381 @@
+#!/usr/bin/env python
+# Copyright 2018 The Tor Project, Inc.  See LICENSE file for licensing info.
+
+"""This script looks through all the directories for files matching *.c or
+   *.h, and checks their #include directives to make sure that only "permitted"
+   headers are included.
+
+   Any #include directives with angle brackets (like #include <stdio.h>) are
+   ignored -- only directives with quotes (like #include "foo.h") are
+   considered.
+
+   To decide what includes are permitted, this script looks at a .may_include
+   file in each directory.  This file contains empty lines, #-prefixed
+   comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h)
+   for files that are permitted.
+
+   The script exits with an error if any non-permitted includes are found.
+   .may_include files that contain "!advisory" are considered advisory.
+   Advisory .may_include files only result in warnings, rather than errors.
+"""
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import fnmatch
+import os
+import re
+import sys
+
+if sys.version_info[0] <= 2:
+    def open_file(fname):
+        return open(fname, 'r')
+else:
+    def open_file(fname):
+        return open(fname, 'r', encoding='utf-8')
+
+def warn(msg):
+    print(msg, file=sys.stderr)
+
+def fname_is_c(fname):
+    """ Return true iff 'fname' is the name of a file that we should
+        search for possibly disallowed #include directives. """
+    if fname.endswith(".h") or fname.endswith(".c"):
+        bname = os.path.basename(fname)
+        return not (bname.startswith(".") or bname.startswith("#"))
+    else:
+        return False
+
+INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"')
+RULES_FNAME = ".may_include"
+
+ALLOWED_PATTERNS = [
+    re.compile(r'^.*\*\.(h|inc)$'),
+    re.compile(r'^.*/.*\.h$'),
+    re.compile(r'^ext/.*\.c$'),
+    re.compile(r'^orconfig.h$'),
+    re.compile(r'^micro-revision.i$'),
+]
+
+TOPDIR = "src"
+
+def pattern_is_normal(s):
+    for p in ALLOWED_PATTERNS:
+        if p.match(s):
+            return True
+    return False
+
+class Error(object):
+    def __init__(self, location, msg, is_advisory=False):
+        self.location = location
+        self.msg = msg
+        self.is_advisory = is_advisory
+
+    def __str__(self):
+        return "{} at {}".format(self.msg, self.location)
+
+class Rules(object):
+    """ A 'Rules' object is the parsed version of a .may_include file. """
+    def __init__(self, dirpath):
+        self.dirpath = dirpath
+        if dirpath.startswith("src/"):
+            self.incpath = dirpath[4:]
+        else:
+            self.incpath = dirpath
+        self.patterns = []
+        self.usedPatterns = set()
+        self.is_advisory = False
+
+    def addPattern(self, pattern):
+        if pattern == "!advisory":
+            self.is_advisory = True
+            return
+        if not pattern_is_normal(pattern):
+            warn("Unusual pattern {} in {}".format(pattern, self.dirpath))
+        self.patterns.append(pattern)
+
+    def includeOk(self, path):
+        for pattern in self.patterns:
+            if fnmatch.fnmatchcase(path, pattern):
+                self.usedPatterns.add(pattern)
+                return True
+        return False
+
+    def applyToLines(self, lines, loc_prefix=""):
+        lineno = 0
+        for line in lines:
+            lineno += 1
+            m = INCLUDE_PATTERN.match(line)
+            if m:
+                include = m.group(1)
+                if not self.includeOk(include):
+                    yield Error("{}{}".format(loc_prefix,str(lineno)),
+                                "Forbidden include of {}".format(include),
+                                is_advisory=self.is_advisory)
+
+    def applyToFile(self, fname, f):
+        for error in self.applyToLines(iter(f), "{}:".format(fname)):
+            yield error
+
+    def noteUnusedRules(self):
+        for p in self.patterns:
+            if p not in self.usedPatterns:
+                warn("Pattern {} in {} was never used.".format(p, self.dirpath))
+
+    def getAllowedDirectories(self):
+        allowed = []
+        for p in self.patterns:
+            m = re.match(r'^(.*)/\*\.(h|inc)$', p)
+            if m:
+                allowed.append(m.group(1))
+                continue
+            m = re.match(r'^(.*)/[^/]*$', p)
+            if m:
+                allowed.append(m.group(1))
+                continue
+
+        return allowed
+
+
+def normalize_srcdir(fname):
+    """given the name of a source directory or file, return its name
+        relative to `src` in a unix-like format.
+    """
+    orig = fname
+    dirname, dirfile = os.path.split(fname)
+    if re.match(r'.*\.[ch]$', dirfile):
+        fname = dirname
+
+    # Now we have a directory.
+    dirname, result = os.path.split(fname)
+    for _ in range(100):
+        # prevent excess looping in case I missed a tricky case
+        dirname, dirpart = os.path.split(dirname)
+        if dirpart == 'src' or dirname == "":
+            #print(orig,"=>",result)
+            return result
+        result = "{}/{}".format(dirpart,result)
+
+    print("No progress!")
+    assert False
+
+include_rules_cache = {}
+
+def load_include_rules(fname):
+    """ Read a rules file from 'fname', and return it as a Rules object.
+        Return 'None' if fname does not exist.
+    """
+    if fname in include_rules_cache:
+        return include_rules_cache[fname]
+    if not os.path.exists(fname):
+        include_rules_cache[fname] = None
+        return None
+    result = Rules(os.path.split(fname)[0])
+    with open_file(fname) as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("#") or not line:
+                continue
+            result.addPattern(line)
+    include_rules_cache[fname] = result
+    return result
+
+def get_all_include_rules():
+    """Return a list of all the Rules objects we have loaded so far,
+       sorted by their directory names."""
+    return [ rules for (fname,rules) in
+             sorted(include_rules_cache.items())
+             if rules is not None ]
+
+def remove_self_edges(graph):
+    """Takes a directed graph in as an adjacency mapping (a mapping from
+       node to a list of the nodes to which it connects).
+
+       Remove all edges from a node to itself."""
+
+    for k in list(graph):
+        graph[k] = [ d for d in graph[k] if d != k ]
+
+def closure(graph):
+    """Takes a directed graph in as an adjacency mapping (a mapping from
+       node to a list of the nodes to which it connects), and completes
+       its closure.
+    """
+    graph = graph.copy()
+    changed = False
+    for k in graph.keys():
+        graph[k] = set(graph[k])
+    while True:
+        for k in graph.keys():
+            sz = len(graph[k])
+            for v in list(graph[k]):
+                graph[k].update(graph.get(v, []))
+            if sz != len(graph[k]):
+                changed = True
+
+        if not changed:
+            return graph
+        changed = False
+
+def toposort(graph, limit=100):
+    """Takes a directed graph in as an adjacency mapping (a mapping from
+       node to a list of the nodes to which it connects).  Tries to
+       perform a topological sort on the graph, arranging the nodes into
+       "levels", such that every member of each level is only reachable
+       by members of later levels.
+
+       Returns a list of the members of each level.
+
+       Modifies the input graph, removing every member that could be
+       sorted.  If the graph does not become empty, then it contains a
+       cycle.
+
+       "limit" is the max depth of the graph after which we give up trying
+       to sort it and conclude we have a cycle.
+    """
+    all_levels = []
+
+    n = 0
+    while graph:
+        n += 0
+        cur_level = []
+        all_levels.append(cur_level)
+        for k in list(graph):
+            graph[k] = [ d for d in graph[k] if d in graph ]
+            if graph[k] == []:
+                cur_level.append(k)
+        for k in cur_level:
+            del graph[k]
+        n += 1
+        if n > limit:
+            break
+
+    return all_levels
+
+def consider_include_rules(fname, f):
+    dirpath = os.path.split(fname)[0]
+    rules_fname = os.path.join(dirpath, RULES_FNAME)
+    rules = load_include_rules(os.path.join(dirpath, RULES_FNAME))
+    if rules is None:
+        return
+
+    for err in rules.applyToFile(fname, f):
+        yield err
+
+    list_unused = False
+    log_sorted_levels = False
+
+def walk_c_files(topdir="src"):
+    """Run through all .c and .h files under topdir, looking for
+       include-rule violations. Yield those violations."""
+
+    for dirpath, dirnames, fnames in os.walk(topdir):
+        for fname in fnames:
+            if fname_is_c(fname):
+                fullpath = os.path.join(dirpath,fname)
+                with open(fullpath) as f:
+                    for err in consider_include_rules(fullpath, f):
+                        yield err
+
+def open_or_stdin(fname):
+    if fname == '-':
+        return sys.stdin
+    else:
+        return open(fname)
+
+def check_subsys_file(fname, uses_dirs):
+    if not uses_dirs:
+        # We're doing a distcheck build, or for some other reason there are
+        # no .may_include files.
+        print("SKIPPING")
+        return False
+
+    uses_dirs = { normalize_srcdir(k) : { normalize_srcdir(d) for d in v }
+                  for (k,v) in uses_dirs.items() }
+    uses_closure = closure(uses_dirs)
+    ok = True
+    previous_subsystems = []
+
+    with open_or_stdin(fname) as f:
+        for line in f:
+            _, name, fname = line.split()
+            fname = normalize_srcdir(fname)
+            for prev in previous_subsystems:
+                if fname in uses_closure[prev]:
+                    print("INVERSION: {} uses {}".format(prev,fname))
+                    ok = False
+            previous_subsystems.append(fname)
+    return not ok
+
+def run_check_includes(topdir, list_unused=False, log_sorted_levels=False,
+                       list_advisories=False, check_subsystem_order=None):
+    trouble = False
+
+    for err in walk_c_files(topdir):
+        if err.is_advisory and not list_advisories:
+            continue
+        print(err, file=sys.stderr)
+        if not err.is_advisory:
+            trouble = True
+
+    if trouble:
+        warn(
+    """To change which includes are allowed in a C file, edit the {}
+    files in its enclosing directory.""".format(RULES_FNAME))
+        sys.exit(1)
+
+    if list_unused:
+        for rules in get_all_include_rules():
+            rules.noteUnusedRules()
+
+    uses_dirs = { }
+    for rules in get_all_include_rules():
+        uses_dirs[rules.incpath] = rules.getAllowedDirectories()
+
+    remove_self_edges(uses_dirs)
+
+    if check_subsystem_order:
+        if check_subsys_file(check_subsystem_order, uses_dirs):
+            sys.exit(1)
+
+    all_levels = toposort(uses_dirs)
+
+    if log_sorted_levels:
+        for (n, cur_level) in enumerate(all_levels):
+            if cur_level:
+                print(n, cur_level)
+
+    if uses_dirs:
+        print("There are circular .may_include dependencies in here somewhere:",
+              uses_dirs)
+        sys.exit(1)
+
+def main(argv):
+    import argparse
+
+    progname = argv[0]
+    parser = argparse.ArgumentParser(prog=progname)
+    parser.add_argument("--toposort", action="store_true",
+                        help="Print a topologically sorted list of modules")
+    parser.add_argument("--list-unused", action="store_true",
+                        help="List unused lines in .may_include files.")
+    parser.add_argument("--list-advisories", action="store_true",
+                        help="List advisories as well as forbidden includes")
+    parser.add_argument("--check-subsystem-order", action="store",
+                        help="Check a list of subsystems for ordering")
+    parser.add_argument("topdir", default="src", nargs="?",
+                        help="Top-level directory for the tor source")
+    args = parser.parse_args(argv[1:])
+
+    global TOPDIR
+    TOPDIR = args.topdir
+    run_check_includes(topdir=args.topdir,
+                       log_sorted_levels=args.toposort,
+                       list_unused=args.list_unused,
+                       list_advisories=args.list_advisories,
+                       check_subsystem_order=args.check_subsystem_order)
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py
new file mode 100644
index 0000000000..300a4501a9
--- /dev/null
+++ b/scripts/maint/practracker/metrics.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+# Implementation of various source code metrics.
+# These are currently ad-hoc string operations and regexps.
+# We might want to use a proper static analysis library in the future, if we want to get more advanced metrics.
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import re
+
+def get_file_len(f):
+    """Get file length of file"""
+    i = -1
+    for i, l in enumerate(f):
+        pass
+    return i + 1
+
+def get_include_count(f):
+    """Get number of #include statements in the file"""
+    include_count = 0
+    for line in f:
+        if re.match(r'\s*#\s*include', line):
+            include_count += 1
+    return include_count
+
+def get_function_lines(f):
+    """
+    Return iterator which iterates over functions and returns (function name, function lines)
+    """
+
+    # Skip lines that look like they are defining functions with these
+    # names: they aren't real function definitions.
+    REGEXP_CONFUSE_TERMS = {"MOCK_IMPL", "MOCK_DECL", "HANDLE_DECL",
+                            "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING",
+                            "DUMMY_TYPECHECK_INSTANCE",
+                            "DISABLE_GCC_WARNING", "DISABLE_GCC_WARNINGS"}
+
+    in_function = False
+    found_openbrace = False
+    for lineno, line in enumerate(f):
+        if not in_function:
+            # find the start of a function
+            m = re.match(r'^([a-zA-Z_][a-zA-Z_0-9]*),?\(', line)
+            if m:
+                func_name = m.group(1)
+                if func_name in REGEXP_CONFUSE_TERMS:
+                    continue
+                func_start = lineno
+                in_function = True
+        elif not found_openbrace and line.startswith("{"):
+            found_openbrace = True
+            func_start = lineno
+        else:
+            # Find the end of a function
+            if line.startswith("}"):
+                n_lines = lineno - func_start + 1
+                in_function = False
+                found_openbrace = False
+                yield (func_name, n_lines)
diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py
new file mode 100755
index 0000000000..76ffd64cfb
--- /dev/null
+++ b/scripts/maint/practracker/practracker.py
@@ -0,0 +1,320 @@
+#!/usr/bin/env python
+
+"""
+Best-practices tracker for Tor source code.
+
+Go through the various .c files and collect metrics about them. If the metrics
+violate some of our best practices and they are not found in the optional
+exceptions file, then log a problem about them.
+
+We currently do metrics about file size, function size and number of includes,
+for C source files and headers.
+
+practracker.py should be run with its second argument pointing to the Tor
+top-level source directory like this:
+  $ python3 ./scripts/maint/practracker/practracker.py .
+
+To regenerate the exceptions file so that it allows all current
+problems in the Tor source, use the --regen flag:
+  $ python3 --regen ./scripts/maint/practracker/practracker.py .
+"""
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import codecs, os, sys
+
+import metrics
+import util
+import problem
+import includes
+import shutil
+
+# The filename of the exceptions file (it should be placed in the practracker directory)
+EXCEPTIONS_FNAME = "./exceptions.txt"
+
+# Recommended file size
+MAX_FILE_SIZE = 3000 # lines
+# Recommended function size
+MAX_FUNCTION_SIZE = 100 # lines
+# Recommended number of #includes
+MAX_INCLUDE_COUNT = 50
+# Recommended file size for headers
+MAX_H_FILE_SIZE = 500
+# Recommended include count for headers
+MAX_H_INCLUDE_COUNT = 15
+# Recommended number of dependency violations
+MAX_DEP_VIOLATIONS = 0
+
+# Map from problem type to functions that adjust for tolerance
+TOLERANCE_FNS = {
+    'include-count': lambda n: int(n*1.1),
+    'function-size': lambda n: int(n*1.1),
+    'file-size': lambda n: int(n*1.02),
+    'dependency-violation': lambda n: (n+2)
+}
+
+#######################################################
+
+# The Tor source code topdir
+TOR_TOPDIR = None
+
+#######################################################
+
+def open_file(fname):
+    return codecs.open(fname, 'r', encoding='utf-8')
+
+def consider_file_size(fname, f):
+    """Consider the size of 'f' and yield an FileSizeItem for it.
+    """
+    file_size = metrics.get_file_len(f)
+    yield problem.FileSizeItem(fname, file_size)
+
+def consider_includes(fname, f):
+    """Consider the #include count in for 'f' and yield an IncludeCountItem
+        for it.
+    """
+    include_count = metrics.get_include_count(f)
+
+    yield problem.IncludeCountItem(fname, include_count)
+
+def consider_function_size(fname, f):
+    """yield a FunctionSizeItem for every function in f.
+    """
+
+    for name, lines in metrics.get_function_lines(f):
+        canonical_function_name = "%s:%s()" % (fname, name)
+        yield problem.FunctionSizeItem(canonical_function_name, lines)
+
+def consider_include_violations(fname, real_fname, f):
+    n = 0
+    for item in includes.consider_include_rules(real_fname, f):
+        n += 1
+    if n:
+        yield problem.DependencyViolationItem(fname, n)
+
+
+#######################################################
+
+def consider_all_metrics(files_list):
+    """Consider metrics for all files, and yield a sequence of problem.Item
+       object for those issues."""
+    for fname in files_list:
+        with open_file(fname) as f:
+            for item in consider_metrics_for_file(fname, f):
+                yield item
+
+def consider_metrics_for_file(fname, f):
+    """
+       Yield a sequence of problem.Item objects for all of the metrics in
+       'f'.
+    """
+    real_fname = fname
+    # Strip the useless part of the path
+    if fname.startswith(TOR_TOPDIR):
+        fname = fname[len(TOR_TOPDIR):]
+
+    # Get file length
+    for item in consider_file_size(fname, f):
+        yield item
+
+    # Consider number of #includes
+    f.seek(0)
+    for item in consider_includes(fname, f):
+        yield item
+
+    # Get function length
+    f.seek(0)
+    for item in consider_function_size(fname, f):
+        yield item
+
+    # Check for "upward" includes
+    f.seek(0)
+    for item in consider_include_violations(fname, real_fname, f):
+        yield item
+
+HEADER="""\
+# Welcome to the exceptions file for Tor's best-practices tracker!
+#
+# Each line of this file represents a single violation of Tor's best
+# practices -- typically, a violation that we had before practracker.py
+# first existed.
+#
+# There are three kinds of problems that we recognize right now:
+#   function-size -- a function of more than {MAX_FUNCTION_SIZE} lines.
+#   file-size -- a .c file of more than {MAX_FILE_SIZE} lines, or a .h
+#      file with more than {MAX_H_FILE_SIZE} lines.
+#   include-count -- a .c file with more than {MAX_INCLUDE_COUNT} #includes,
+#      or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes.
+#   dependency-violation -- a file includes a header that it should
+#      not, according to an advisory .may_include file.
+#
+# Each line below represents a single exception that practracker should
+# _ignore_. Each line has four parts:
+#  1. The word "problem".
+#  2. The kind of problem.
+#  3. The location of the problem: either a filename, or a
+#     filename:functionname pair.
+#  4. The magnitude of the problem to ignore.
+#
+# So for example, consider this line:
+#    problem file-size /src/core/or/connection_or.c 3200
+#
+# It tells practracker to allow the mentioned file to be up to 3200 lines
+# long, even though ordinarily it would warn about any file with more than
+# {MAX_FILE_SIZE} lines.
+#
+# You can either edit this file by hand, or regenerate it completely by
+# running `make practracker-regen`.
+#
+# Remember: It is better to fix the problem than to add a new exception!
+
+""".format(**globals())
+
+def main(argv):
+    import argparse
+
+    progname = argv[0]
+    parser = argparse.ArgumentParser(prog=progname)
+    parser.add_argument("--regen", action="store_true",
+                        help="Regenerate the exceptions file")
+    parser.add_argument("--list-overbroad", action="store_true",
+                        help="List over-broad exceptions")
+    parser.add_argument("--regen-overbroad", action="store_true",
+                        help="Regenerate the exceptions file, "
+                             "removing over-broad exceptions.")
+    parser.add_argument("--exceptions",
+                        help="Override the location for the exceptions file")
+    parser.add_argument("--strict", action="store_true",
+                        help="Make all warnings into errors")
+    parser.add_argument("--terse", action="store_true",
+                        help="Do not emit helpful instructions.")
+    parser.add_argument("--max-h-file-size", default=MAX_H_FILE_SIZE,
+                        help="Maximum lines per .h file")
+    parser.add_argument("--max-h-include-count", default=MAX_H_INCLUDE_COUNT,
+                        help="Maximum includes per .h file")
+    parser.add_argument("--max-file-size", default=MAX_FILE_SIZE,
+                        help="Maximum lines per .c file")
+    parser.add_argument("--max-include-count", default=MAX_INCLUDE_COUNT,
+                        help="Maximum includes per .c file")
+    parser.add_argument("--max-function-size", default=MAX_FUNCTION_SIZE,
+                        help="Maximum lines per function")
+    parser.add_argument("--max-dependency-violations", default=MAX_DEP_VIOLATIONS,
+                        help="Maximum number of dependency violations to allow")
+    parser.add_argument("--include-dir", action="append",
+                        default=["src"],
+                        help="A directory (under topdir) to search for source")
+    parser.add_argument("topdir", default=".", nargs="?",
+                        help="Top-level directory for the tor source")
+    args = parser.parse_args(argv[1:])
+
+    global TOR_TOPDIR
+    TOR_TOPDIR = args.topdir
+    if args.exceptions:
+        exceptions_file = args.exceptions
+    else:
+        exceptions_file = os.path.join(TOR_TOPDIR, "scripts/maint/practracker", EXCEPTIONS_FNAME)
+
+    # 0) Configure our thresholds of "what is a problem actually"
+    filt = problem.ProblemFilter()
+    filt.addThreshold(problem.FileSizeItem("*.c", int(args.max_file_size)))
+    filt.addThreshold(problem.IncludeCountItem("*.c", int(args.max_include_count)))
+    filt.addThreshold(problem.FileSizeItem("*.h", int(args.max_h_file_size)))
+    filt.addThreshold(problem.IncludeCountItem("*.h", int(args.max_h_include_count)))
+    filt.addThreshold(problem.FunctionSizeItem("*.c", int(args.max_function_size)))
+    filt.addThreshold(problem.DependencyViolationItem("*.c", int(args.max_dependency_violations)))
+    filt.addThreshold(problem.DependencyViolationItem("*.h", int(args.max_dependency_violations)))
+
+    if args.list_overbroad + args.regen + args.regen_overbroad > 1:
+        print("Cannot use more than one of --regen, --list-overbroad, and "
+              "--regen-overbroad.",
+              file=sys.stderr)
+        sys.exit(1)
+
+    # 1) Get all the .c files we care about
+    files_list = util.get_tor_c_files(TOR_TOPDIR, args.include_dir)
+
+    # 2) Initialize problem vault and load an optional exceptions file so that
+    # we don't warn about the past
+    if args.regen:
+        tmpname = exceptions_file + ".tmp"
+        tmpfile = open(tmpname, "w")
+        problem_file = tmpfile
+        problem_file.write(HEADER)
+        ProblemVault = problem.ProblemVault()
+    else:
+        ProblemVault = problem.ProblemVault(exceptions_file)
+        problem_file = sys.stdout
+
+    if args.list_overbroad or args.regen_overbroad:
+        # If we're looking for overbroad exceptions, don't list problems
+        # immediately to the problem file.
+        problem_file = util.NullFile()
+
+    # 2.1) Adjust the exceptions so that we warn only about small problems,
+    # and produce errors on big ones.
+    if not (args.regen or args.list_overbroad or args.regen_overbroad or
+            args.strict):
+        ProblemVault.set_tolerances(TOLERANCE_FNS)
+
+    # 3) Go through all the files and report problems if they are not exceptions
+    found_new_issues = 0
+    for item in filt.filter(consider_all_metrics(files_list)):
+        status = ProblemVault.register_problem(item)
+        if status == problem.STATUS_ERR:
+            print(item, file=problem_file)
+            found_new_issues += 1
+        elif status == problem.STATUS_WARN:
+            # warnings always go to stdout.
+            print("(warning) {}".format(item))
+
+    if args.regen:
+        tmpfile.close()
+        shutil.move(tmpname, exceptions_file)
+        sys.exit(0)
+
+    if args.regen_overbroad:
+        tmpname = exceptions_file + ".tmp"
+        tmpfile = open(tmpname, "w")
+        tmpfile.write(HEADER)
+        for item in ProblemVault.list_exceptions_without_overbroad():
+            print(item, file=tmpfile)
+        tmpfile.close()
+        shutil.move(tmpname, exceptions_file)
+        sys.exit(0)
+
+    # If new issues were found, try to give out some advice to the developer on how to resolve it.
+    if found_new_issues and not args.regen and not args.terse:
+        new_issues_str = """\
+FAILURE: practracker found {} new problem(s) in the code: see warnings above.
+
+Please fix the problems if you can, and update the exceptions file
+({}) if you can't.
+
+See doc/HACKING/HelpfulTools.md for more information on using practracker.\
+
+You can disable this message by setting the TOR_DISABLE_PRACTRACKER environment
+variable.
+""".format(found_new_issues, exceptions_file)
+        print(new_issues_str)
+
+    if args.list_overbroad:
+        def k_fn(tup):
+            return tup[0].key()
+        for (ex,p) in sorted(ProblemVault.list_overbroad_exceptions(), key=k_fn):
+            if p is None:
+                print(ex, "->", 0)
+            else:
+                print(ex, "->", p.metric_value)
+
+
+    sys.exit(found_new_issues)
+
+if __name__ == '__main__':
+    if os.environ.get("TOR_DISABLE_PRACTRACKER"):
+        print("TOR_DISABLE_PRACTRACKER is set, skipping practracker tests.",
+              file=sys.stderr)
+        sys.exit(0)
+    main(sys.argv)
diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py
new file mode 100755
index 0000000000..e03c9e05ae
--- /dev/null
+++ b/scripts/maint/practracker/practracker_tests.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+"""Some simple tests for practracker metrics"""
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import unittest
+
+try:
+    # python 2 names the module this way...
+    from StringIO import StringIO
+except ImportError:
+    # python 3 names the module this way.
+    from io import StringIO
+
+import metrics
+
+function_file = """static void
+fun(directory_request_t *req, const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+}
+
+static void
+fun(directory_request_t *req,
+      const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+}
+
+MOCK_IMPL(void,
+fun,(
+       uint8_t dir_purpose,
+       uint8_t router_purpose,
+       const char *resource,
+       int pds_flags,
+       download_want_authority_t want_authority))
+{
+  const routerstatus_t *rs = NULL;
+  const or_options_t *options = get_options();
+}
+"""
+
+class TestFunctionLength(unittest.TestCase):
+    def test_function_length(self):
+        funcs = StringIO(function_file)
+        # All functions should have length 2
+        for name, lines in metrics.get_function_lines(funcs):
+            self.assertEqual(name, "fun")
+
+        funcs.seek(0)
+
+        for name, lines in metrics.get_function_lines(funcs):
+            self.assertEqual(lines, 4)
+
+class TestIncludeCount(unittest.TestCase):
+    def test_include_count(self):
+        f = StringIO("""
+  #   include <abc.h>
+  #   include "def.h"
+#include "ghi.h"
+\t#\t include "jkl.h"
+""")
+        self.assertEqual(metrics.get_include_count(f),4)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py
new file mode 100644
index 0000000000..a3255dcc80
--- /dev/null
+++ b/scripts/maint/practracker/problem.py
@@ -0,0 +1,266 @@
+"""
+In this file we define a ProblemVault class where we store all the
+exceptions and all the problems we find with the code.
+
+The ProblemVault is capable of registering problems and also figuring out if a
+problem is worse than a registered exception so that it only warns when things
+get worse.
+"""
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os.path
+import re
+import sys
+
+STATUS_ERR = 2
+STATUS_WARN = 1
+STATUS_OK = 0
+
+class ProblemVault(object):
+    """
+    Singleton where we store the various new problems we
+    found in the code, and also the old problems we read from the exception
+    file.
+    """
+    def __init__(self, exception_fname=None):
+        # Exception dictionary: { problem.key() : Problem object }
+        self.exceptions = {}
+        # Exception list: list of Problem objects, in the order added.
+        self.exception_list = []
+        # Exception dictionary: maps key to the problem it was used to
+        # suppress.
+        self.used_exception_for = {}
+
+        if exception_fname == None:
+            return
+
+        try:
+            with open(exception_fname, 'r') as exception_f:
+                self.register_exceptions(exception_f)
+        except IOError:
+            print("No exception file provided", file=sys.stderr)
+
+    def register_exceptions(self, exception_file):
+        # Register exceptions
+        for lineno, line in enumerate(exception_file, 1):
+            try:
+                problem = get_old_problem_from_exception_str(line)
+            except ValueError as v:
+                print("Exception file line {} not recognized: {}"
+                      .format(lineno,v),
+                      file=sys.stderr)
+                continue
+
+            if problem is None:
+                continue
+
+            # Fail if we see dup exceptions. There is really no reason to have dup exceptions.
+            if problem.key() in self.exceptions:
+                print("Duplicate exceptions lines found in exception file:\n\t{}\n\t{}\nAborting...".format(problem, self.exceptions[problem.key()]),
+                      file=sys.stderr)
+                sys.exit(1)
+
+            self.exceptions[problem.key()] = problem
+            self.exception_list.append(problem)
+            #print "Registering exception: %s" % problem
+
+    def register_problem(self, problem):
+        """
+        Register this problem to the problem value. Return true if it was a new
+        problem or it worsens an already existing problem.  A true
+        value may be STATUS_ERR to indicate a hard violation, or STATUS_WARN
+        to indicate a warning.
+        """
+        # This is a new problem, print it
+        if problem.key() not in self.exceptions:
+            return STATUS_ERR
+
+        # If it's an old problem, we don't warn if the situation got better
+        # (e.g. we went from 4k LoC to 3k LoC), but we do warn if the
+        # situation worsened (e.g. we went from 60 includes to 80).
+        status = problem.is_worse_than(self.exceptions[problem.key()])
+
+        # Remember that we used this exception, so that we can later
+        # determine whether the exception was overbroad.
+        self.used_exception_for[problem.key()] = problem
+
+        return status
+
+    def list_overbroad_exceptions(self):
+        """Return an iterator of tuples containing (ex,prob) where ex is an
+           exceptions in this vault that are stricter than it needs to be, and
+           prob is the worst problem (if any) that it covered.
+        """
+        for k in self.exceptions:
+            e = self.exceptions[k]
+            p = self.used_exception_for.get(k)
+            if p is None or e.is_worse_than(p):
+                yield (e, p)
+
+    def list_exceptions_without_overbroad(self):
+        """Return an iterator of new problems, such that overbroad
+           exceptions are replaced with minimally broad versions, or removed.
+        """
+        for e in self.exception_list:
+            p = self.used_exception_for.get(e.key())
+            if p is None:
+                # This exception wasn't needed at all.
+                continue
+            if e.is_worse_than(p):
+                # The exception is worse than the problem we found.
+                # Yield the problem as the new exception value.
+                yield p
+            else:
+                # The problem is as bad as the exception, or worse.
+                # Yield the exception.
+                yield e
+
+    def set_tolerances(self, fns):
+        """Adjust the tolerances for the exceptions in this vault.  Takes
+           a map of problem type to a function that adjusts the permitted
+           function to its new maximum value."""
+        for k in self.exceptions:
+            ex = self.exceptions[k]
+            fn = fns.get(ex.problem_type)
+            if fn is not None:
+                ex.metric_value = fn(ex.metric_value)
+
+class ProblemFilter(object):
+    def __init__(self):
+        self.thresholds = dict()
+
+    def addThreshold(self, item):
+        self.thresholds[(item.get_type(),item.get_file_type())] = item
+
+    def matches(self, item):
+        key = (item.get_type(), item.get_file_type())
+        filt = self.thresholds.get(key, None)
+        if filt is None:
+            return False
+        return item.is_worse_than(filt)
+
+    def filter(self, sequence):
+        for item in iter(sequence):
+            if self.matches(item):
+                yield item
+
+class Item(object):
+    """
+    A generic measurement about some aspect of our source code. See
+    the subclasses below for the specific problems we are trying to tackle.
+    """
+    def __init__(self, problem_type, problem_location, metric_value):
+        self.problem_location = problem_location
+        self.metric_value = int(metric_value)
+        self.warning_threshold = self.metric_value
+        self.problem_type = problem_type
+
+    def is_worse_than(self, other_problem):
+        """Return STATUS_ERR if this is a worse problem than other_problem.
+           Return STATUS_WARN if it is a little worse, but falls within the
+           warning threshold.  Return STATUS_OK if this problem is not
+           at all worse than other_problem.
+        """
+        if self.metric_value > other_problem.metric_value:
+            return STATUS_ERR
+        elif self.metric_value > other_problem.warning_threshold:
+            return STATUS_WARN
+        else:
+            return STATUS_OK
+
+    def key(self):
+        """Generate a unique key that describes this problem that can be used as a dictionary key"""
+        # Item location is a filesystem path, so we need to normalize this
+        # across platforms otherwise same paths are not gonna match.
+        canonical_location = os.path.normcase(self.problem_location)
+        return "%s:%s" % (canonical_location, self.problem_type)
+
+    def __str__(self):
+        return "problem %s %s %s" % (self.problem_type, self.problem_location, self.metric_value)
+
+    def get_type(self):
+        return self.problem_type
+
+    def get_file_type(self):
+        if self.problem_location.endswith(".h"):
+            return "*.h"
+        else:
+            return "*.c"
+
+class FileSizeItem(Item):
+    """
+    Denotes a problem with the size of a .c file.
+
+    The 'problem_location' is the filesystem path of the .c file, and the
+    'metric_value' is the number of lines in the .c file.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(FileSizeItem, self).__init__("file-size", problem_location, metric_value)
+
+class IncludeCountItem(Item):
+    """
+    Denotes a problem with the number of #includes in a .c file.
+
+    The 'problem_location' is the filesystem path of the .c file, and the
+    'metric_value' is the number of #includes in the .c file.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(IncludeCountItem, self).__init__("include-count", problem_location, metric_value)
+
+class FunctionSizeItem(Item):
+    """
+    Denotes a problem with a size of a function in a .c file.
+
+    The 'problem_location' is "<path>:<function>()" where <path> is the
+    filesystem path of the .c file and <function> is the name of the offending
+    function.
+
+    The 'metric_value' is the size of the offending function in lines.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(FunctionSizeItem, self).__init__("function-size", problem_location, metric_value)
+
+class DependencyViolationItem(Item):
+    """
+    Denotes a dependency violation in a .c or .h file.  A dependency violation
+    occurs when a file includes a file from some module that is not listed
+    in its .may_include file.
+
+    The 'problem_location' is the file that contains the problem.
+
+    The 'metric_value' is the number of forbidden includes.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(DependencyViolationItem, self).__init__("dependency-violation",
+                                                      problem_location,
+                                                      metric_value)
+
+comment_re = re.compile(r'#.*$')
+
+def get_old_problem_from_exception_str(exception_str):
+    orig_str = exception_str
+    exception_str = comment_re.sub("", exception_str)
+    fields = exception_str.split()
+    if len(fields) == 0:
+        # empty line or comment
+        return None
+    elif len(fields) == 4:
+        # valid line
+        _, problem_type, problem_location, metric_value = fields
+    else:
+        raise ValueError("Misformatted line {!r}".format(orig_str))
+
+    if problem_type == "file-size":
+        return FileSizeItem(problem_location, metric_value)
+    elif problem_type == "include-count":
+        return IncludeCountItem(problem_location, metric_value)
+    elif problem_type == "function-size":
+        return FunctionSizeItem(problem_location, metric_value)
+    elif problem_type == "dependency-violation":
+        return DependencyViolationItem(problem_location, metric_value)
+    else:
+        raise ValueError("Unknown exception type {!r}".format(orig_str))
diff --git a/scripts/maint/practracker/test_practracker.sh b/scripts/maint/practracker/test_practracker.sh
new file mode 100755
index 0000000000..e29b9106de
--- /dev/null
+++ b/scripts/maint/practracker/test_practracker.sh
@@ -0,0 +1,96 @@
+#!/bin/sh
+
+# Fail if any subprocess fails unexpectedly
+set -e
+
+umask 077
+unset TOR_DISABLE_PRACTRACKER
+
+TMPDIR=""
+clean () {
+  if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then
+    rm -rf "$TMPDIR"
+  fi
+}
+trap clean EXIT HUP INT TERM
+
+if test "${PRACTRACKER_DIR}" = "" ||
+        test ! -e "${PRACTRACKER_DIR}/practracker.py" ; then
+    PRACTRACKER_DIR=$(dirname "$0")
+fi
+
+# Change to the tor directory, and canonicalise PRACTRACKER_DIR,
+# so paths in practracker output are consistent, even in out-of-tree builds
+cd "${PRACTRACKER_DIR}"/../../..
+PRACTRACKER_DIR="scripts/maint/practracker"
+
+TMPDIR="$(mktemp -d -t pracktracker.test.XXXXXX)"
+if test -z "${TMPDIR}" || test ! -d "${TMPDIR}" ; then
+    echo >&2 "mktemp failed."
+    exit 1;
+fi
+
+DATA="${PRACTRACKER_DIR}/testdata"
+
+run_practracker() {
+    "${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker.py" \
+        --include-dir "" \
+        --max-file-size=0 \
+        --max-function-size=0 \
+        --max-h-file-size=0 \
+        --max-h-include-count=0 \
+        --max-include-count=0 \
+        --terse \
+        "${DATA}/" "$@" || echo "practracker exit status: $?"
+}
+compare() {
+    # we can't use cmp because we need to use -b for windows
+    diff -b -u "$@" > "${TMPDIR}/test-diff" || true
+    if test -z "$(cat "${TMPDIR}"/test-diff)"; then
+        echo "OK"
+    else
+        cat "${TMPDIR}/test-diff"
+        echo "FAILED"
+        exit 1
+    fi
+}
+
+echo "unit tests:"
+
+"${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker_tests.py"
+
+echo "ex0:"
+
+run_practracker --exceptions "${DATA}/ex0.txt" \
+                > "${TMPDIR}/ex0-received.txt" 2>&1
+
+compare "${TMPDIR}/ex0-received.txt" \
+        "${DATA}/ex0-expected.txt"
+
+echo "ex1:"
+
+run_practracker --exceptions "${DATA}/ex1.txt" \
+                > "${TMPDIR}/ex1-received.txt" 2>&1
+
+compare "${TMPDIR}/ex1-received.txt" \
+        "${DATA}/ex1-expected.txt"
+
+echo "ex1.overbroad:"
+
+run_practracker --exceptions "${DATA}/ex1.txt" --list-overbroad \
+                > "${TMPDIR}/ex1-overbroad-received.txt" 2>&1
+
+compare "${TMPDIR}/ex1-overbroad-received.txt" \
+        "${DATA}/ex1-overbroad-expected.txt"
+
+echo "ex1.regen:"
+
+cp "${DATA}/ex1.txt" "${TMPDIR}/ex1-copy.txt"
+run_practracker --exceptions "${TMPDIR}/ex1-copy.txt" --regen >/dev/null 2>&1
+compare "${TMPDIR}/ex1-copy.txt" "${DATA}/ex1-regen-expected.txt"
+
+echo "ex1.regen_overbroad:"
+
+cp "${DATA}/ex1.txt" "${TMPDIR}/ex1-copy.txt"
+run_practracker --exceptions "${TMPDIR}/ex1-copy.txt" --regen-overbroad >/dev/null 2>&1
+compare "${TMPDIR}/ex1-copy.txt" "${DATA}/ex1-regen-overbroad-expected.txt"
diff --git a/scripts/maint/practracker/testdata/.may_include b/scripts/maint/practracker/testdata/.may_include
new file mode 100644
index 0000000000..8542a35807
--- /dev/null
+++ b/scripts/maint/practracker/testdata/.may_include
@@ -0,0 +1,4 @@
+!advisory
+
+permitted.h
+ext/good.c
diff --git a/scripts/maint/practracker/testdata/a.c b/scripts/maint/practracker/testdata/a.c
new file mode 100644
index 0000000000..3c338ab40d
--- /dev/null
+++ b/scripts/maint/practracker/testdata/a.c
@@ -0,0 +1,41 @@
+
+#include "one.h"
+#include "two.h"
+#incldue "three.h"
+
+# include "permitted.h"
+
+#include "ext/good.c"
+#include "bad.c"
+
+int
+i_am_a_function(void)
+{
+  call();
+  call();
+  /* comment
+
+     another */
+
+  return 3;
+}
+
+#	include  "five.h"
+
+long
+another_function(long x,
+                 long y)
+{
+  int abcd;
+
+  abcd = x+y;
+  abcd *= abcd;
+
+  /* comment here */
+
+  return abcd +
+    abcd +
+    abcd;
+}
+
+/* And a comment to grow! */
diff --git a/scripts/maint/practracker/testdata/b.c b/scripts/maint/practracker/testdata/b.c
new file mode 100644
index 0000000000..bef277aaae
--- /dev/null
+++ b/scripts/maint/practracker/testdata/b.c
@@ -0,0 +1,15 @@
+
+MOCK_IMPL(int,
+foo,(void))
+{
+  // blah1
+  return 0;
+}
+
+MOCK_IMPL(int,
+bar,( long z))
+{
+  // blah2
+
+  return (int)(z+2);
+}
diff --git a/scripts/maint/practracker/testdata/ex.txt b/scripts/maint/practracker/testdata/ex.txt
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex.txt
diff --git a/scripts/maint/practracker/testdata/ex0-expected.txt b/scripts/maint/practracker/testdata/ex0-expected.txt
new file mode 100644
index 0000000000..c9fb83bac3
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex0-expected.txt
@@ -0,0 +1,13 @@
+Unusual pattern permitted.h in scripts/maint/practracker/testdata
+problem file-size a.c 41
+problem include-count a.c 6
+problem function-size a.c:i_am_a_function() 9
+problem function-size a.c:another_function() 12
+problem dependency-violation a.c 4
+problem file-size b.c 15
+problem function-size b.c:foo() 4
+problem function-size b.c:bar() 5
+problem file-size header.h 8
+problem include-count header.h 4
+problem dependency-violation header.h 3
+practracker exit status: 11
diff --git a/scripts/maint/practracker/testdata/ex0.txt b/scripts/maint/practracker/testdata/ex0.txt
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex0.txt
diff --git a/scripts/maint/practracker/testdata/ex1-expected.txt b/scripts/maint/practracker/testdata/ex1-expected.txt
new file mode 100644
index 0000000000..2713338ae4
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1-expected.txt
@@ -0,0 +1,5 @@
+Unusual pattern permitted.h in scripts/maint/practracker/testdata
+problem function-size a.c:i_am_a_function() 9
+(warning) problem function-size a.c:another_function() 12
+problem function-size b.c:foo() 4
+practracker exit status: 2
diff --git a/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt
new file mode 100644
index 0000000000..5ca480dc04
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt
@@ -0,0 +1,4 @@
+Unusual pattern permitted.h in scripts/maint/practracker/testdata
+problem file-size a.c 45 -> 41
+problem file-size z.c 100 -> 0
+practracker exit status: 3
diff --git a/scripts/maint/practracker/testdata/ex1-regen-expected.txt b/scripts/maint/practracker/testdata/ex1-regen-expected.txt
new file mode 100644
index 0000000000..bdf3681edf
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1-regen-expected.txt
@@ -0,0 +1,46 @@
+# Welcome to the exceptions file for Tor's best-practices tracker!
+#
+# Each line of this file represents a single violation of Tor's best
+# practices -- typically, a violation that we had before practracker.py
+# first existed.
+#
+# There are three kinds of problems that we recognize right now:
+#   function-size -- a function of more than 100 lines.
+#   file-size -- a .c file of more than 3000 lines, or a .h
+#      file with more than 500 lines.
+#   include-count -- a .c file with more than 50 #includes,
+#      or a .h file with more than 15 #includes.
+#   dependency-violation -- a file includes a header that it should
+#      not, according to an advisory .may_include file.
+#
+# Each line below represents a single exception that practracker should
+# _ignore_. Each line has four parts:
+#  1. The word "problem".
+#  2. The kind of problem.
+#  3. The location of the problem: either a filename, or a
+#     filename:functionname pair.
+#  4. The magnitude of the problem to ignore.
+#
+# So for example, consider this line:
+#    problem file-size /src/core/or/connection_or.c 3200
+#
+# It tells practracker to allow the mentioned file to be up to 3200 lines
+# long, even though ordinarily it would warn about any file with more than
+# 3000 lines.
+#
+# You can either edit this file by hand, or regenerate it completely by
+# running `make practracker-regen`.
+#
+# Remember: It is better to fix the problem than to add a new exception!
+
+problem file-size a.c 41
+problem include-count a.c 6
+problem function-size a.c:i_am_a_function() 9
+problem function-size a.c:another_function() 12
+problem dependency-violation a.c 4
+problem file-size b.c 15
+problem function-size b.c:foo() 4
+problem function-size b.c:bar() 5
+problem file-size header.h 8
+problem include-count header.h 4
+problem dependency-violation header.h 3
diff --git a/scripts/maint/practracker/testdata/ex1-regen-overbroad-expected.txt b/scripts/maint/practracker/testdata/ex1-regen-overbroad-expected.txt
new file mode 100644
index 0000000000..4521029b10
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1-regen-overbroad-expected.txt
@@ -0,0 +1,45 @@
+# Welcome to the exceptions file for Tor's best-practices tracker!
+#
+# Each line of this file represents a single violation of Tor's best
+# practices -- typically, a violation that we had before practracker.py
+# first existed.
+#
+# There are three kinds of problems that we recognize right now:
+#   function-size -- a function of more than 100 lines.
+#   file-size -- a .c file of more than 3000 lines, or a .h
+#      file with more than 500 lines.
+#   include-count -- a .c file with more than 50 #includes,
+#      or a .h file with more than 15 #includes.
+#   dependency-violation -- a file includes a header that it should
+#      not, according to an advisory .may_include file.
+#
+# Each line below represents a single exception that practracker should
+# _ignore_. Each line has four parts:
+#  1. The word "problem".
+#  2. The kind of problem.
+#  3. The location of the problem: either a filename, or a
+#     filename:functionname pair.
+#  4. The magnitude of the problem to ignore.
+#
+# So for example, consider this line:
+#    problem file-size /src/core/or/connection_or.c 3200
+#
+# It tells practracker to allow the mentioned file to be up to 3200 lines
+# long, even though ordinarily it would warn about any file with more than
+# 3000 lines.
+#
+# You can either edit this file by hand, or regenerate it completely by
+# running `make practracker-regen`.
+#
+# Remember: It is better to fix the problem than to add a new exception!
+
+problem file-size a.c 41
+problem include-count a.c 6
+problem function-size a.c:i_am_a_function() 8
+problem function-size a.c:another_function() 11
+problem file-size b.c 15
+problem function-size b.c:bar() 5
+problem dependency-violation a.c 4
+problem dependency-violation header.h 3
+problem file-size header.h 8
+problem include-count header.h 4
diff --git a/scripts/maint/practracker/testdata/ex1.txt b/scripts/maint/practracker/testdata/ex1.txt
new file mode 100644
index 0000000000..af8de03291
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1.txt
@@ -0,0 +1,18 @@
+
+problem file-size a.c 45
+problem include-count a.c 6
+# this problem will produce an error
+problem function-size a.c:i_am_a_function() 8
+# this problem will produce a warning
+problem function-size a.c:another_function() 11
+problem file-size b.c 15
+# This is removed, and so will produce an error.
+# problem function-size b.c:foo() 4
+# This exception isn't used.
+problem file-size z.c 100
+
+problem function-size b.c:bar() 5
+problem dependency-violation a.c 4
+problem dependency-violation header.h 3
+problem file-size header.h 8
+problem include-count header.h 4
diff --git a/scripts/maint/practracker/testdata/header.h b/scripts/maint/practracker/testdata/header.h
new file mode 100644
index 0000000000..1183f5db9a
--- /dev/null
+++ b/scripts/maint/practracker/testdata/header.h
@@ -0,0 +1,8 @@
+
+// some forbidden includes
+#include "foo.h"
+#include "quux.h"
+#include "quup.h"
+
+// a permitted include
+#include "permitted.h"
diff --git a/scripts/maint/practracker/testdata/not_c_file b/scripts/maint/practracker/testdata/not_c_file
new file mode 100644
index 0000000000..e150962c02
--- /dev/null
+++ b/scripts/maint/practracker/testdata/not_c_file
@@ -0,0 +1,2 @@
+
+This isn't a C file, so practracker shouldn't care about it.
diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py
new file mode 100644
index 0000000000..c52ca2fbbf
--- /dev/null
+++ b/scripts/maint/practracker/util.py
@@ -0,0 +1,61 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+
+# We don't want to run metrics for unittests, automatically-generated C files,
+# external libraries or git leftovers.
+EXCLUDE_SOURCE_DIRS = {"src/test/", "src/trunnel/", "src/rust/",
+                       "src/ext/" }
+
+EXCLUDE_FILES = {"orconfig.h"}
+
+def _norm(p):
+    return os.path.normcase(os.path.normpath(p))
+
+def get_tor_c_files(tor_topdir, include_dirs=None):
+    """
+    Return a list with the .c and .h filenames we want to get metrics of.
+    """
+    files_list = []
+    exclude_dirs = { _norm(os.path.join(tor_topdir, p)) for p in EXCLUDE_SOURCE_DIRS }
+
+    if include_dirs is None:
+        topdirs = [ tor_topdir ]
+    else:
+        topdirs = [ os.path.join(tor_topdir, inc) for inc in include_dirs ]
+
+    for topdir in topdirs:
+        for root, directories, filenames in os.walk(topdir):
+            # Remove all the directories that are excluded.
+            directories[:] = [ d for d in directories
+                               if _norm(os.path.join(root,d)) not in exclude_dirs ]
+            directories.sort()
+            filenames.sort()
+            for filename in filenames:
+                # We only care about .c and .h files
+                if not (filename.endswith(".c") or filename.endswith(".h")):
+                    continue
+                if filename in EXCLUDE_FILES:
+                    continue
+                # Avoid editor temporary files
+                bname = os.path.basename(filename)
+                if bname.startswith("."):
+                    continue
+                if bname.startswith("#"):
+                    continue
+
+                full_path = os.path.join(root,filename)
+
+                files_list.append(full_path)
+
+    return files_list
+
+class NullFile:
+    """A file-like object that we can us to suppress output."""
+    def __init__(self):
+        pass
+    def write(self, s):
+        pass
diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py
index 401fadae6d..6c7b252535 100755
--- a/scripts/maint/rectify_include_paths.py
+++ b/scripts/maint/rectify_include_paths.py
@@ -1,8 +1,17 @@
-#!/usr/bin/python3
+#!/usr/bin/env python
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import os.path
 import re
+import sys
+
+def warn(msg):
+    sys.stderr.write("WARNING: %s\n"%msg)
 
 # Find all the include files, map them to their real names.
 
@@ -11,6 +20,8 @@ def exclude(paths, dirnames):
         if p in dirnames:
             dirnames.remove(p)
 
+DUPLICATE = object()
+
 def get_include_map():
     includes = { }
 
@@ -18,8 +29,17 @@ def get_include_map():
         exclude(["ext", "win32"], dirnames)
 
         for fname in fnames:
+            # Avoid editor temporary files
+            if fname.startswith("."):
+                continue
+            if fname.startswith("#"):
+                continue
+
             if fname.endswith(".h"):
-                assert fname not in includes
+                if fname in includes:
+                    warn("Multiple headers named %s"%fname)
+                    includes[fname] = DUPLICATE
+                    continue
                 include = os.path.join(dirpath, fname)
                 assert include.startswith("src/")
                 includes[fname] = include[4:]
@@ -37,7 +57,7 @@ def fix_includes(inp, out, mapping):
         if m:
             include,hdr,rest = m.groups()
             basehdr = get_base_header_name(hdr)
-            if basehdr in mapping:
+            if basehdr in mapping and mapping[basehdr] is not DUPLICATE:
                 out.write('{}{}{}\n'.format(include,mapping[basehdr],rest))
                 continue
 
@@ -49,6 +69,12 @@ for dirpath,dirnames,fnames in os.walk("src"):
     exclude(["trunnel"], dirnames)
 
     for fname in fnames:
+        # Avoid editor temporary files
+        if fname.startswith("."):
+            continue
+        if fname.startswith("#"):
+            continue
+
         if fname.endswith(".c") or fname.endswith(".h"):
             fname = os.path.join(dirpath, fname)
             tmpfile = fname+".tmp"
diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py
index 203cce0107..12b02c8a44 100755
--- a/scripts/maint/redox.py
+++ b/scripts/maint/redox.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 #  Copyright (c) 2008-2019, The Tor Project, Inc.
 #  See LICENSE for licensing information.
@@ -29,6 +29,19 @@
 #     "mv fname.c.newdoc fname.c".  Otherwise, you'll need to merge
 #     the parts you like by hand.
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import re
+import sys
+
+try:
+    xrange  # Python 2
+except NameError:
+    xrange = range  # Python 3
+
 # Which files should we ignore warning from?  Mostly, these are external
 # files that we've snarfed in from somebody else, whose C we do no intend
 # to document for them.
@@ -52,9 +65,6 @@ ADD_DOCDOCS_TO_TYPES += [ 'variable', ]
 # ====================
 # The rest of this should not need hacking.
 
-import re
-import sys
-
 KINDS = [ "type", "field", "typedef", "define", "function", "variable",
           "enumeration" ]
 
@@ -73,7 +83,7 @@ def parsething(thing):
     else:
         m = THING_RE.match(thing)
         if not m:
-            print thing, "???? Format didn't match."
+            print(thing, "???? Format didn't match.")
             return None, None
         else:
             name, tp, parent = m.groups()
@@ -150,7 +160,7 @@ def checkf(fn, errs):
     """
     for skip in SKIP_FILES:
         if fn.endswith(skip):
-            print "Skipping",fn
+            print("Skipping",fn)
             return
 
     comments = []
@@ -169,8 +179,8 @@ def checkf(fn, errs):
 
         ln = findline(lines, line, name)
         if ln == None:
-            print "Couldn't find the definition of %s allegedly on %s of %s"%(
-                name, line, fn)
+            print("Couldn't find the definition of %s allegedly on %s of %s"%(
+                name, line, fn))
         else:
             if hasdocdoc(lines, line, kind):
 #                print "Has a DOCDOC"
@@ -215,12 +225,12 @@ def applyComments(fn, entries):
         outf.write(line)
     outf.close()
 
-    print "Added %s DOCDOCs to %s" %(N, fn)
+    print("Added %s DOCDOCs to %s" %(N, fn))
 
 e = read()
 
 for fn, errs in e.iteritems():
-    print `(fn, errs)`
+    print(repr((fn, errs)))
     comments = checkf(fn, errs)
     if comments:
         applyComments(fn, comments)
diff --git a/scripts/maint/rename_c_identifier.py b/scripts/maint/rename_c_identifier.py
new file mode 100755
index 0000000000..77802e10f3
--- /dev/null
+++ b/scripts/maint/rename_c_identifier.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2001 Matej Pfajfar.
+# Copyright (c) 2001-2004, Roger Dingledine.
+# Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+# Copyright (c) 2007-2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+"""
+Helpful script to replace one or more C identifiers, and optionally
+generate a commit message explaining what happened.
+"""
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import fileinput
+import os
+import re
+import shlex
+import subprocess
+import sys
+import tempfile
+
+TOPDIR = "src"
+
+
+def is_c_file(fn):
+    """
+       Return true iff fn is the name of a C file.
+
+       >>> is_c_file("a/b/module.c")
+       True
+       >>> is_c_file("a/b/module.h")
+       True
+       >>> is_c_file("a/b/module.c~")
+       False
+       >>> is_c_file("a/b/.module.c")
+       False
+       >>> is_c_file("a/b/module.cpp")
+       False
+    """
+    fn = os.path.split(fn)[1]
+    # Avoid editor temporary files
+    if fn.startswith(".") or fn.startswith("#"):
+        return False
+    ext = os.path.splitext(fn)[1]
+    return ext in {".c", ".h", ".i", ".inc"}
+
+
+def list_c_files(topdir=TOPDIR):
+    """
+       Use git to list all the C files under version control.
+
+       >>> lst = list(list_c_files())
+       >>> "src/core/mainloop/mainloop.c" in lst
+       True
+       >>> "src/core/mainloop/twiddledeedoo.c" in lst
+       False
+       >>> "micro-revision.i" in lst
+       False
+    """
+    proc = subprocess.Popen(
+        ["git", "ls-tree", "--name-only", "-r", "HEAD", topdir],
+        stdout=subprocess.PIPE,
+        encoding="utf-8")
+    for line in proc.stdout.readlines():
+        line = line.strip()
+        if is_c_file(line):
+            yield line
+
+
+class Rewriter:
+    """
+       A rewriter applies a series of word-by-word replacements, in
+       sequence.  Replacements only happen at "word boundaries",
+       as determined by the \\b regular expression marker.
+
+       ("A word is defined as a sequence of alphanumeric or underscore
+       characters", according to the documentation.)
+
+       >>> R = Rewriter([("magic", "secret"), ("words", "codes")])
+       >>> R.apply("The magic words are rambunctious bluejay")
+       'The secret codes are rambunctious bluejay'
+       >>> R.apply("The magical words are rambunctious bluejay")
+       'The magical codes are rambunctious bluejay'
+       >>> R.get_count()
+       3
+
+    """
+
+    def __init__(self, replacements):
+        """Make a new Rewriter. Takes a sequence of pairs of
+           (from_id, to_id), where from_id is an identifier to replace,
+           and to_id is its replacement.
+        """
+        self._patterns = []
+        for id1, id2 in replacements:
+            pat = re.compile(r"\b{}\b".format(re.escape(id1)))
+            self._patterns.append((pat, id2))
+
+        self._count = 0
+
+    def apply(self, line):
+        """Return `line` as transformed by this rewriter."""
+        for pat, ident in self._patterns:
+            line, count = pat.subn(ident, line)
+            self._count += count
+        return line
+
+    def get_count(self):
+        """Return the number of identifiers that this rewriter has
+           rewritten."""
+        return self._count
+
+
+def rewrite_files(files, rewriter):
+    """
+       Apply `rewriter` to every file in `files`, replacing those files
+       with their rewritten contents.
+    """
+    for line in fileinput.input(files, inplace=True):
+        sys.stdout.write(rewriter.apply(line))
+
+
+def make_commit_msg(pairs, no_verify):
+    """Return a commit message to explain what was replaced by the provided
+       arguments.
+    """
+    script = ["./scripts/maint/rename_c_identifier.py"]
+    for id1, id2 in pairs:
+        qid1 = shlex.quote(id1)
+        qid2 = shlex.quote(id2)
+        script.append("        {} {}".format(qid1, qid2))
+    script = " \\\n".join(script)
+
+    if len(pairs) == 1:
+        line1 = "Rename {} to {}".format(*pairs[0])
+    else:
+        line1 = "Replace several C identifiers."
+
+    msg = """\
+{}
+
+This is an automated commit, generated by this command:
+
+{}
+""".format(line1, script)
+
+    if no_verify:
+        msg += """
+It was generated with --no-verify, so it probably breaks some commit hooks.
+The commiter should be sure to fix them up in a subsequent commit.
+"""
+
+    return msg
+
+
+def commit(pairs, no_verify=False):
+    """Try to commit the current git state, generating the commit message as
+       appropriate.  If `no_verify` is True, pass the --no-verify argument to
+       git commit.
+    """
+    args = []
+    if no_verify:
+        args.append("--no-verify")
+
+    # We have to use a try block to delete the temporary file here, since we
+    # are using tempfile with delete=False. We have to use delete=False,
+    # since otherwise we are not guaranteed to be able to give the file to
+    # git for it to open.
+    fname = None
+    try:
+        with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
+            fname = f.name
+            f.write(make_commit_msg(pairs, no_verify))
+        s = subprocess.run(["git", "commit", "-a", "-F", fname, "--edit"]+args)
+        if s.returncode != 0 and not no_verify:
+            print('"git commit" failed. Maybe retry with --no-verify?',
+                  file=sys.stderr)
+            revert_changes()
+            return False
+    finally:
+        os.unlink(fname)
+
+    return True
+
+
+def any_uncommitted_changes():
+    """Return True if git says there are any uncommitted changes in the current
+       working tree; false otherwise.
+    """
+    s = subprocess.run(["git", "diff-index", "--quiet", "HEAD"])
+    return s.returncode != 0
+
+
+DESC = "Replace one identifier with another throughout our source."
+EXAMPLES = """\
+Examples:
+
+   rename_c_identifier.py set_ctrl_id set_controller_id
+      (Replaces every occurrence of "set_ctrl_id" with "set_controller_id".)
+
+   rename_c_identifier.py --commit set_ctrl_id set_controller_id
+      (As above, but also generate a git commit with an appropriate message.)
+
+   rename_c_identifier.py a b c d
+      (Replace "a" with "b", and "c" with "d".)"""
+
+
+def revert_changes():
+    """Tell git to revert all the changes in the current working tree.
+    """
+    print('Reverting changes.', file=sys.stderr)
+    subprocess.run(["git", "checkout", "--quiet", TOPDIR])
+
+
+def main(argv):
+    import argparse
+    parser = argparse.ArgumentParser(description=DESC, epilog=EXAMPLES,
+                                     # prevent re-wrapping the examples
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument("--commit", action='store_true',
+                        help="Generate a Git commit.")
+    parser.add_argument("--no-verify", action='store_true',
+                        help="Tell Git not to run its pre-commit hooks.")
+    parser.add_argument("from_id", type=str,  help="Original identifier")
+    parser.add_argument("to_id", type=str, help="New identifier")
+    parser.add_argument("more", type=str, nargs=argparse.REMAINDER,
+                        help="Additional identifier pairs")
+
+    args = parser.parse_args(argv[1:])
+
+    if len(args.more) % 2 != 0:
+        print("I require an even number of identifiers.", file=sys.stderr)
+        return 1
+
+    if any_uncommitted_changes():
+        print("Uncommitted changes found. Not running.", file=sys.stderr)
+        return 1
+
+    pairs = []
+    print("renaming {} to {}".format(args.from_id, args.to_id), file=sys.stderr)
+    pairs.append((args.from_id, args.to_id))
+    for idx in range(0, len(args.more), 2):
+        id1 = args.more[idx]
+        id2 = args.more[idx+1]
+        print("renaming {} to {}".format(id1, id2))
+        pairs.append((id1, id2))
+
+    rewriter = Rewriter(pairs)
+
+    rewrite_files(list_c_files(), rewriter)
+
+    print("Replaced {} identifiers".format(rewriter.get_count()),
+          file=sys.stderr)
+
+    if args.commit:
+        commit(pairs, args.no_verify)
+
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh
index efb8706fea..b0268322f4 100755
--- a/scripts/maint/run_calltool.sh
+++ b/scripts/maint/run_calltool.sh
@@ -15,10 +15,10 @@ SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgr
 
 for calculation in $SUBITEMS; do
   echo "======== $calculation"
-  python -m calltool $calculation > callgraph/$calculation
+  python -m calltool "$calculation" > callgraph/"$calculation"
 done
 
-echo <<EOF > callgraph/README
+cat <<EOF > callgraph/README
 This directory holds output from calltool, as run on Tor.  For more
 information about each of these files, see the NOTES and README files in
 the calltool distribution.
diff --git a/scripts/maint/run_check_subsystem_order.sh b/scripts/maint/run_check_subsystem_order.sh
new file mode 100755
index 0000000000..8e98f1e49c
--- /dev/null
+++ b/scripts/maint/run_check_subsystem_order.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+TOR="${abs_top_builddir:-.}/src/app/tor"
+
+INCLUDES_PY="${abs_top_srcdir:-.}/scripts/maint/practracker/includes.py"
+
+if ! test -x "${INCLUDES_PY}" ; then
+    echo "skip"
+    exit 77
+fi
+
+"${TOR}" --dbg-dump-subsystem-list | \
+    "${PYTHON:-python}" \
+    "${INCLUDES_PY}" --check-subsystem-order - "${abs_top_srcdir}/src"
+
+echo ok
diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py
index 986b94b025..5f6324e387 100755
--- a/scripts/maint/sortChanges.py
+++ b/scripts/maint/sortChanges.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Copyright (c) 2014-2019, The Tor Project, Inc.
 # See LICENSE for licensing information
 
@@ -7,6 +7,11 @@
    changelog.
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import re
 import sys
 
@@ -43,7 +48,7 @@ REPLACEMENTS = {
 def score(s,fname=None):
     m = re.match(r'^ +o ([^\n]*)\n(.*)', s, re.M|re.S)
     if not m:
-        print >>sys.stderr, "Can't score %r from %s"%(s,fname)
+        print("Can't score %r from %s"%(s,fname), file=sys.stderr)
     heading = m.group(1)
     heading = REPLACEMENTS.get(heading, heading)
     lw = m.group(1).lower()
@@ -100,9 +105,9 @@ changes.sort()
 last_lw = "this is not a header"
 for _, lw, header, rest in changes:
     if lw == last_lw:
-        print rest,
+        print(rest, end="")
     else:
-        print
-        print "  o",header
-        print rest,
+        print()
+        print("  o",header)
+        print(rest, end="")
         last_lw = lw
diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl
index bd24377d38..6800032f87 100755
--- a/scripts/maint/updateCopyright.pl
+++ b/scripts/maint/updateCopyright.pl
@@ -1,7 +1,9 @@
 #!/usr/bin/perl -i -w -p
 
-$NEWYEAR=2018;
+@now = gmtime();
 
-s/Copyright(.*) (201[^8]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
+$NEWYEAR=$now[5]+1900;
+
+s/Copyright([^-]*) (20[^-]*), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
 
 s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/;
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
deleted file mode 100755
index 0ea3992d8f..0000000000
--- a/scripts/maint/updateFallbackDirs.py
+++ /dev/null
@@ -1,2216 +0,0 @@
-#!/usr/bin/env python
-
-# Usage:
-#
-# Regenerate the list:
-# scripts/maint/updateFallbackDirs.py > src/app/config/fallback_dirs.inc 2> fallback_dirs.log
-#
-# Check the existing list:
-# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log
-# mv fallback_dirs.inc.ok src/app/config/fallback_dirs.inc
-#
-# This script should be run from a stable, reliable network connection,
-# with no other network activity (and not over tor).
-# If this is not possible, please disable:
-# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS
-#
-# Needs dateutil, stem, and potentially other python packages.
-# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package)
-# for netblock analysis.
-#
-# Then read the logs to make sure the fallbacks aren't dominated by a single
-# netblock or port.
-
-# Script by weasel, April 2015
-# Portions by gsathya & karsten, 2013
-# https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.py
-# Modifications by teor, 2015
-
-import StringIO
-import string
-import re
-import datetime
-import gzip
-import os.path
-import json
-import math
-import sys
-import urllib
-import urllib2
-import hashlib
-import dateutil.parser
-# bson_lazy provides bson
-#from bson import json_util
-import copy
-import re
-
-from stem.descriptor import DocumentHandler
-from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS
-
-import logging
-logging.root.name = ''
-
-HAVE_IPADDRESS = False
-try:
-  # python 3 builtin, or install package py2-ipaddress
-  # there are several ipaddress implementations for python 2
-  # with slightly different semantics with str typed text
-  # fortunately, all our IP addresses are in unicode
-  import ipaddress
-  HAVE_IPADDRESS = True
-except ImportError:
-  # if this happens, we avoid doing netblock analysis
-  logging.warning('Unable to import ipaddress, please install py2-ipaddress.' +
-                  ' A fallback list will be created, but optional netblock' +
-                  ' analysis will not be performed.')
-
-## Top-Level Configuration
-
-# We use semantic versioning: https://semver.org
-# In particular:
-# * major changes include removing a mandatory field, or anything else that
-#   would break an appropriately tolerant parser,
-# * minor changes include adding a field,
-# * patch changes include changing header comments or other unstructured
-#   content
-FALLBACK_FORMAT_VERSION = '2.0.0'
-SECTION_SEPARATOR_BASE = '====='
-SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */'
-
-# Output all candidate fallbacks, or only output selected fallbacks?
-OUTPUT_CANDIDATES = False
-
-# Perform DirPort checks over IPv4?
-# Change this to False if IPv4 doesn't work for you, or if you don't want to
-# download a consensus for each fallback
-# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
-PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
-
-# Perform DirPort checks over IPv6?
-# If you know IPv6 works for you, set this to True
-# This will exclude IPv6 relays without an IPv6 DirPort configured
-# So it's best left at False until #18394 is implemented
-# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
-PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
-
-# Must relays be running now?
-MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS
-                       or PERFORM_IPV6_DIRPORT_CHECKS)
-
-# Clients have been using microdesc consensuses by default for a while now
-DOWNLOAD_MICRODESC_CONSENSUS = True
-
-# If a relay delivers an expired consensus, if it expired less than this many
-# seconds ago, we still allow the relay. This should never be less than -90,
-# as all directory mirrors should have downloaded a consensus 90 minutes
-# before it expires. It should never be more than 24 hours, because clients
-# reject consensuses that are older than REASONABLY_LIVE_TIME.
-# For the consensus expiry check to be accurate, the machine running this
-# script needs an accurate clock.
-#
-# Relays on 0.3.0 and later return a 404 when they are about to serve an
-# expired consensus. This makes them fail the download check.
-# We use a tolerance of 0, so that 0.2.x series relays also fail the download
-# check if they serve an expired consensus.
-CONSENSUS_EXPIRY_TOLERANCE = 0
-
-# Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
-OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
-
-# Output matching ContactInfo in fallbacks list?
-# Useful if you're trying to contact operators
-CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
-
-# How the list should be sorted:
-# fingerprint: is useful for stable diffs of fallback lists
-# measured_bandwidth: is useful when pruning the list based on bandwidth
-# contact: is useful for contacting operators once the list has been pruned
-OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint'
-
-## OnionOO Settings
-
-ONIONOO = 'https://onionoo.torproject.org/'
-#ONIONOO = 'https://onionoo.thecthulhu.com/'
-
-# Don't bother going out to the Internet, just use the files available locally,
-# even if they're very old
-LOCAL_FILES_ONLY = False
-
-## Whitelist / Blacklist Filter Settings
-
-# The whitelist contains entries that are included if all attributes match
-# (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport)
-
-# What happens to entries not in whitelist?
-# When True, they are included, when False, they are excluded
-INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False
-
-WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist'
-FALLBACK_FILE_NAME  = 'src/app/config/fallback_dirs.inc'
-
-# The number of bytes we'll read from a filter file before giving up
-MAX_LIST_FILE_SIZE = 1024 * 1024
-
-## Eligibility Settings
-
-# Require fallbacks to have the same address and port for a set amount of time
-# We used to have this at 1 week, but that caused many fallback failures, which
-# meant that we had to rebuild the list more often. We want fallbacks to be
-# stable for 2 years, so we set it to a few months.
-#
-# If a relay changes address or port, that's it, it's not useful any more,
-# because clients can't find it
-ADDRESS_AND_PORT_STABLE_DAYS = 90
-# We ignore relays that have been down for more than this period
-MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
-# FallbackDirs must have a time-weighted-fraction that is greater than or
-# equal to:
-# Mirrors that are down half the time are still useful half the time
-CUTOFF_RUNNING = .50
-CUTOFF_V2DIR = .50
-# Guard flags are removed for some time after a relay restarts, so we ignore
-# the guard flag.
-CUTOFF_GUARD = .00
-# FallbackDirs must have a time-weighted-fraction that is less than or equal
-# to:
-# .00 means no bad exits
-PERMITTED_BADEXIT = .00
-
-# older entries' weights are adjusted with ALPHA^(age in days)
-AGE_ALPHA = 0.99
-
-# this factor is used to scale OnionOO entries to [0,1]
-ONIONOO_SCALE_ONE = 999.
-
-## Fallback Count Limits
-
-# The target for these parameters is 20% of the guards in the network
-# This is around 200 as of October 2015
-_FB_POG = 0.2
-FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
-
-# Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
-MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
-# Emit a C #error if the number of fallbacks is less than expected
-MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
-
-# The maximum number of fallbacks on the same address, contact, or family
-#
-# With 150 fallbacks, this means each operator sees 5% of client bootstraps.
-# For comparison:
-#  - We try to limit guard and exit operators to 5% of the network
-#  - The directory authorities used to see 11% of client bootstraps each
-#
-# We also don't want too much of the list to go down if a single operator
-# has to move all their relays.
-MAX_FALLBACKS_PER_IP = 1
-MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP
-MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP
-MAX_FALLBACKS_PER_CONTACT = 7
-MAX_FALLBACKS_PER_FAMILY = 7
-
-## Fallback Bandwidth Requirements
-
-# Any fallback with the Exit flag has its bandwidth multiplied by this fraction
-# to make sure we aren't further overloading exits
-# (Set to 1.0, because we asked that only lightly loaded exits opt-in,
-# and the extra load really isn't that much for large relays.)
-EXIT_BANDWIDTH_FRACTION = 1.0
-
-# If a single fallback's bandwidth is too low, it's pointless adding it
-# We expect fallbacks to handle an extra 10 kilobytes per second of traffic
-# Make sure they can support fifty times the expected extra load
-#
-# We convert this to a consensus weight before applying the filter,
-# because all the bandwidth amounts are specified by the relay
-MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0
-
-# Clients will time out after 30 seconds trying to download a consensus
-# So allow fallback directories half that to deliver a consensus
-# The exact download times might change based on the network connection
-# running this script, but only by a few seconds
-# There is also about a second of python overhead
-CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0
-# If the relay fails a consensus check, retry the download
-# This avoids delisting a relay due to transient network conditions
-CONSENSUS_DOWNLOAD_RETRY = True
-
-## Parsing Functions
-
-def parse_ts(t):
-  return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
-
-def remove_bad_chars(raw_string, bad_char_list):
-  # Remove each character in the bad_char_list
-  cleansed_string = raw_string
-  for c in bad_char_list:
-    cleansed_string = cleansed_string.replace(c, '')
-  return cleansed_string
-
-def cleanse_unprintable(raw_string):
-  # Remove all unprintable characters
-  cleansed_string = ''
-  for c in raw_string:
-    if c in string.printable:
-      cleansed_string += c
-  return cleansed_string
-
-def cleanse_whitespace(raw_string):
-  # Replace all whitespace characters with a space
-  cleansed_string = raw_string
-  for c in string.whitespace:
-    cleansed_string = cleansed_string.replace(c, ' ')
-  return cleansed_string
-
-def cleanse_c_multiline_comment(raw_string):
-  cleansed_string = raw_string
-  # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
-  cleansed_string = cleanse_whitespace(cleansed_string)
-  # ContactInfo and Version can be arbitrary binary data
-  cleansed_string = cleanse_unprintable(cleansed_string)
-  # Prevent a malicious / unanticipated string from breaking out
-  # of a C-style multiline comment
-  # This removes '/*' and '*/' and '//'
-  bad_char_list = '*/'
-  # Prevent a malicious string from using C nulls
-  bad_char_list += '\0'
-  # Avoid confusing parsers by making sure there is only one comma per fallback
-  bad_char_list += ','
-  # Avoid confusing parsers by making sure there is only one equals per field
-  bad_char_list += '='
-  # Be safer by removing bad characters entirely
-  cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
-  # Some compilers may further process the content of comments
-  # There isn't much we can do to cover every possible case
-  # But comment-based directives are typically only advisory
-  return cleansed_string
-
-def cleanse_c_string(raw_string):
-  cleansed_string = raw_string
-  # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
-  cleansed_string = cleanse_whitespace(cleansed_string)
-  # ContactInfo and Version can be arbitrary binary data
-  cleansed_string = cleanse_unprintable(cleansed_string)
-  # Prevent a malicious address/fingerprint string from breaking out
-  # of a C-style string
-  bad_char_list = '"'
-  # Prevent a malicious string from using escapes
-  bad_char_list += '\\'
-  # Prevent a malicious string from using C nulls
-  bad_char_list += '\0'
-  # Avoid confusing parsers by making sure there is only one comma per fallback
-  bad_char_list += ','
-  # Avoid confusing parsers by making sure there is only one equals per field
-  bad_char_list += '='
-  # Be safer by removing bad characters entirely
-  cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
-  # Some compilers may further process the content of strings
-  # There isn't much we can do to cover every possible case
-  # But this typically only results in changes to the string data
-  return cleansed_string
-
-## OnionOO Source Functions
-
-# a dictionary of source metadata for each onionoo query we've made
-fetch_source = {}
-
-# register source metadata for 'what'
-# assumes we only retrieve one document for each 'what'
-def register_fetch_source(what, url, relays_published, version):
-  fetch_source[what] = {}
-  fetch_source[what]['url'] = url
-  fetch_source[what]['relays_published'] = relays_published
-  fetch_source[what]['version'] = version
-
-# list each registered source's 'what'
-def fetch_source_list():
-  return sorted(fetch_source.keys())
-
-# given 'what', provide a multiline C comment describing the source
-def describe_fetch_source(what):
-  desc = '/*'
-  desc += '\n'
-  desc += 'Onionoo Source: '
-  desc += cleanse_c_multiline_comment(what)
-  desc += ' Date: '
-  desc += cleanse_c_multiline_comment(fetch_source[what]['relays_published'])
-  desc += ' Version: '
-  desc += cleanse_c_multiline_comment(fetch_source[what]['version'])
-  desc += '\n'
-  desc += 'URL: '
-  desc += cleanse_c_multiline_comment(fetch_source[what]['url'])
-  desc += '\n'
-  desc += '*/'
-  return desc
-
-## File Processing Functions
-
-def write_to_file(str, file_name, max_len):
-  try:
-    with open(file_name, 'w') as f:
-      f.write(str[0:max_len])
-  except EnvironmentError, error:
-    logging.error('Writing file %s failed: %d: %s'%
-                  (file_name,
-                   error.errno,
-                   error.strerror)
-                  )
-
-def read_from_file(file_name, max_len):
-  try:
-    if os.path.isfile(file_name):
-      with open(file_name, 'r') as f:
-        return f.read(max_len)
-  except EnvironmentError, error:
-    logging.info('Loading file %s failed: %d: %s'%
-                 (file_name,
-                  error.errno,
-                  error.strerror)
-                 )
-  return None
-
-def parse_fallback_file(file_name):
-  file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
-  file_data = cleanse_unprintable(file_data)
-  file_data = remove_bad_chars(file_data, '\n"\0')
-  file_data = re.sub('/\*.*?\*/', '', file_data)
-  file_data = file_data.replace(',', '\n')
-  file_data = file_data.replace(' weight=10', '')
-  return file_data
-
-def load_possibly_compressed_response_json(response):
-    if response.info().get('Content-Encoding') == 'gzip':
-      buf = StringIO.StringIO( response.read() )
-      f = gzip.GzipFile(fileobj=buf)
-      return json.load(f)
-    else:
-      return json.load(response)
-
-def load_json_from_file(json_file_name):
-    # An exception here may be resolved by deleting the .last_modified
-    # and .json files, and re-running the script
-    try:
-      with open(json_file_name, 'r') as f:
-        return json.load(f)
-    except EnvironmentError, error:
-      raise Exception('Reading not-modified json file %s failed: %d: %s'%
-                    (json_file_name,
-                     error.errno,
-                     error.strerror)
-                    )
-
-## OnionOO Functions
-
-def datestr_to_datetime(datestr):
-  # Parse datetimes like: Fri, 02 Oct 2015 13:34:14 GMT
-  if datestr is not None:
-    dt = dateutil.parser.parse(datestr)
-  else:
-    # Never modified - use start of epoch
-    dt = datetime.datetime.utcfromtimestamp(0)
-  # strip any timezone out (in case they're supported in future)
-  dt = dt.replace(tzinfo=None)
-  return dt
-
-def onionoo_fetch(what, **kwargs):
-  params = kwargs
-  params['type'] = 'relay'
-  #params['limit'] = 10
-  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
-  params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
-  params['flag'] = 'V2Dir'
-  url = ONIONOO + what + '?' + urllib.urlencode(params)
-
-  # Unfortunately, the URL is too long for some OS filenames,
-  # but we still don't want to get files from different URLs mixed up
-  base_file_name = what + '-' + hashlib.sha1(url).hexdigest()
-
-  full_url_file_name = base_file_name + '.full_url'
-  MAX_FULL_URL_LENGTH = 1024
-
-  last_modified_file_name = base_file_name + '.last_modified'
-  MAX_LAST_MODIFIED_LENGTH = 64
-
-  json_file_name = base_file_name + '.json'
-
-  if LOCAL_FILES_ONLY:
-    # Read from the local file, don't write to anything
-    response_json = load_json_from_file(json_file_name)
-  else:
-    # store the full URL to a file for debugging
-    # no need to compare as long as you trust SHA-1
-    write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH)
-
-    request = urllib2.Request(url)
-    request.add_header('Accept-encoding', 'gzip')
-
-    # load the last modified date from the file, if it exists
-    last_mod_date = read_from_file(last_modified_file_name,
-                                   MAX_LAST_MODIFIED_LENGTH)
-    if last_mod_date is not None:
-      request.add_header('If-modified-since', last_mod_date)
-
-    # Parse last modified date
-    last_mod = datestr_to_datetime(last_mod_date)
-
-    # Not Modified and still recent enough to be useful
-    # Onionoo / Globe used to use 6 hours, but we can afford a day
-    required_freshness = datetime.datetime.utcnow()
-    # strip any timezone out (to match dateutil.parser)
-    required_freshness = required_freshness.replace(tzinfo=None)
-    required_freshness -= datetime.timedelta(hours=24)
-
-    # Make the OnionOO request
-    response_code = 0
-    try:
-      response = urllib2.urlopen(request)
-      response_code = response.getcode()
-    except urllib2.HTTPError, error:
-      response_code = error.code
-      if response_code == 304: # not modified
-        pass
-      else:
-        raise Exception("Could not get " + url + ": "
-                        + str(error.code) + ": " + error.reason)
-
-    if response_code == 200: # OK
-      last_mod = datestr_to_datetime(response.info().get('Last-Modified'))
-
-    # Check for freshness
-    if last_mod < required_freshness:
-      if last_mod_date is not None:
-        # This check sometimes fails transiently, retry the script if it does
-        date_message = "Outdated data: last updated " + last_mod_date
-      else:
-        date_message = "No data: never downloaded "
-      raise Exception(date_message + " from " + url)
-
-    # Process the data
-    if response_code == 200: # OK
-
-      response_json = load_possibly_compressed_response_json(response)
-
-      with open(json_file_name, 'w') as f:
-        # use the most compact json representation to save space
-        json.dump(response_json, f, separators=(',',':'))
-
-      # store the last modified date in its own file
-      if response.info().get('Last-modified') is not None:
-        write_to_file(response.info().get('Last-Modified'),
-                      last_modified_file_name,
-                      MAX_LAST_MODIFIED_LENGTH)
-
-    elif response_code == 304: # Not Modified
-
-      response_json = load_json_from_file(json_file_name)
-
-    else: # Unexpected HTTP response code not covered in the HTTPError above
-      raise Exception("Unexpected HTTP response code to " + url + ": "
-                      + str(response_code))
-
-  register_fetch_source(what,
-                        url,
-                        response_json['relays_published'],
-                        response_json['version'])
-
-  return response_json
-
-def fetch(what, **kwargs):
-  #x = onionoo_fetch(what, **kwargs)
-  # don't use sort_keys, as the order of or_addresses is significant
-  #print json.dumps(x, indent=4, separators=(',', ': '))
-  #sys.exit(0)
-
-  return onionoo_fetch(what, **kwargs)
-
-## Fallback Candidate Class
-
-class Candidate(object):
-  CUTOFF_ADDRESS_AND_PORT_STABLE = (datetime.datetime.utcnow()
-                            - datetime.timedelta(ADDRESS_AND_PORT_STABLE_DAYS))
-
-  def __init__(self, details):
-    for f in ['fingerprint', 'nickname', 'last_changed_address_or_port',
-              'consensus_weight', 'or_addresses', 'dir_address']:
-      if not f in details: raise Exception("Document has no %s field."%(f,))
-
-    if not 'contact' in details:
-      details['contact'] = None
-    if not 'flags' in details or details['flags'] is None:
-      details['flags'] = []
-    if (not 'advertised_bandwidth' in details
-        or details['advertised_bandwidth'] is None):
-      # relays without advertised bandwidth have it calculated from their
-      # consensus weight
-      details['advertised_bandwidth'] = 0
-    if (not 'effective_family' in details
-        or details['effective_family'] is None):
-      details['effective_family'] = []
-    if not 'platform' in details:
-      details['platform'] = None
-    details['last_changed_address_or_port'] = parse_ts(
-                                      details['last_changed_address_or_port'])
-    self._data = details
-    self._stable_sort_or_addresses()
-
-    self._fpr = self._data['fingerprint']
-    self._running = self._guard = self._v2dir = 0.
-    self._split_dirport()
-    self._compute_orport()
-    if self.orport is None:
-      raise Exception("Failed to get an orport for %s."%(self._fpr,))
-    self._compute_ipv6addr()
-    if not self.has_ipv6():
-      logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
-    self._compute_version()
-    self._extra_info_cache = None
-
-  def _stable_sort_or_addresses(self):
-    # replace self._data['or_addresses'] with a stable ordering,
-    # sorting the secondary addresses in string order
-    # leave the received order in self._data['or_addresses_raw']
-    self._data['or_addresses_raw'] = self._data['or_addresses']
-    or_address_primary = self._data['or_addresses'][:1]
-    # subsequent entries in the or_addresses array are in an arbitrary order
-    # so we stabilise the addresses by sorting them in string order
-    or_addresses_secondaries_stable = sorted(self._data['or_addresses'][1:])
-    or_addresses_stable = or_address_primary + or_addresses_secondaries_stable
-    self._data['or_addresses'] = or_addresses_stable
-
-  def get_fingerprint(self):
-    return self._fpr
-
-  # is_valid_ipv[46]_address by gsathya, karsten, 2013
-  @staticmethod
-  def is_valid_ipv4_address(address):
-    if not isinstance(address, (str, unicode)):
-      return False
-
-    # check if there are four period separated values
-    if address.count(".") != 3:
-      return False
-
-    # checks that each value in the octet are decimal values between 0-255
-    for entry in address.split("."):
-      if not entry.isdigit() or int(entry) < 0 or int(entry) > 255:
-        return False
-      elif entry[0] == "0" and len(entry) > 1:
-        return False  # leading zeros, for instance in "1.2.3.001"
-
-    return True
-
-  @staticmethod
-  def is_valid_ipv6_address(address):
-    if not isinstance(address, (str, unicode)):
-      return False
-
-    # remove brackets
-    address = address[1:-1]
-
-    # addresses are made up of eight colon separated groups of four hex digits
-    # with leading zeros being optional
-    # https://en.wikipedia.org/wiki/IPv6#Address_format
-
-    colon_count = address.count(":")
-
-    if colon_count > 7:
-      return False  # too many groups
-    elif colon_count != 7 and not "::" in address:
-      return False  # not enough groups and none are collapsed
-    elif address.count("::") > 1 or ":::" in address:
-      return False  # multiple groupings of zeros can't be collapsed
-
-    found_ipv4_on_previous_entry = False
-    for entry in address.split(":"):
-      # If an IPv6 address has an embedded IPv4 address,
-      # it must be the last entry
-      if found_ipv4_on_previous_entry:
-        return False
-      if not re.match("^[0-9a-fA-f]{0,4}$", entry):
-        if not Candidate.is_valid_ipv4_address(entry):
-          return False
-        else:
-          found_ipv4_on_previous_entry = True
-
-    return True
-
-  def _split_dirport(self):
-    # Split the dir_address into dirip and dirport
-    (self.dirip, _dirport) = self._data['dir_address'].split(':', 2)
-    self.dirport = int(_dirport)
-
-  def _compute_orport(self):
-    # Choose the first ORPort that's on the same IPv4 address as the DirPort.
-    # In rare circumstances, this might not be the primary ORPort address.
-    # However, _stable_sort_or_addresses() ensures we choose the same one
-    # every time, even if onionoo changes the order of the secondaries.
-    self._split_dirport()
-    self.orport = None
-    for i in self._data['or_addresses']:
-      if i != self._data['or_addresses'][0]:
-        logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i))
-      (ipaddr, port) = i.rsplit(':', 1)
-      if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr):
-        self.orport = int(port)
-        return
-
-  def _compute_ipv6addr(self):
-    # Choose the first IPv6 address that uses the same port as the ORPort
-    # Or, choose the first IPv6 address in the list
-    # _stable_sort_or_addresses() ensures we choose the same IPv6 address
-    # every time, even if onionoo changes the order of the secondaries.
-    self.ipv6addr = None
-    self.ipv6orport = None
-    # Choose the first IPv6 address that uses the same port as the ORPort
-    for i in self._data['or_addresses']:
-      (ipaddr, port) = i.rsplit(':', 1)
-      if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr):
-        self.ipv6addr = ipaddr
-        self.ipv6orport = int(port)
-        return
-    # Choose the first IPv6 address in the list
-    for i in self._data['or_addresses']:
-      (ipaddr, port) = i.rsplit(':', 1)
-      if Candidate.is_valid_ipv6_address(ipaddr):
-        self.ipv6addr = ipaddr
-        self.ipv6orport = int(port)
-        return
-
-  def _compute_version(self):
-    # parse the version out of the platform string
-    # The platform looks like: "Tor 0.2.7.6 on Linux"
-    self._data['version'] = None
-    if self._data['platform'] is None:
-      return
-    # be tolerant of weird whitespacing, use a whitespace split
-    tokens = self._data['platform'].split()
-    for token in tokens:
-      vnums = token.split('.')
-      # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc
-      if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and
-          vnums[2].isdigit()):
-        self._data['version'] = token
-        return
-
-  # From #20509
-  # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev
-  # and version 0.3.0.0-alpha-dev
-  # Exhaustive lists are hard to get wrong
-  STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev',
-                              '0.2.9.2-alpha',
-                              '0.2.9.2-alpha-dev',
-                              '0.2.9.3-alpha',
-                              '0.2.9.3-alpha-dev',
-                              '0.2.9.4-alpha',
-                              '0.2.9.4-alpha-dev',
-                              '0.3.0.0-alpha-dev'
-                              ]
-
-  def is_valid_version(self):
-    # call _compute_version before calling this
-    # is the version of the relay a version we want as a fallback?
-    # checks both recommended versions and bug #20499 / #20509
-    #
-    # if the relay doesn't have a recommended version field, exclude the relay
-    if not self._data.has_key('recommended_version'):
-      log_excluded('%s not a candidate: no recommended_version field',
-                   self._fpr)
-      return False
-    if not self._data['recommended_version']:
-      log_excluded('%s not a candidate: version not recommended', self._fpr)
-      return False
-    # if the relay doesn't have version field, exclude the relay
-    if not self._data.has_key('version'):
-      log_excluded('%s not a candidate: no version field', self._fpr)
-      return False
-    if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS:
-      logging.warning('%s not a candidate: version delivers stale consensuses',
-                      self._fpr)
-      return False
-    return True
-
-  @staticmethod
-  def _extract_generic_history(history, which='unknown'):
-    # given a tree like this:
-    #   {
-    #     "1_month": {
-    #         "count": 187,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2015-02-27 06:00:00",
-    #         "interval": 14400,
-    #         "last": "2015-03-30 06:00:00",
-    #         "values": [
-    #             999,
-    #             999
-    #         ]
-    #     },
-    #     "1_week": {
-    #         "count": 169,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2015-03-23 07:30:00",
-    #         "interval": 3600,
-    #         "last": "2015-03-30 07:30:00",
-    #         "values": [ ...]
-    #     },
-    #     "1_year": {
-    #         "count": 177,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2014-04-11 00:00:00",
-    #         "interval": 172800,
-    #         "last": "2015-03-29 00:00:00",
-    #         "values": [ ...]
-    #     },
-    #     "3_months": {
-    #         "count": 185,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2014-12-28 06:00:00",
-    #         "interval": 43200,
-    #         "last": "2015-03-30 06:00:00",
-    #         "values": [ ...]
-    #     }
-    #   },
-    # extract exactly one piece of data per time interval,
-    # using smaller intervals where available.
-    #
-    # returns list of (age, length, value) dictionaries.
-
-    generic_history = []
-
-    periods = history.keys()
-    periods.sort(key = lambda x: history[x]['interval'])
-    now = datetime.datetime.utcnow()
-    newest = now
-    for p in periods:
-      h = history[p]
-      interval = datetime.timedelta(seconds = h['interval'])
-      this_ts = parse_ts(h['last'])
-
-      if (len(h['values']) != h['count']):
-        logging.warning('Inconsistent value count in %s document for %s'
-                        %(p, which))
-      for v in reversed(h['values']):
-        if (this_ts <= newest):
-          agt1 = now - this_ts
-          agt2 = interval
-          agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600)
-                     * 10**6) / 10**6
-          agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600)
-                     * 10**6) / 10**6
-          generic_history.append(
-            { 'age': agetmp1,
-              'length': agetmp2,
-              'value': v
-            })
-          newest = this_ts
-        this_ts -= interval
-
-      if (this_ts + interval != parse_ts(h['first'])):
-        logging.warning('Inconsistent time information in %s document for %s'
-                        %(p, which))
-
-    #print json.dumps(generic_history, sort_keys=True,
-    #                  indent=4, separators=(',', ': '))
-    return generic_history
-
-  @staticmethod
-  def _avg_generic_history(generic_history):
-    a = []
-    for i in generic_history:
-      if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600):
-        continue
-      if (i['length'] is not None
-          and i['age'] is not None
-          and i['value'] is not None):
-        w = i['length'] * math.pow(AGE_ALPHA, i['age']/(3600*24))
-        a.append( (i['value'] * w, w) )
-
-    sv = math.fsum(map(lambda x: x[0], a))
-    sw = math.fsum(map(lambda x: x[1], a))
-
-    if sw == 0.0:
-      svw = 0.0
-    else:
-      svw = sv/sw
-    return svw
-
-  def _add_generic_history(self, history):
-    periods = r['read_history'].keys()
-    periods.sort(key = lambda x: r['read_history'][x]['interval'] )
-
-    print periods
-
-  def add_running_history(self, history):
-    pass
-
-  def add_uptime(self, uptime):
-    logging.debug('Adding uptime %s.'%(self._fpr,))
-
-    # flags we care about: Running, V2Dir, Guard
-    if not 'flags' in uptime:
-      logging.debug('No flags in document for %s.'%(self._fpr,))
-      return
-
-    for f in ['Running', 'Guard', 'V2Dir']:
-      if not f in uptime['flags']:
-        logging.debug('No %s in flags for %s.'%(f, self._fpr,))
-        return
-
-    running = self._extract_generic_history(uptime['flags']['Running'],
-                                            '%s-Running'%(self._fpr))
-    guard = self._extract_generic_history(uptime['flags']['Guard'],
-                                          '%s-Guard'%(self._fpr))
-    v2dir = self._extract_generic_history(uptime['flags']['V2Dir'],
-                                          '%s-V2Dir'%(self._fpr))
-    if 'BadExit' in uptime['flags']:
-      badexit = self._extract_generic_history(uptime['flags']['BadExit'],
-                                              '%s-BadExit'%(self._fpr))
-
-    self._running = self._avg_generic_history(running) / ONIONOO_SCALE_ONE
-    self._guard = self._avg_generic_history(guard) / ONIONOO_SCALE_ONE
-    self._v2dir = self._avg_generic_history(v2dir) / ONIONOO_SCALE_ONE
-    self._badexit = None
-    if 'BadExit' in uptime['flags']:
-      self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
-
-  def is_candidate(self):
-    try:
-      if (MUST_BE_RUNNING_NOW and not self.is_running()):
-        log_excluded('%s not a candidate: not running now, unable to check ' +
-                     'DirPort consensus download', self._fpr)
-        return False
-      if (self._data['last_changed_address_or_port'] >
-          self.CUTOFF_ADDRESS_AND_PORT_STABLE):
-        log_excluded('%s not a candidate: changed address/port recently (%s)',
-                     self._fpr, self._data['last_changed_address_or_port'])
-        return False
-      if self._running < CUTOFF_RUNNING:
-        log_excluded('%s not a candidate: running avg too low (%lf)',
-                     self._fpr, self._running)
-        return False
-      if self._v2dir < CUTOFF_V2DIR:
-        log_excluded('%s not a candidate: v2dir avg too low (%lf)',
-                     self._fpr, self._v2dir)
-        return False
-      if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
-        log_excluded('%s not a candidate: badexit avg too high (%lf)',
-                     self._fpr, self._badexit)
-        return False
-      # this function logs a message depending on which check fails
-      if not self.is_valid_version():
-        return False
-      if self._guard < CUTOFF_GUARD:
-        log_excluded('%s not a candidate: guard avg too low (%lf)',
-                     self._fpr, self._guard)
-        return False
-      if (not self._data.has_key('consensus_weight')
-          or self._data['consensus_weight'] < 1):
-        log_excluded('%s not a candidate: consensus weight invalid', self._fpr)
-        return False
-    except BaseException as e:
-      logging.warning("Exception %s when checking if fallback is a candidate",
-                      str(e))
-      return False
-    return True
-
-  def is_in_whitelist(self, relaylist):
-    """ A fallback matches if each key in the whitelist line matches:
-          ipv4
-          dirport
-          orport
-          id
-          ipv6 address and port (if present)
-        If the fallback has an ipv6 key, the whitelist line must also have
-        it, and vice versa, otherwise they don't match. """
-    ipv6 = None
-    if self.has_ipv6():
-      ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
-    for entry in relaylist:
-      if entry['id'] != self._fpr:
-        # can't log here unless we match an IP and port, because every relay's
-        # fingerprint is compared to every entry's fingerprint
-        if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport:
-          logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' +
-                          '%s?', entry['id'], self.dirip, self.orport,
-                          self._fpr)
-        if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6:
-          logging.warning('%s excluded: has OR %s changed fingerprint to ' +
-                          '%s?', entry['id'], ipv6, self._fpr)
-        continue
-      if entry['ipv4'] != self.dirip:
-        logging.warning('%s excluded: has it changed IPv4 from %s to %s?',
-                        self._fpr, entry['ipv4'], self.dirip)
-        continue
-      if int(entry['dirport']) != self.dirport:
-        logging.warning('%s excluded: has it changed DirPort from %s:%d to ' +
-                        '%s:%d?', self._fpr, self.dirip, int(entry['dirport']),
-                        self.dirip, self.dirport)
-        continue
-      if int(entry['orport']) != self.orport:
-        logging.warning('%s excluded: has it changed ORPort from %s:%d to ' +
-                        '%s:%d?', self._fpr, self.dirip, int(entry['orport']),
-                        self.dirip, self.orport)
-        continue
-      if entry.has_key('ipv6') and self.has_ipv6():
-        # if both entry and fallback have an ipv6 address, compare them
-        if entry['ipv6'] != ipv6:
-          logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' +
-                          'to %s?', self._fpr, entry['ipv6'], ipv6)
-          continue
-      # if the fallback has an IPv6 address but the whitelist entry
-      # doesn't, or vice versa, the whitelist entry doesn't match
-      elif entry.has_key('ipv6') and not self.has_ipv6():
-        logging.warning('%s excluded: has it lost its former IPv6 address %s?',
-                        self._fpr, entry['ipv6'])
-        continue
-      elif not entry.has_key('ipv6') and self.has_ipv6():
-        logging.warning('%s excluded: has it gained an IPv6 address %s?',
-                        self._fpr, ipv6)
-        continue
-      return True
-    return False
-
-  def cw_to_bw_factor(self):
-    # any relays with a missing or zero consensus weight are not candidates
-    # any relays with a missing advertised bandwidth have it set to zero
-    return self._data['advertised_bandwidth'] / self._data['consensus_weight']
-
-  # since advertised_bandwidth is reported by the relay, it can be gamed
-  # to avoid this, use the median consensus weight to bandwidth factor to
-  # estimate this relay's measured bandwidth, and make that the upper limit
-  def measured_bandwidth(self, median_cw_to_bw_factor):
-    cw_to_bw= median_cw_to_bw_factor
-    # Reduce exit bandwidth to make sure we're not overloading them
-    if self.is_exit():
-      cw_to_bw *= EXIT_BANDWIDTH_FRACTION
-    measured_bandwidth = self._data['consensus_weight'] * cw_to_bw
-    if self._data['advertised_bandwidth'] != 0:
-      # limit advertised bandwidth (if available) to measured bandwidth
-      return min(measured_bandwidth, self._data['advertised_bandwidth'])
-    else:
-      return measured_bandwidth
-
-  def set_measured_bandwidth(self, median_cw_to_bw_factor):
-    self._data['measured_bandwidth'] = self.measured_bandwidth(
-                                                      median_cw_to_bw_factor)
-
-  def is_exit(self):
-    return 'Exit' in self._data['flags']
-
-  def is_guard(self):
-    return 'Guard' in self._data['flags']
-
-  def is_running(self):
-    return 'Running' in self._data['flags']
-
-  # does this fallback have an IPv6 address and orport?
-  def has_ipv6(self):
-    return self.ipv6addr is not None and self.ipv6orport is not None
-
-  # strip leading and trailing brackets from an IPv6 address
-  # safe to use on non-bracketed IPv6 and on IPv4 addresses
-  # also convert to unicode, and make None appear as ''
-  @staticmethod
-  def strip_ipv6_brackets(ip):
-    if ip is None:
-      return unicode('')
-    if len(ip) < 2:
-      return unicode(ip)
-    if ip[0] == '[' and ip[-1] == ']':
-      return unicode(ip[1:-1])
-    return unicode(ip)
-
-  # are ip_a and ip_b in the same netblock?
-  # mask_bits is the size of the netblock
-  # takes both IPv4 and IPv6 addresses
-  # the versions of ip_a and ip_b must be the same
-  # the mask must be valid for the IP version
-  @staticmethod
-  def netblocks_equal(ip_a, ip_b, mask_bits):
-    if ip_a is None or ip_b is None:
-      return False
-    ip_a = Candidate.strip_ipv6_brackets(ip_a)
-    ip_b = Candidate.strip_ipv6_brackets(ip_b)
-    a = ipaddress.ip_address(ip_a)
-    b = ipaddress.ip_address(ip_b)
-    if a.version != b.version:
-      raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b))
-    if mask_bits > a.max_prefixlen:
-      logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
-      mask_bits = a.max_prefixlen
-    if mask_bits < 0:
-      logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
-      mask_bits = 0
-    a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False)
-    return b in a_net
-
-  # is this fallback's IPv4 address (dirip) in the same netblock as other's
-  # IPv4 address?
-  # mask_bits is the size of the netblock
-  def ipv4_netblocks_equal(self, other, mask_bits):
-    return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits)
-
-  # is this fallback's IPv6 address (ipv6addr) in the same netblock as
-  # other's IPv6 address?
-  # Returns False if either fallback has no IPv6 address
-  # mask_bits is the size of the netblock
-  def ipv6_netblocks_equal(self, other, mask_bits):
-    if not self.has_ipv6() or not other.has_ipv6():
-      return False
-    return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits)
-
-  # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort?
-  def dirport_equal(self, other):
-    return self.dirport == other.dirport
-
-  # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort?
-  def ipv4_orport_equal(self, other):
-    return self.orport == other.orport
-
-  # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort?
-  # Returns False if either fallback has no IPv6 address
-  def ipv6_orport_equal(self, other):
-    if not self.has_ipv6() or not other.has_ipv6():
-      return False
-    return self.ipv6orport == other.ipv6orport
-
-  # does this fallback have the same DirPort, IPv4 ORPort, or
-  # IPv6 ORPort as other?
-  # Ignores IPv6 ORPort if either fallback has no IPv6 address
-  def port_equal(self, other):
-    return (self.dirport_equal(other) or self.ipv4_orport_equal(other)
-            or self.ipv6_orport_equal(other))
-
-  # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present)
-  def port_list(self):
-    ports = [self.dirport, self.orport]
-    if self.has_ipv6() and not self.ipv6orport in ports:
-      ports.append(self.ipv6orport)
-    return ports
-
-  # does this fallback share a port with other, regardless of whether the
-  # port types match?
-  # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80,
-  # return True
-  def port_shared(self, other):
-    for p in self.port_list():
-      if p in other.port_list():
-        return True
-    return False
-
-  # log how long it takes to download a consensus from dirip:dirport
-  # returns True if the download failed, False if it succeeded within max_time
-  @staticmethod
-  def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint,
-                                        max_time):
-    download_failed = False
-    # some directory mirrors respond to requests in ways that hang python
-    # sockets, which is why we log this line here
-    logging.info('Initiating %sconsensus download from %s (%s:%d) %s.',
-                 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '',
-                 nickname, dirip, dirport, fingerprint)
-    # there appears to be about 1 second of overhead when comparing stem's
-    # internal trace time and the elapsed time calculated here
-    TIMEOUT_SLOP = 1.0
-    start = datetime.datetime.utcnow()
-    try:
-      consensus = get_consensus(
-                              endpoints = [(dirip, dirport)],
-                              timeout = (max_time + TIMEOUT_SLOP),
-                              validate = True,
-                              retries = 0,
-                              fall_back_to_authority = False,
-                              document_handler = DocumentHandler.BARE_DOCUMENT,
-                              microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS
-                                ).run()[0]
-      end = datetime.datetime.utcnow()
-      time_since_expiry = (end - consensus.valid_until).total_seconds()
-    except Exception, stem_error:
-      end = datetime.datetime.utcnow()
-      log_excluded('Unable to retrieve a consensus from %s: %s', nickname,
-                    stem_error)
-      status = 'error: "%s"' % (stem_error)
-      level = logging.WARNING
-      download_failed = True
-    elapsed = (end - start).total_seconds()
-    if download_failed:
-      # keep the error failure status, and avoid using the variables
-      pass
-    elif elapsed > max_time:
-      status = 'too slow'
-      level = logging.WARNING
-      download_failed = True
-    elif (time_since_expiry > 0):
-      status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry))
-      if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE:
-        status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE)
-        level = logging.INFO
-      else:
-        status += ', invalid'
-        level = logging.WARNING
-        download_failed = True
-    else:
-      status = 'ok'
-      level = logging.DEBUG
-    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' +
-                 'max download time %0.1fs.', elapsed, status, nickname,
-                 dirip, dirport, fingerprint, max_time)
-    return download_failed
-
-  # does this fallback download the consensus fast enough?
-  def check_fallback_download_consensus(self):
-    # include the relay if we're not doing a check, or we can't check (IPv6)
-    ipv4_failed = False
-    ipv6_failed = False
-    if PERFORM_IPV4_DIRPORT_CHECKS:
-      ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
-                                                self.dirport,
-                                                self._data['nickname'],
-                                                self._fpr,
-                                                CONSENSUS_DOWNLOAD_SPEED_MAX)
-    if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
-      # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
-      ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
-                                                self.dirport,
-                                                self._data['nickname'],
-                                                self._fpr,
-                                                CONSENSUS_DOWNLOAD_SPEED_MAX)
-    return ((not ipv4_failed) and (not ipv6_failed))
-
-  # if this fallback has not passed a download check, try it again,
-  # and record the result, available in get_fallback_download_consensus
-  def try_fallback_download_consensus(self):
-    if not self.get_fallback_download_consensus():
-      self._data['download_check'] = self.check_fallback_download_consensus()
-
-  # did this fallback pass the download check?
-  def get_fallback_download_consensus(self):
-    # if we're not performing checks, return True
-    if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS:
-      return True
-    # if we are performing checks, but haven't done one, return False
-    if not self._data.has_key('download_check'):
-      return False
-    return self._data['download_check']
-
-  # output an optional header comment and info for this fallback
-  # try_fallback_download_consensus before calling this
-  def fallbackdir_line(self, fallbacks, prefilter_fallbacks):
-    s = ''
-    if OUTPUT_COMMENTS:
-      s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks)
-    # if the download speed is ok, output a C string
-    # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string
-    if self.get_fallback_download_consensus() or OUTPUT_COMMENTS:
-      s += self.fallbackdir_info(self.get_fallback_download_consensus())
-    return s
-
-  # output a header comment for this fallback
-  def fallbackdir_comment(self, fallbacks, prefilter_fallbacks):
-    # /*
-    # nickname
-    # flags
-    # adjusted bandwidth, consensus weight
-    # [contact]
-    # [identical contact counts]
-    # */
-    # Multiline C comment
-    s = '/*'
-    s += '\n'
-    s += cleanse_c_multiline_comment(self._data['nickname'])
-    s += '\n'
-    s += 'Flags: '
-    s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
-    s += '\n'
-    # this is an adjusted bandwidth, see calculate_measured_bandwidth()
-    bandwidth = self._data['measured_bandwidth']
-    weight = self._data['consensus_weight']
-    s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%(
-        bandwidth/(1024.0*1024.0),
-        weight)
-    s += '\n'
-    if self._data['contact'] is not None:
-      s += cleanse_c_multiline_comment(self._data['contact'])
-      if CONTACT_COUNT:
-        fallback_count = len([f for f in fallbacks
-                              if f._data['contact'] == self._data['contact']])
-        if fallback_count > 1:
-          s += '\n'
-          s += '%d identical contacts listed' % (fallback_count)
-
-  # output the fallback info C string for this fallback
-  # this is the text that would go after FallbackDir in a torrc
-  # if this relay failed the download test and we OUTPUT_COMMENTS,
-  # comment-out the returned string
-  def fallbackdir_info(self, dl_speed_ok):
-    # "address:dirport orport=port id=fingerprint"
-    # (insert additional madatory fields here)
-    # "[ipv6=addr:orport]"
-    # (insert additional optional fields here)
-    # /* nickname=name */
-    # /* extrainfo={0,1} */
-    # (insert additional comment fields here)
-    # /* ===== */
-    # ,
-    #
-    # Do we want a C string, or a commented-out string?
-    c_string = dl_speed_ok
-    comment_string = not dl_speed_ok and OUTPUT_COMMENTS
-    # If we don't want either kind of string, bail
-    if not c_string and not comment_string:
-      return ''
-    s = ''
-    # Comment out the fallback directory entry if it's too slow
-    # See the debug output for which address and port is failing
-    if comment_string:
-      s += '/* Consensus download failed or was too slow:\n'
-    # Multi-Line C string with trailing comma (part of a string list)
-    # This makes it easier to diff the file, and remove IPv6 lines using grep
-    # Integers don't need escaping
-    s += '"%s orport=%d id=%s"'%(
-            cleanse_c_string(self._data['dir_address']),
-            self.orport,
-            cleanse_c_string(self._fpr))
-    s += '\n'
-    # (insert additional madatory fields here)
-    if self.has_ipv6():
-      s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport)
-      s += '\n'
-    # (insert additional optional fields here)
-    if not comment_string:
-      s += '/* '
-    s += 'nickname=%s'%(cleanse_c_string(self._data['nickname']))
-    if not comment_string:
-      s += ' */'
-    s += '\n'
-    # if we know that the fallback is an extrainfo cache, flag it
-    # and if we don't know, assume it is not
-    if not comment_string:
-      s += '/* '
-    s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0)
-    if not comment_string:
-      s += ' */'
-    s += '\n'
-    # (insert additional comment fields here)
-    # The terminator and comma must be the last line in each fallback entry
-    if not comment_string:
-      s += '/* '
-    s += SECTION_SEPARATOR_BASE
-    if not comment_string:
-      s += ' */'
-    s += '\n'
-    s += ','
-    if comment_string:
-      s += '\n'
-      s += '*/'
-    return s
-
-## Fallback Candidate List Class
-
-class CandidateList(dict):
-  def __init__(self):
-    pass
-
-  def _add_relay(self, details):
-    if not 'dir_address' in details: return
-    c = Candidate(details)
-    self[ c.get_fingerprint() ] = c
-
-  def _add_uptime(self, uptime):
-    try:
-      fpr = uptime['fingerprint']
-    except KeyError:
-      raise Exception("Document has no fingerprint field.")
-
-    try:
-      c = self[fpr]
-    except KeyError:
-      logging.debug('Got unknown relay %s in uptime document.'%(fpr,))
-      return
-
-    c.add_uptime(uptime)
-
-  def _add_details(self):
-    logging.debug('Loading details document.')
-    d = fetch('details',
-        fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
-                'consensus_weight,advertised_bandwidth,or_addresses,' +
-                'dir_address,recommended_version,flags,effective_family,' +
-                'platform'))
-    logging.debug('Loading details document done.')
-
-    if not 'relays' in d: raise Exception("No relays found in document.")
-
-    for r in d['relays']: self._add_relay(r)
-
-  def _add_uptimes(self):
-    logging.debug('Loading uptime document.')
-    d = fetch('uptime')
-    logging.debug('Loading uptime document done.')
-
-    if not 'relays' in d: raise Exception("No relays found in document.")
-    for r in d['relays']: self._add_uptime(r)
-
-  def add_relays(self):
-    self._add_details()
-    self._add_uptimes()
-
-  def count_guards(self):
-    guard_count = 0
-    for fpr in self.keys():
-      if self[fpr].is_guard():
-        guard_count += 1
-    return guard_count
-
-  # Find fallbacks that fit the uptime, stability, and flags criteria,
-  # and make an array of them in self.fallbacks
-  def compute_fallbacks(self):
-    self.fallbacks = map(lambda x: self[x],
-                         filter(lambda x: self[x].is_candidate(),
-                                self.keys()))
-
-  # sort fallbacks by their consensus weight to advertised bandwidth factor,
-  # lowest to highest
-  # used to find the median cw_to_bw_factor()
-  def sort_fallbacks_by_cw_to_bw_factor(self):
-    self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor())
-
-  # sort fallbacks by their measured bandwidth, highest to lowest
-  # calculate_measured_bandwidth before calling this
-  # this is useful for reviewing candidates in priority order
-  def sort_fallbacks_by_measured_bandwidth(self):
-    self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
-                        reverse=True)
-
-  # sort fallbacks by the data field data_field, lowest to highest
-  def sort_fallbacks_by(self, data_field):
-    self.fallbacks.sort(key=lambda f: f._data[data_field])
-
-  @staticmethod
-  def load_relaylist(file_obj):
-    """ Read each line in the file, and parse it like a FallbackDir line:
-        an IPv4 address and optional port:
-          <IPv4 address>:<port>
-        which are parsed into dictionary entries:
-          ipv4=<IPv4 address>
-          dirport=<port>
-        followed by a series of key=value entries:
-          orport=<port>
-          id=<fingerprint>
-          ipv6=<IPv6 address>:<IPv6 orport>
-        each line's key/value pairs are placed in a dictonary,
-        (of string -> string key/value pairs),
-        and these dictionaries are placed in an array.
-        comments start with # and are ignored """
-    file_data = file_obj['data']
-    file_name = file_obj['name']
-    relaylist = []
-    if file_data is None:
-      return relaylist
-    for line in file_data.split('\n'):
-      relay_entry = {}
-      # ignore comments
-      line_comment_split = line.split('#')
-      line = line_comment_split[0]
-      # cleanup whitespace
-      line = cleanse_whitespace(line)
-      line = line.strip()
-      if len(line) == 0:
-        continue
-      for item in line.split(' '):
-        item = item.strip()
-        if len(item) == 0:
-          continue
-        key_value_split = item.split('=')
-        kvl = len(key_value_split)
-        if kvl < 1 or kvl > 2:
-          print '#error Bad %s item: %s, format is key=value.'%(
-                                                 file_name, item)
-        if kvl == 1:
-          # assume that entries without a key are the ipv4 address,
-          # perhaps with a dirport
-          ipv4_maybe_dirport = key_value_split[0]
-          ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':')
-          dirl = len(ipv4_maybe_dirport_split)
-          if dirl < 1 or dirl > 2:
-            print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%(
-                                                        file_name, item)
-          if dirl >= 1:
-            relay_entry['ipv4'] = ipv4_maybe_dirport_split[0]
-          if dirl == 2:
-            relay_entry['dirport'] = ipv4_maybe_dirport_split[1]
-        elif kvl == 2:
-          relay_entry[key_value_split[0]] = key_value_split[1]
-      relaylist.append(relay_entry)
-    return relaylist
-
-  # apply the fallback whitelist
-  def apply_filter_lists(self, whitelist_obj):
-    excluded_count = 0
-    logging.debug('Applying whitelist')
-    # parse the whitelist
-    whitelist = self.load_relaylist(whitelist_obj)
-    filtered_fallbacks = []
-    for f in self.fallbacks:
-      in_whitelist = f.is_in_whitelist(whitelist)
-      if in_whitelist:
-        # include
-        filtered_fallbacks.append(f)
-      elif INCLUDE_UNLISTED_ENTRIES:
-          # include
-          filtered_fallbacks.append(f)
-      else:
-          # exclude
-          excluded_count += 1
-          log_excluded('Excluding %s: not in whitelist.',
-                       f._fpr)
-    self.fallbacks = filtered_fallbacks
-    return excluded_count
-
-  @staticmethod
-  def summarise_filters(initial_count, excluded_count):
-    return '/* Whitelist excluded %d of %d candidates. */'%(
-                                                excluded_count, initial_count)
-
-  # calculate each fallback's measured bandwidth based on the median
-  # consensus weight to advertised bandwidth ratio
-  def calculate_measured_bandwidth(self):
-    self.sort_fallbacks_by_cw_to_bw_factor()
-    median_fallback = self.fallback_median(True)
-    if median_fallback is not None:
-      median_cw_to_bw_factor = median_fallback.cw_to_bw_factor()
-    else:
-      # this will never be used, because there are no fallbacks
-      median_cw_to_bw_factor = None
-    for f in self.fallbacks:
-      f.set_measured_bandwidth(median_cw_to_bw_factor)
-
-  # remove relays with low measured bandwidth from the fallback list
-  # calculate_measured_bandwidth for each relay before calling this
-  def remove_low_bandwidth_relays(self):
-    if MIN_BANDWIDTH is None:
-      return
-    above_min_bw_fallbacks = []
-    for f in self.fallbacks:
-      if f._data['measured_bandwidth'] >= MIN_BANDWIDTH:
-        above_min_bw_fallbacks.append(f)
-      else:
-        # the bandwidth we log here is limited by the relay's consensus weight
-        # as well as its adverttised bandwidth. See set_measured_bandwidth
-        # for details
-        log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' +
-                     'must be at least %.1fMByte/s', f._fpr,
-                     f._data['measured_bandwidth']/(1024.0*1024.0),
-                     MIN_BANDWIDTH/(1024.0*1024.0))
-    self.fallbacks = above_min_bw_fallbacks
-
-  # the minimum fallback in the list
-  # call one of the sort_fallbacks_* functions before calling this
-  def fallback_min(self):
-    if len(self.fallbacks) > 0:
-      return self.fallbacks[-1]
-    else:
-      return None
-
-  # the median fallback in the list
-  # call one of the sort_fallbacks_* functions before calling this
-  def fallback_median(self, require_advertised_bandwidth):
-    # use the low-median when there are an evan number of fallbacks,
-    # for consistency with the bandwidth authorities
-    if len(self.fallbacks) > 0:
-      median_position = (len(self.fallbacks) - 1) / 2
-      if not require_advertised_bandwidth:
-        return self.fallbacks[median_position]
-      # if we need advertised_bandwidth but this relay doesn't have it,
-      # move to a fallback with greater consensus weight until we find one
-      while not self.fallbacks[median_position]._data['advertised_bandwidth']:
-        median_position += 1
-        if median_position >= len(self.fallbacks):
-          return None
-      return self.fallbacks[median_position]
-    else:
-      return None
-
-  # the maximum fallback in the list
-  # call one of the sort_fallbacks_* functions before calling this
-  def fallback_max(self):
-    if len(self.fallbacks) > 0:
-      return self.fallbacks[0]
-    else:
-      return None
-
-  # return a new bag suitable for storing attributes
-  @staticmethod
-  def attribute_new():
-    return dict()
-
-  # get the count of attribute in attribute_bag
-  # if attribute is None or the empty string, return 0
-  @staticmethod
-  def attribute_count(attribute, attribute_bag):
-    if attribute is None or attribute == '':
-      return 0
-    if attribute not in attribute_bag:
-      return 0
-    return attribute_bag[attribute]
-
-  # does attribute_bag contain more than max_count instances of attribute?
-  # if so, return False
-  # if not, return True
-  # if attribute is None or the empty string, or max_count is invalid,
-  # always return True
-  @staticmethod
-  def attribute_allow(attribute, attribute_bag, max_count=1):
-    if attribute is None or attribute == '' or max_count <= 0:
-      return True
-    elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count:
-      return False
-    else:
-      return True
-
-  # add attribute to attribute_bag, incrementing the count if it is already
-  # present
-  # if attribute is None or the empty string, or count is invalid,
-  # do nothing
-  @staticmethod
-  def attribute_add(attribute, attribute_bag, count=1):
-    if attribute is None or attribute == '' or count <= 0:
-      pass
-    attribute_bag.setdefault(attribute, 0)
-    attribute_bag[attribute] += count
-
-  # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address,
-  # and per IPv6 address
-  # there is only one IPv4 address on each fallback: the IPv4 DirPort address
-  # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
-  # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
-  # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
-  # (clients only use the IPv6 ORPort)
-  # if there is no IPv6 address, only the IPv4 address is checked
-  # return the number of candidates we excluded
-  def limit_fallbacks_same_ip(self):
-    ip_limit_fallbacks = []
-    ip_list = CandidateList.attribute_new()
-    for f in self.fallbacks:
-      if (CandidateList.attribute_allow(f.dirip, ip_list,
-                                        MAX_FALLBACKS_PER_IPV4)
-          and CandidateList.attribute_allow(f.ipv6addr, ip_list,
-                                            MAX_FALLBACKS_PER_IPV6)):
-        ip_limit_fallbacks.append(f)
-        CandidateList.attribute_add(f.dirip, ip_list)
-        if f.has_ipv6():
-          CandidateList.attribute_add(f.ipv6addr, ip_list)
-      elif not CandidateList.attribute_allow(f.dirip, ip_list,
-                                             MAX_FALLBACKS_PER_IPV4):
-        log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s'
-                     %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list),
-                       f.dirip))
-      elif (f.has_ipv6() and
-            not CandidateList.attribute_allow(f.ipv6addr, ip_list,
-                                              MAX_FALLBACKS_PER_IPV6)):
-        log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s'
-                     %(f._fpr, CandidateList.attribute_count(f.ipv6addr,
-                                                             ip_list),
-                       f.ipv6addr))
-    original_count = len(self.fallbacks)
-    self.fallbacks = ip_limit_fallbacks
-    return original_count - len(self.fallbacks)
-
-  # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each
-  # ContactInfo
-  # if there is no ContactInfo, allow the fallback
-  # this check can be gamed by providing no ContactInfo, or by setting the
-  # ContactInfo to match another fallback
-  # However, given the likelihood that relays with the same ContactInfo will
-  # go down at similar times, its usefulness outweighs the risk
-  def limit_fallbacks_same_contact(self):
-    contact_limit_fallbacks = []
-    contact_list = CandidateList.attribute_new()
-    for f in self.fallbacks:
-      if CandidateList.attribute_allow(f._data['contact'], contact_list,
-                                       MAX_FALLBACKS_PER_CONTACT):
-        contact_limit_fallbacks.append(f)
-        CandidateList.attribute_add(f._data['contact'], contact_list)
-      else:
-        log_excluded(
-          'Eliminated %s: already have %d fallback(s) on ContactInfo %s'
-          %(f._fpr, CandidateList.attribute_count(f._data['contact'],
-                                                  contact_list),
-            f._data['contact']))
-    original_count = len(self.fallbacks)
-    self.fallbacks = contact_limit_fallbacks
-    return original_count - len(self.fallbacks)
-
-  # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective
-  # family
-  # if there is no family, allow the fallback
-  # we use effective family, which ensures mutual family declarations
-  # but the check can be gamed by not declaring a family at all
-  # if any indirect families exist, the result depends on the order in which
-  # fallbacks are sorted in the list
-  def limit_fallbacks_same_family(self):
-    family_limit_fallbacks = []
-    fingerprint_list = CandidateList.attribute_new()
-    for f in self.fallbacks:
-      if CandidateList.attribute_allow(f._fpr, fingerprint_list,
-                                       MAX_FALLBACKS_PER_FAMILY):
-        family_limit_fallbacks.append(f)
-        CandidateList.attribute_add(f._fpr, fingerprint_list)
-        for family_fingerprint in f._data['effective_family']:
-          CandidateList.attribute_add(family_fingerprint, fingerprint_list)
-      else:
-        # we already have a fallback with this fallback in its effective
-        # family
-        log_excluded(
-          'Eliminated %s: already have %d fallback(s) in effective family'
-          %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list)))
-    original_count = len(self.fallbacks)
-    self.fallbacks = family_limit_fallbacks
-    return original_count - len(self.fallbacks)
-
-  # try once to get the descriptors for fingerprint_list using stem
-  # returns an empty list on exception
-  @staticmethod
-  def get_fallback_descriptors_once(fingerprint_list):
-    desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True)
-    return desc_list
-
-  # try up to max_retries times to get the descriptors for fingerprint_list
-  # using stem. Stops retrying when all descriptors have been retrieved.
-  # returns a list containing the descriptors that were retrieved
-  @staticmethod
-  def get_fallback_descriptors(fingerprint_list, max_retries=5):
-    # we can't use stem's retries=, because we want to support more than 96
-    # descriptors
-    #
-    # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list
-    max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS
-    remaining_list = fingerprint_list
-    desc_list = []
-    for _ in xrange(max_retries):
-      if len(remaining_list) == 0:
-        break
-      new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS])
-      for d in new_desc_list:
-        try:
-          remaining_list.remove(d.fingerprint)
-        except ValueError:
-          # warn and ignore if a directory mirror returned a bad descriptor
-          logging.warning("Directory mirror returned unwanted descriptor %s, ignoring",
-                          d.fingerprint)
-          continue
-        desc_list.append(d)
-    return desc_list
-
-  # find the fallbacks that cache extra-info documents
-  # Onionoo doesn't know this, so we have to use stem
-  def mark_extra_info_caches(self):
-    fingerprint_list = [ f._fpr for f in self.fallbacks ]
-    logging.info("Downloading fallback descriptors to find extra-info caches")
-    desc_list = CandidateList.get_fallback_descriptors(fingerprint_list)
-    for d in desc_list:
-      self[d.fingerprint]._extra_info_cache = d.extra_info_cache
-    missing_descriptor_list = [ f._fpr for f in self.fallbacks
-                                if f._extra_info_cache is None ]
-    for f in missing_descriptor_list:
-      logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f))
-
-  # try a download check on each fallback candidate in order
-  # stop after max_count successful downloads
-  # but don't remove any candidates from the array
-  def try_download_consensus_checks(self, max_count):
-    dl_ok_count = 0
-    for f in self.fallbacks:
-      f.try_fallback_download_consensus()
-      if f.get_fallback_download_consensus():
-        # this fallback downloaded a consensus ok
-        dl_ok_count += 1
-        if dl_ok_count >= max_count:
-          # we have enough fallbacks
-          return
-
-  # put max_count successful candidates in the fallbacks array:
-  # - perform download checks on each fallback candidate
-  # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set
-  # - eliminate failed candidates
-  # - if there are more than max_count candidates, eliminate lowest bandwidth
-  # - if there are fewer than max_count candidates, leave only successful
-  # Return the number of fallbacks that failed the consensus check
-  def perform_download_consensus_checks(self, max_count):
-    self.sort_fallbacks_by_measured_bandwidth()
-    self.try_download_consensus_checks(max_count)
-    if CONSENSUS_DOWNLOAD_RETRY:
-      # try unsuccessful candidates again
-      # we could end up with more than max_count successful candidates here
-      self.try_download_consensus_checks(max_count)
-    # now we have at least max_count successful candidates,
-    # or we've tried them all
-    original_count = len(self.fallbacks)
-    self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(),
-                            self.fallbacks)
-    # some of these failed the check, others skipped the check,
-    # if we already had enough successful downloads
-    failed_count = original_count - len(self.fallbacks)
-    self.fallbacks = self.fallbacks[:max_count]
-    return failed_count
-
-  # return a string that describes a/b as a percentage
-  @staticmethod
-  def describe_percentage(a, b):
-    if b != 0:
-      return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b)
-    else:
-      # technically, 0/0 is undefined, but 0.0% is a sensible result
-      return '%d/%d = %.0f%%'%(a, b, 0.0)
-
-  # return a dictionary of lists of fallbacks by IPv4 netblock
-  # the dictionary is keyed by the fingerprint of an arbitrary fallback
-  # in each netblock
-  # mask_bits is the size of the netblock
-  def fallbacks_by_ipv4_netblock(self, mask_bits):
-    netblocks = {}
-    for f in self.fallbacks:
-      found_netblock = False
-      for b in netblocks.keys():
-        # we found an existing netblock containing this fallback
-        if f.ipv4_netblocks_equal(self[b], mask_bits):
-          # add it to the list
-          netblocks[b].append(f)
-          found_netblock = True
-          break
-      # make a new netblock based on this fallback's fingerprint
-      if not found_netblock:
-        netblocks[f._fpr] = [f]
-    return netblocks
-
-  # return a dictionary of lists of fallbacks by IPv6 netblock
-  # where mask_bits is the size of the netblock
-  def fallbacks_by_ipv6_netblock(self, mask_bits):
-    netblocks = {}
-    for f in self.fallbacks:
-      # skip fallbacks without IPv6 addresses
-      if not f.has_ipv6():
-        continue
-      found_netblock = False
-      for b in netblocks.keys():
-        # we found an existing netblock containing this fallback
-        if f.ipv6_netblocks_equal(self[b], mask_bits):
-          # add it to the list
-          netblocks[b].append(f)
-          found_netblock = True
-          break
-      # make a new netblock based on this fallback's fingerprint
-      if not found_netblock:
-        netblocks[f._fpr] = [f]
-    return netblocks
-
-  # log a message about the proportion of fallbacks in each IPv4 netblock,
-  # where mask_bits is the size of the netblock
-  def describe_fallback_ipv4_netblock_mask(self, mask_bits):
-    fallback_count = len(self.fallbacks)
-    shared_netblock_fallback_count = 0
-    most_frequent_netblock = None
-    netblocks = self.fallbacks_by_ipv4_netblock(mask_bits)
-    for b in netblocks.keys():
-      if len(netblocks[b]) > 1:
-        # how many fallbacks are in a netblock with other fallbacks?
-        shared_netblock_fallback_count += len(netblocks[b])
-        # what's the netblock with the most fallbacks?
-        if (most_frequent_netblock is None
-            or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
-          most_frequent_netblock = b
-        logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits))
-        for f in netblocks[b]:
-          logging.debug('%s - %s', f.dirip, f._fpr)
-    if most_frequent_netblock is not None:
-      logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%(
-                                    CandidateList.describe_percentage(
-                                      len(netblocks[most_frequent_netblock]),
-                                      fallback_count),
-                                    mask_bits,
-                                    self[most_frequent_netblock].dirip))
-    if shared_netblock_fallback_count > 0:
-      logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' +
-                       'fallbacks')%(CandidateList.describe_percentage(
-                                                shared_netblock_fallback_count,
-                                                fallback_count),
-                                     mask_bits))
-
-  # log a message about the proportion of fallbacks in each IPv6 netblock,
-  # where mask_bits is the size of the netblock
-  def describe_fallback_ipv6_netblock_mask(self, mask_bits):
-    fallback_count = len(self.fallbacks_with_ipv6())
-    shared_netblock_fallback_count = 0
-    most_frequent_netblock = None
-    netblocks = self.fallbacks_by_ipv6_netblock(mask_bits)
-    for b in netblocks.keys():
-      if len(netblocks[b]) > 1:
-        # how many fallbacks are in a netblock with other fallbacks?
-        shared_netblock_fallback_count += len(netblocks[b])
-        # what's the netblock with the most fallbacks?
-        if (most_frequent_netblock is None
-            or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
-          most_frequent_netblock = b
-        logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits))
-        for f in netblocks[b]:
-          logging.debug('%s - %s', f.ipv6addr, f._fpr)
-    if most_frequent_netblock is not None:
-      logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%(
-                                    CandidateList.describe_percentage(
-                                      len(netblocks[most_frequent_netblock]),
-                                      fallback_count),
-                                    mask_bits,
-                                    self[most_frequent_netblock].ipv6addr))
-    if shared_netblock_fallback_count > 0:
-      logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' +
-                       'fallbacks')%(CandidateList.describe_percentage(
-                                                shared_netblock_fallback_count,
-                                                fallback_count),
-                                     mask_bits))
-
-  # log a message about the proportion of fallbacks in each IPv4 /8, /16,
-  # and /24
-  def describe_fallback_ipv4_netblocks(self):
-   # this doesn't actually tell us anything useful
-   #self.describe_fallback_ipv4_netblock_mask(8)
-   self.describe_fallback_ipv4_netblock_mask(16)
-   #self.describe_fallback_ipv4_netblock_mask(24)
-
-  # log a message about the proportion of fallbacks in each IPv6 /12 (RIR),
-  # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host)
-  # https://www.iana.org/assignments/ipv6-unicast-address-assignments/
-  def describe_fallback_ipv6_netblocks(self):
-    # these don't actually tell us anything useful
-    #self.describe_fallback_ipv6_netblock_mask(12)
-    #self.describe_fallback_ipv6_netblock_mask(23)
-    self.describe_fallback_ipv6_netblock_mask(32)
-    #self.describe_fallback_ipv6_netblock_mask(48)
-    self.describe_fallback_ipv6_netblock_mask(64)
-
-  # log a message about the proportion of fallbacks in each IPv4 and IPv6
-  # netblock
-  def describe_fallback_netblocks(self):
-    self.describe_fallback_ipv4_netblocks()
-    self.describe_fallback_ipv6_netblocks()
-
-  # return a list of fallbacks which are on the IPv4 ORPort port
-  def fallbacks_on_ipv4_orport(self, port):
-    return filter(lambda x: x.orport == port, self.fallbacks)
-
-  # return a list of fallbacks which are on the IPv6 ORPort port
-  def fallbacks_on_ipv6_orport(self, port):
-    return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6())
-
-  # return a list of fallbacks which are on the DirPort port
-  def fallbacks_on_dirport(self, port):
-    return filter(lambda x: x.dirport == port, self.fallbacks)
-
-  # log a message about the proportion of fallbacks on IPv4 ORPort port
-  # and return that count
-  def describe_fallback_ipv4_orport(self, port):
-    port_count = len(self.fallbacks_on_ipv4_orport(port))
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks are on IPv4 ORPort %d'%(
-                    CandidateList.describe_percentage(port_count,
-                                                      fallback_count),
-                    port))
-    return port_count
-
-  # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port
-  # and return that count
-  def describe_fallback_ipv6_orport(self, port):
-    port_count = len(self.fallbacks_on_ipv6_orport(port))
-    fallback_count = len(self.fallbacks_with_ipv6())
-    logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%(
-                    CandidateList.describe_percentage(port_count,
-                                                      fallback_count),
-                    port))
-    return port_count
-
-  # log a message about the proportion of fallbacks on DirPort port
-  # and return that count
-  def describe_fallback_dirport(self, port):
-    port_count = len(self.fallbacks_on_dirport(port))
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks are on DirPort %d'%(
-                    CandidateList.describe_percentage(port_count,
-                                                      fallback_count),
-                    port))
-    return port_count
-
-  # log a message about the proportion of fallbacks on each dirport,
-  # each IPv4 orport, and each IPv6 orport
-  def describe_fallback_ports(self):
-    fallback_count = len(self.fallbacks)
-    ipv4_or_count = fallback_count
-    ipv4_or_count -= self.describe_fallback_ipv4_orport(443)
-    ipv4_or_count -= self.describe_fallback_ipv4_orport(9001)
-    logging.warning('%s of fallbacks are on other IPv4 ORPorts'%(
-                    CandidateList.describe_percentage(ipv4_or_count,
-                                                      fallback_count)))
-    ipv6_fallback_count = len(self.fallbacks_with_ipv6())
-    ipv6_or_count = ipv6_fallback_count
-    ipv6_or_count -= self.describe_fallback_ipv6_orport(443)
-    ipv6_or_count -= self.describe_fallback_ipv6_orport(9001)
-    logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%(
-                    CandidateList.describe_percentage(ipv6_or_count,
-                                                      ipv6_fallback_count)))
-    dir_count = fallback_count
-    dir_count -= self.describe_fallback_dirport(80)
-    dir_count -= self.describe_fallback_dirport(9030)
-    logging.warning('%s of fallbacks are on other DirPorts'%(
-                    CandidateList.describe_percentage(dir_count,
-                                                      fallback_count)))
-
-  # return a list of fallbacks which cache extra-info documents
-  def fallbacks_with_extra_info_cache(self):
-    return filter(lambda x: x._extra_info_cache, self.fallbacks)
-
-  # log a message about the proportion of fallbacks that cache extra-info docs
-  def describe_fallback_extra_info_caches(self):
-    extra_info_falback_count = len(self.fallbacks_with_extra_info_cache())
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks cache extra-info documents'%(
-                    CandidateList.describe_percentage(extra_info_falback_count,
-                                                      fallback_count)))
-
-  # return a list of fallbacks which have the Exit flag
-  def fallbacks_with_exit(self):
-    return filter(lambda x: x.is_exit(), self.fallbacks)
-
-  # log a message about the proportion of fallbacks with an Exit flag
-  def describe_fallback_exit_flag(self):
-    exit_falback_count = len(self.fallbacks_with_exit())
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks have the Exit flag'%(
-                    CandidateList.describe_percentage(exit_falback_count,
-                                                      fallback_count)))
-
-  # return a list of fallbacks which have an IPv6 address
-  def fallbacks_with_ipv6(self):
-    return filter(lambda x: x.has_ipv6(), self.fallbacks)
-
-  # log a message about the proportion of fallbacks on IPv6
-  def describe_fallback_ip_family(self):
-    ipv6_falback_count = len(self.fallbacks_with_ipv6())
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks are on IPv6'%(
-                    CandidateList.describe_percentage(ipv6_falback_count,
-                                                      fallback_count)))
-
-  def summarise_fallbacks(self, eligible_count, operator_count, failed_count,
-                          guard_count, target_count):
-    s = ''
-    # Report:
-    #  whether we checked consensus download times
-    #  the number of fallback directories (and limits/exclusions, if relevant)
-    #  min & max fallback bandwidths
-    #  #error if below minimum count
-    if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
-      s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%(
-            'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '',
-            ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS
-                        and PERFORM_IPV6_DIRPORT_CHECKS) else '',
-            'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '',
-            CONSENSUS_DOWNLOAD_SPEED_MAX)
-    else:
-      s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */'
-    s += '\n'
-    # Multiline C comment with #error if things go bad
-    s += '/*'
-    s += '\n'
-    # Integers don't need escaping in C comments
-    fallback_count = len(self.fallbacks)
-    if FALLBACK_PROPORTION_OF_GUARDS is None:
-      fallback_proportion = ''
-    else:
-      fallback_proportion = ', Target %d (%d * %.2f)'%(target_count,
-                                                guard_count,
-                                                FALLBACK_PROPORTION_OF_GUARDS)
-    s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count,
-                                           fallback_proportion)
-    if MAX_FALLBACK_COUNT is not None:
-      s += ', Max %d'%(MAX_FALLBACK_COUNT)
-    s += ')\n'
-    if eligible_count != fallback_count:
-      removed_count = eligible_count - fallback_count
-      excess_to_target_or_max = (eligible_count - operator_count - failed_count
-                                 - fallback_count)
-      # some 'Failed' failed the check, others 'Skipped' the check,
-      # if we already had enough successful downloads
-      s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' +
-            'Excess %d)')%(removed_count, operator_count, failed_count,
-                           excess_to_target_or_max)
-      s += '\n'
-    min_fb = self.fallback_min()
-    min_bw = min_fb._data['measured_bandwidth']
-    max_fb = self.fallback_max()
-    max_bw = max_fb._data['measured_bandwidth']
-    s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0),
-                                                 max_bw/(1024.0*1024.0))
-    s += '\n'
-    s += '*/'
-    if fallback_count < MIN_FALLBACK_COUNT:
-      # We must have a minimum number of fallbacks so they are always
-      # reachable, and are in diverse locations
-      s += '\n'
-      s += '#error Fallback Count %d is too low. '%(fallback_count)
-      s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT)
-      s += 'Try adding entries to the whitelist, '
-      s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
-    return s
-
-def process_existing():
-  logging.basicConfig(level=logging.INFO)
-  logging.getLogger('stem').setLevel(logging.INFO)
-  whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
-               'name': FALLBACK_FILE_NAME}
-  list_fallbacks(whitelist)
-
-def process_default():
-  logging.basicConfig(level=logging.WARNING)
-  logging.getLogger('stem').setLevel(logging.WARNING)
-  whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
-               'name': WHITELIST_FILE_NAME}
-  list_fallbacks(whitelist)
-
-## Main Function
-def main():
-  if get_command() == 'check_existing':
-    process_existing()
-  else:
-    process_default()
-
-def get_command():
-  if len(sys.argv) == 2:
-    return sys.argv[1]
-  else:
-    return None
-
-def log_excluded(msg, *args):
-  if get_command() == 'check_existing':
-    logging.warning(msg, *args)
-  else:
-    logging.info(msg, *args)
-
-def list_fallbacks(whitelist):
-  """ Fetches required onionoo documents and evaluates the
-      fallback directory criteria for each of the relays """
-
-  print "/* type=fallback */"
-  print ("/* version={} */"
-         .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION)))
-  now = datetime.datetime.utcnow()
-  timestamp = now.strftime('%Y%m%d%H%M%S')
-  print ("/* timestamp={} */"
-         .format(cleanse_c_multiline_comment(timestamp)))
-  # end the header with a separator, to make it easier for parsers
-  print SECTION_SEPARATOR_COMMENT
-
-  logging.warning('Downloading and parsing Onionoo data. ' +
-                  'This may take some time.')
-  # find relays that could be fallbacks
-  candidates = CandidateList()
-  candidates.add_relays()
-
-  # work out how many fallbacks we want
-  guard_count = candidates.count_guards()
-  if FALLBACK_PROPORTION_OF_GUARDS is None:
-    target_count = guard_count
-  else:
-    target_count = int(guard_count * FALLBACK_PROPORTION_OF_GUARDS)
-  # the maximum number of fallbacks is the least of:
-  # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count)
-  # - the maximum fallback count (MAX_FALLBACK_COUNT)
-  if MAX_FALLBACK_COUNT is None:
-    max_count = target_count
-  else:
-    max_count = min(target_count, MAX_FALLBACK_COUNT)
-
-  candidates.compute_fallbacks()
-  prefilter_fallbacks = copy.copy(candidates.fallbacks)
-
-  # filter with the whitelist
-  # if a relay has changed IPv4 address or ports recently, it will be excluded
-  # as ineligible before we call apply_filter_lists, and so there will be no
-  # warning that the details have changed from those in the whitelist.
-  # instead, there will be an info-level log during the eligibility check.
-  initial_count = len(candidates.fallbacks)
-  excluded_count = candidates.apply_filter_lists(whitelist)
-  print candidates.summarise_filters(initial_count, excluded_count)
-  eligible_count = len(candidates.fallbacks)
-
-  # calculate the measured bandwidth of each relay,
-  # then remove low-bandwidth relays
-  candidates.calculate_measured_bandwidth()
-  candidates.remove_low_bandwidth_relays()
-
-  # print the raw fallback list
-  #for x in candidates.fallbacks:
-  #  print x.fallbackdir_line(True)
-  #  print json.dumps(candidates[x]._data, sort_keys=True, indent=4,
-  #                   separators=(',', ': '), default=json_util.default)
-
-  # impose mandatory conditions here, like one per contact, family, IP
-  # in measured bandwidth order
-  candidates.sort_fallbacks_by_measured_bandwidth()
-  operator_count = 0
-  # only impose these limits on the final list - operators can nominate
-  # multiple candidate fallbacks, and then we choose the best set
-  if not OUTPUT_CANDIDATES:
-    operator_count += candidates.limit_fallbacks_same_ip()
-    operator_count += candidates.limit_fallbacks_same_contact()
-    operator_count += candidates.limit_fallbacks_same_family()
-
-  # check if each candidate can serve a consensus
-  # there's a small risk we've eliminated relays from the same operator that
-  # can serve a consensus, in favour of one that can't
-  # but given it takes up to 15 seconds to check each consensus download,
-  # the risk is worth it
-  if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
-    logging.warning('Checking consensus download speeds. ' +
-                    'This may take some time.')
-  failed_count = candidates.perform_download_consensus_checks(max_count)
-
-  # work out which fallbacks cache extra-infos
-  candidates.mark_extra_info_caches()
-
-  # analyse and log interesting diversity metrics
-  # like netblock, ports, exit, IPv4-only
-  # (we can't easily analyse AS, and it's hard to accurately analyse country)
-  candidates.describe_fallback_ip_family()
-  # if we can't import the ipaddress module, we can't do netblock analysis
-  if HAVE_IPADDRESS:
-    candidates.describe_fallback_netblocks()
-  candidates.describe_fallback_ports()
-  candidates.describe_fallback_extra_info_caches()
-  candidates.describe_fallback_exit_flag()
-
-  # output C comments summarising the fallback selection process
-  if len(candidates.fallbacks) > 0:
-    print candidates.summarise_fallbacks(eligible_count, operator_count,
-                                         failed_count, guard_count,
-                                         target_count)
-  else:
-    print '/* No Fallbacks met criteria */'
-
-  # output C comments specifying the OnionOO data used to create the list
-  for s in fetch_source_list():
-    print describe_fetch_source(s)
-
-  # start the list with a separator, to make it easy for parsers
-  print SECTION_SEPARATOR_COMMENT
-
-  # sort the list differently depending on why we've created it:
-  # if we're outputting the final fallback list, sort by fingerprint
-  # this makes diffs much more stable
-  # otherwise, if we're trying to find a bandwidth cutoff, or we want to
-  # contact operators in priority order, sort by bandwidth (not yet
-  # implemented)
-  # otherwise, if we're contacting operators, sort by contact
-  candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD)
-
-  for x in candidates.fallbacks:
-    print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
-
-if __name__ == "__main__":
-  main()
diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh
index a5a92579d3..6d0587351f 100755
--- a/scripts/maint/updateRustDependencies.sh
+++ b/scripts/maint/updateRustDependencies.sh
@@ -20,26 +20,26 @@
 
 set -e
 
-HERE=`dirname $(realpath $0)`
-TOPLEVEL=`dirname $(dirname $HERE)`
+HERE=$(dirname "$(realpath "$0")")
+TOPLEVEL=$(dirname "$(dirname "$HERE")")
 TOML="$TOPLEVEL/src/rust/Cargo.toml"
 VENDORED="$TOPLEVEL/src/ext/rust/crates"
-CARGO=`which cargo`
+CARGO=$(command -v cargo)
 
 if ! test -f "$TOML"  ; then
-    printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML"
+    printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML"
 fi
 
 if ! test -d "$VENDORED" ; then
-    printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED"
+    printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED"
 fi
 
 if test -z "$CARGO" ; then
-    printf "Error: cargo must be installed and in your \$PATH\n"
+    printf "Error: cargo must be installed and in your \$PATH\\n"
 fi
 
-if test -z `cargo --list | grep vendor` ; then
-    printf "Error: cargo-vendor not installed\n"
+if test -z "$(cargo --list | grep vendor)" ; then
+    printf "Error: cargo-vendor not installed\\n"
 fi
 
-$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED
+$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED"
diff --git a/scripts/maint/updateVersions.pl.in b/scripts/maint/updateVersions.pl.in
deleted file mode 100755
index 65c51a1f2d..0000000000
--- a/scripts/maint/updateVersions.pl.in
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/perl -w
-
-$CONFIGURE_IN = '@abs_top_srcdir@/configure.ac';
-$ORCONFIG_H = '@abs_top_srcdir@/src/win32/orconfig.h';
-$TOR_NSI = '@abs_top_srcdir@/contrib/win32build/tor-mingw.nsi.in';
-
-$quiet = 1;
-
-sub demand {
-    my $fn = shift;
-    die "Missing file $fn" unless (-f $fn);
-}
-
-demand($CONFIGURE_IN);
-demand($ORCONFIG_H);
-demand($TOR_NSI);
-
-# extract version from configure.ac
-
-open(F, $CONFIGURE_IN) or die "$!";
-$version = undef;
-while (<F>) {
-    if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) {
-	$version = $1;
-	last;
-    }
-}
-die "No version found" unless $version;
-print "Tor version is $version\n" unless $quiet;
-close F;
-
-sub correctversion {
-    my ($fn, $defchar) = @_;
-    undef $/;
-    open(F, $fn) or die "$!";
-    my $s = <F>;
-    close F;
-    if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) {
-	$oldver = $1;
-	if ($oldver ne $version) {
-	    print "Version mismatch in $fn: It thinks that the version is $oldver.  I think it's $version.  Fixing.\n";
-	    $line = $defchar . "define VERSION \"$version\"";
-	    open(F, ">$fn.bak");
-	    print F $s;
-	    close F;
-	    $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m;
-	    open(F, ">$fn");
-	    print F $s;
-	    close F;
-	} else {
-	    print "$fn has the correct version. Good.\n" unless $quiet;
-	}
-    } else {
-	print "Didn't find a version line in $fn -- uh oh.\n";
-    }
-}
-
-correctversion($TOR_NSI, "!");
-correctversion($ORCONFIG_H, "#");
diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py
new file mode 100755
index 0000000000..07de1c343a
--- /dev/null
+++ b/scripts/maint/update_versions.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import io
+import os
+import re
+import sys
+import time
+
+def P(path):
+    """
+    Give 'path' as a path relative to the abs_top_srcdir environment
+    variable.
+    """
+    return os.path.join(
+        os.environ.get('abs_top_srcdir', "."),
+        path)
+
+def warn(msg):
+    """
+    Print an warning message.
+    """
+    print("WARNING: {}".format(msg), file=sys.stderr)
+
+def find_version(infile):
+    """
+    Given an open file (or some other iterator of lines) holding a
+    configure.ac file, find the current version line.
+    """
+    for line in infile:
+        m = re.search(r'AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)', line)
+        if m:
+            return m.group(1)
+
+    return None
+
+def update_version_in(infile, outfile, regex, versionline):
+    """
+    Copy every line from infile to outfile. If any line matches 'regex',
+    replace it with 'versionline'.  Return True if any line was changed;
+    false otherwise.
+
+    'versionline' is either a string -- in which case it is used literally,
+    or a function that receives the output of 'regex.match'.
+    """
+    found = False
+    have_changed = False
+    for line in infile:
+        m = regex.match(line)
+        if m:
+            found = True
+            oldline = line
+            if type(versionline) == type(u""):
+                line = versionline
+            else:
+                line = versionline(m)
+            if not line.endswith("\n"):
+                line += "\n"
+            if oldline != line:
+                have_changed = True
+        outfile.write(line)
+
+    if not found:
+        warn("didn't find any version line to replace in {}".format(infile.name))
+
+    return have_changed
+
+def replace_on_change(fname, change):
+    """
+    If "change" is true, replace fname with fname.tmp.  Otherwise,
+    delete fname.tmp.  Log what we're doing to stderr.
+    """
+    if not change:
+        print("No change in {}".format(fname))
+        os.unlink(fname+".tmp")
+    else:
+        print("Updating {}".format(fname))
+        os.rename(fname+".tmp", fname)
+
+
+def update_file(fname,
+                regex,
+                versionline,
+                encoding="utf-8"):
+    """
+    Replace any line matching 'regex' in 'fname' with 'versionline'.
+    Do not modify 'fname' if there are no changes made.  Use the
+    provided encoding to read and write.
+    """
+    with io.open(fname, "r", encoding=encoding) as f, \
+         io.open(fname+".tmp", "w", encoding=encoding) as outf:
+        have_changed = update_version_in(f, outf, regex, versionline)
+
+    replace_on_change(fname, have_changed)
+
+# Find out our version
+with open(P("configure.ac")) as f:
+    version = find_version(f)
+
+# If we have no version, we can't proceed.
+if version == None:
+    print("No version found in configure.ac", file=sys.stderr())
+    sys.exit(1)
+
+print("The version is {}".format(version))
+
+today = time.strftime("%Y-%m-%d", time.gmtime())
+
+# In configure.ac, we replace the definition of APPROX_RELEASE_DATE
+# with "{today} for {version}", but only if the version does not match
+# what is already there.
+def replace_fn(m):
+    if m.group(1) != version:
+        # The version changed -- we change the date.
+        return u'AC_DEFINE(APPROX_RELEASE_DATE, ["{}"], # for {}'.format(today, version)
+    else:
+        # No changes.
+        return m.group(0)
+update_file(P("configure.ac"),
+            re.compile(r'AC_DEFINE\(APPROX_RELEASE_DATE.* for (.*)'),
+            replace_fn)
+
+# In tor-mingw.nsi.in, we replace the definition of VERSION.
+update_file(P("contrib/win32build/tor-mingw.nsi.in"),
+            re.compile(r'!define VERSION .*'),
+            u'!define VERSION "{}"'.format(version),
+            encoding="iso-8859-1")
+
+# In src/win32/orconfig.h, we replace the definition of VERSION.
+update_file(P("src/win32/orconfig.h"),
+            re.compile(r'#define VERSION .*'),
+            u'#define VERSION "{}"'.format(version))
diff --git a/scripts/test/appveyor-irc-notify.py b/scripts/test/appveyor-irc-notify.py
index cfe0afe7ae..598a68f47d 100644
--- a/scripts/test/appveyor-irc-notify.py
+++ b/scripts/test/appveyor-irc-notify.py
@@ -75,8 +75,10 @@ in Appveyor's YAML:
       - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
 from __future__ import print_function
-from __future__ import absolute_import
+from __future__ import unicode_literals
 
 import os
 import random
diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh
index 8a3f2c70c8..dcf8ab1102 100755
--- a/scripts/test/chutney-git-bisect.sh
+++ b/scripts/test/chutney-git-bisect.sh
@@ -15,21 +15,21 @@
 # Skips the test if <skip-flavour> fails (default no skip).
 
 CHUTNEY_TRIES=3
-if [ ! -z "$1" ]; then
+if [ -n "$1" ]; then
     CHUTNEY_TRIES="$1"
 fi
 
-if [ ! -z "$2" ]; then
-    cd "$2"
+if [ -n "$2" ]; then
+    cd "$2" || exit
 fi
 
 CHUTNEY_TEST_CMD="make test-network-all"
-if [ ! -z "$3" ]; then
+if [ -n "$3" ]; then
     CHUTNEY_TEST_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $3"
 fi
 
 CHUTNEY_SKIP_ON_FAIL_CMD="true"
-if [ ! -z "$4" ]; then
+if [ -n "$4" ]; then
     CHUTNEY_SKIP_ON_FAIL_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $4"
 fi
 
@@ -54,9 +54,9 @@ while [ "$i" -le "$CHUTNEY_TRIES" ]; do
         echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good"
         exit 0
     fi
-    i=$[$i+1]
+    i=$((i+1))
 done
 
-i=$[$i-1]
+i=$((i-1))
 echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad"
 exit 1
diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff
index 6179dff63e..8751800966 100755
--- a/scripts/test/cov-diff
+++ b/scripts/test/cov-diff
@@ -7,15 +7,14 @@
 DIRA="$1"
 DIRB="$2"
 
-for B in $DIRB/*; do
-  A=$DIRA/`basename $B`
-  if [ -f $A ]; then
+for B in "$DIRB"/*; do
+  A=$DIRA/$(basename "$B")
+  if [ -f "$A" ]; then
       perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
   else
       cat /dev/null > "$A.tmp"
   fi
   perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
   diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;'
-  rm "$A.tmp" "$B.tmp"
+  rm -f "$A.tmp" "$B.tmp"
 done
-
diff --git a/scripts/test/cov-test-determinism.sh b/scripts/test/cov-test-determinism.sh
new file mode 100755
index 0000000000..3458f96968
--- /dev/null
+++ b/scripts/test/cov-test-determinism.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# To use this script, build Tor with coverage enabled, and then say:
+#  ./scripts/test/cov-test-determinism.sh run
+#
+# Let it run for a long time so it can run the tests over and over.  It
+# will put their coverage outputs in coverage-raw/coverage-*/.
+#
+# Then say:
+#  ./scripts/test/cov-test-determinism.sh check
+#
+# It will diff the other coverage outputs to the first one, and put their
+# diffs in coverage-raw/diff-coverage-*.
+
+run=0
+check=0
+
+if test "$1" = run; then
+    run=1
+elif test "$1" = check; then
+    check=1
+else
+    echo "First use 'run' with this script, then use 'check'."
+    exit 1
+fi
+
+if test "$run" = 1; then
+    # same seed as in travis.yml
+    TOR_TEST_RNG_SEED="636f766572616765"
+    export TOR_TEST_RNG_SEED
+    while true; do
+        make reset-gcov
+        CD=coverage-raw/coverage-$(date +%s)
+        make -j5 check
+        mkdir -p "$CD"
+        ./scripts/test/coverage "$CD"
+    done
+fi
+
+if test "$check" = 1; then
+    cd coverage-raw || exit 1
+
+    FIRST="$(find . -name "coverage-*" -type d | head -1)"
+    rm -f A
+    ln -sf "$FIRST" A
+    for dir in coverage-*; do
+        rm -f B
+        ln -sf "$dir" B
+        ../scripts/test/cov-diff A B > "diff-$dir"
+    done
+fi
diff --git a/scripts/test/coverage b/scripts/test/coverage
index b6e17abe25..f61c83bc72 100755
--- a/scripts/test/coverage
+++ b/scripts/test/coverage
@@ -8,9 +8,9 @@
 dst=$1
 
 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do
-    BN=`basename $fn`
-    DN=`dirname $fn`
-    F=`echo $BN | sed -e 's/\.c$//;'`
+    BN=$(basename "$fn")
+    DN=$(dirname "$fn")
+    F=$(echo "$BN" | sed -e 's/\.c$//;')
     GC="${BN}.gcov"
     # Figure out the object file names
     ONS=$(echo "${DN}"/*testing_a-"${F}".o)
@@ -20,18 +20,18 @@ for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do
     then
       for on in $ONS; do
         # We should have a gcno file
-        GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'`
-        if [ -e $GCNO ]
+        GCNO=$(echo "$on" | sed -e 's/\.o$/\.gcno/;')
+        if [ -e "$GCNO" ]
         then
           # No need to test for gcda, since gcov assumes no execution
           # if it's absent
-          rm -f $GC
-          gcov -o $on $fn
-          if [ -e $GC ]
+          rm -f "$GC"
+          gcov -o "$on" "$fn"
+          if [ -e "$GC" ]
           then
             if [ -d "$dst" ]
             then
-              mv $GC $dst/$GC
+              mv "$GC" "$dst"/"$GC"
             fi
           else
             echo "gcov -o $on $fn didn't make a .gcov file"
diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh
index 8d126cbcee..26e05ff101 100755
--- a/scripts/test/scan-build.sh
+++ b/scripts/test/scan-build.sh
@@ -33,6 +33,7 @@ CHECKERS="\
     -enable-checker security.insecureAPI.strcpy \
 "
 
+# shellcheck disable=SC2034
 # These have high false-positive rates.
 EXTRA_CHECKERS="\
     -enable-checker alpha.security.ArrayBoundV2 \
@@ -40,6 +41,7 @@ EXTRA_CHECKERS="\
     -enable-checker alpha.core.CastSize \
 "
 
+# shellcheck disable=SC2034
 # These don't seem to generate anything useful
 NOISY_CHECKERS="\
     -enable-checker alpha.clone.CloneChecker \
@@ -52,6 +54,7 @@ else
    OUTPUTARG=""
 fi
 
+# shellcheck disable=SC2086
 scan-build \
     $CHECKERS \
     ./configure
@@ -61,11 +64,13 @@ scan-build \
 
 # Make this not get scanned for dead assignments, since it has lots of
 # dead assignments we don't care about.
+# shellcheck disable=SC2086
 scan-build \
     $CHECKERS \
     -disable-checker deadcode.DeadStores \
     make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a
 
+# shellcheck disable=SC2086
 scan-build \
     $CHECKERS $OUTPUTARG \
     make -j5 -k
diff --git a/src/app/app.md b/src/app/app.md
new file mode 100644
index 0000000000..138e75b127
--- /dev/null
+++ b/src/app/app.md
@@ -0,0 +1,6 @@
+@dir /app
+@brief app: top-level entry point for Tor
+
+The "app" directory has Tor's main entry point and configuration logic,
+and is responsible for initializing and managing the other modules in
+Tor.
diff --git a/src/app/config/.may_include b/src/app/config/.may_include
new file mode 100644
index 0000000000..11c5ffbb14
--- /dev/null
+++ b/src/app/config/.may_include
@@ -0,0 +1,2 @@
+*.h
+*.inc
diff --git a/src/app/config/app_config.md b/src/app/config/app_config.md
new file mode 100644
index 0000000000..b359ce77f6
--- /dev/null
+++ b/src/app/config/app_config.md
@@ -0,0 +1,6 @@
+@dir /app/config
+@brief app/config: Top-level configuration code
+
+Refactoring this module is a work in progress, see
+[ticket 29211](https://trac.torproject.org/projects/tor/ticket/29211).
+
diff --git a/src/app/config/config.c b/src/app/config/config.c
index c1f1f153e7..a0c188adc4 100644
--- a/src/app/config/config.c
+++ b/src/app/config/config.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,8 @@
  *
  * To add new items to the torrc, there are a minimum of three places to edit:
  * <ul>
- *   <li>The or_options_t structure in or.h, where the options are stored.
+ *   <li>The or_options_t structure in or_options_st.h, where the options are
+ *       stored.
  *   <li>The option_vars_ array below in this module, which configures
  *       the names of the torrc options, their types, their multiplicities,
  *       and their mappings to fields in or_options_t.
@@ -32,16 +33,18 @@
  *
  * Additionally, you might need to edit these places too:
  * <ul>
- *   <li>options_validate() below, in case you want to reject some possible
+ *   <li>options_validate_cb() below, in case you want to reject some possible
  *       values of the new configuration option.
  *   <li>options_transition_allowed() below, in case you need to
  *       forbid some or all changes in the option while Tor is
  *       running.
  *   <li>options_transition_affects_workers(), in case changes in the option
  *       might require Tor to relaunch or reconfigure its worker threads.
+ *       (This function is now in the relay module.)
  *   <li>options_transition_affects_descriptor(), in case changes in the
  *       option might require a Tor relay to build and publish a new server
  *       descriptor.
+ *       (This function is now in the relay module.)
  *   <li>options_act() and/or options_act_reversible(), in case there's some
  *       action that needs to be taken immediately based on the option's
  *       value.
@@ -61,21 +64,19 @@
 #define CONFIG_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "app/config/statefile.h"
 #include "app/main/main.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
-#include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
-#include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitstats.h"
 #include "core/or/connection_edge.h"
-#include "core/or/connection_or.h"
 #include "core/or/dos.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
@@ -85,11 +86,9 @@
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
-#include "feature/dirauth/bwauth.h"
-#include "feature/dirauth/guardfraction.h"
-#include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_config.h"
 #include "feature/nodelist/dirlist.h"
@@ -101,20 +100,21 @@
 #include "feature/relay/dns.h"
 #include "feature/relay/ext_orport.h"
 #include "feature/relay/routermode.h"
+#include "feature/relay/relay_config.h"
+#include "feature/relay/transport_config.h"
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
 #include "lib/geoip/geoip.h"
 #include "feature/stats/geoip_stats.h"
-#include "feature/stats/predict_ports.h"
-#include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
+#include "lib/confmgt/structvar.h"
 #include "lib/crypt_ops/crypto_init.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/encoding/confline.h"
-#include "lib/log/git_revision.h"
 #include "lib/net/resolve.h"
 #include "lib/sandbox/sandbox.h"
+#include "lib/version/torversion.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
@@ -144,7 +144,7 @@
 #include "lib/process/pidfile.h"
 #include "lib/process/restrict.h"
 #include "lib/process/setuid.h"
-#include "lib/process/subprocess.h"
+#include "lib/process/process.h"
 #include "lib/net/gethostname.h"
 #include "lib/thread/numcpus.h"
 
@@ -152,9 +152,8 @@
 #include "lib/fs/conffile.h"
 #include "lib/evloop/procmon.h"
 
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/recommend_pkg.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_config.h"
 
 #include "core/or/connection_st.h"
 #include "core/or/port_cfg_st.h"
@@ -181,12 +180,16 @@ static const char unix_q_socket_prefix[] = "unix:\"";
 
 /** macro to help with the bulk rename of *DownloadSchedule to
  * *DowloadInitialDelay . */
+#ifndef COCCI
 #define DOWNLOAD_SCHEDULE(name) \
-  { #name "DownloadSchedule", #name "DownloadInitialDelay", 0, 1 }
+  { (#name "DownloadSchedule"), (#name "DownloadInitialDelay"), 0, 1 }
+#else
+#define DOWNLOAD_SCHEDULE(name) { NULL, NULL, 0, 1 }
+#endif /* !defined(COCCI) */
 
 /** A list of abbreviations and aliases to map command-line options, obsolete
  * option names, or alternative option names, to their current values. */
-static config_abbrev_t option_abbrevs_[] = {
+static const config_abbrev_t option_abbrevs_[] = {
   PLURAL(AuthDirBadDirCC),
   PLURAL(AuthDirBadExitCC),
   PLURAL(AuthDirInvalidCC),
@@ -249,22 +252,43 @@ static config_abbrev_t option_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_options_t);
 
-/** An entry for config_vars: "The option <b>name</b> has type
+/** An entry for config_vars: "The option <b>varname</b> has type
  * CONFIG_TYPE_<b>conftype</b>, and corresponds to
  * or_options_t.<b>member</b>"
  */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member),     \
-      initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, 0, initvalue)
+
+/* As VAR, but uses a type definition in addition to a type enum. */
+#define VAR_D(varname,conftype,member,initvalue)                        \
+  CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, 0, initvalue)
+
+#define VAR_NODUMP(varname,conftype,member,initvalue)             \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP, initvalue)
+#define VAR_NODUMP_IMMUTABLE(varname,conftype,member,initvalue)   \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP | CFLG_IMMUTABLE, initvalue)
+#define VAR_INVIS(varname,conftype,member,initvalue)              \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP | CFLG_NOSET | CFLG_NOLIST, initvalue)
+
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
-/** An entry for config_vars: "The option <b>name</b> is obsolete." */
-#ifdef TOR_UNIT_TESTS
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} }
-#else
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL }
-#endif
+
+#define VAR_IMMUTABLE(varname, conftype, member, initvalue)             \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,             \
+                   CFLG_IMMUTABLE, initvalue)
+
+#define V_IMMUTABLE(member,conftype,initvalue)                 \
+  VAR_IMMUTABLE(#member, conftype, member, initvalue)
+
+/** As V, but uses a type definition instead of a type enum */
+#define V_D(member,type,initvalue)              \
+  VAR_D(#member, type, member, initvalue)
+
+/** An entry for config_vars: "The option <b>varname</b> is obsolete." */
+#define OBSOLETE(varname) CONFIG_VAR_OBSOLETE(varname)
 
 /**
  * Macro to declare *Port options.  Each one comes in three entries.
@@ -276,7 +300,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
 #define VPORT(member)                                           \
   VAR(#member "Lines", LINELIST_V, member ## _lines, NULL),     \
   VAR(#member, LINELIST_S, member ## _lines, NULL),             \
-  VAR("__" #member, LINELIST_S, member ## _lines, NULL)
+  VAR_NODUMP("__" #member, LINELIST_S, member ## _lines, NULL)
 
 /** UINT64_MAX as a decimal string */
 #define UINT64_MAX_STRING "18446744073709551615"
@@ -285,7 +309,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
  * abbreviations, order is significant, since the first matching option will
  * be chosen first.
  */
-static config_var_t option_vars_[] = {
+static const config_var_t option_vars_[] = {
   V(AccountingMax,               MEMUNIT,  "0 bytes"),
   VAR("AccountingRule",          STRING,   AccountingRule_option,  "max"),
   V(AccountingStart,             STRING,   NULL),
@@ -305,17 +329,11 @@ static config_var_t option_vars_[] = {
   V(AuthDirBadExitCCs,           CSV,      ""),
   V(AuthDirInvalid,              LINELIST, NULL),
   V(AuthDirInvalidCCs,           CSV,      ""),
-  V(AuthDirFastGuarantee,        MEMUNIT,  "100 KB"),
-  V(AuthDirGuardBWGuarantee,     MEMUNIT,  "2 MB"),
-  V(AuthDirPinKeys,              BOOL,     "1"),
   V(AuthDirReject,               LINELIST, NULL),
   V(AuthDirRejectCCs,            CSV,      ""),
   OBSOLETE("AuthDirRejectUnlisted"),
   OBSOLETE("AuthDirListBadDirs"),
-  V(AuthDirListBadExits,         BOOL,     "0"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "2"),
   OBSOLETE("AuthDirMaxServersPerAuthAddr"),
-  V(AuthDirHasIPv6Connectivity,  BOOL,     "0"),
   VAR("AuthoritativeDirectory",  BOOL, AuthoritativeDir,    "0"),
   V(AutomapHostsOnResolve,       BOOL,     "0"),
   V(AutomapHostsSuffixes,        CSV,      ".onion,.exit"),
@@ -328,7 +346,7 @@ static config_var_t option_vars_[] = {
   V(BridgeRecordUsageByCountry,  BOOL,     "1"),
   V(BridgeRelay,                 BOOL,     "0"),
   V(BridgeDistribution,          STRING,   NULL),
-  VAR("CacheDirectory",          FILENAME, CacheDirectory_option, NULL),
+  VAR_IMMUTABLE("CacheDirectory",FILENAME, CacheDirectory_option, NULL),
   V(CacheDirectoryGroupReadable, AUTOBOOL,     "auto"),
   V(CellStatistics,              BOOL,     "0"),
   V(PaddingStatistics,           BOOL,     "1"),
@@ -339,15 +357,21 @@ static config_var_t option_vars_[] = {
   V(CircuitStreamTimeout,        INTERVAL, "0"),
   V(CircuitPriorityHalflife,     DOUBLE,  "-1.0"), /*negative:'Use default'*/
   V(ClientDNSRejectInternalAddresses, BOOL,"1"),
+#if defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS)
+  /* The unit tests expect the ClientOnly default to be 0. */
   V(ClientOnly,                  BOOL,     "0"),
+#else
+  /* We must be a Client if the relay module is disabled. */
+  V(ClientOnly,                  BOOL,     "1"),
+#endif /* defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS) */
   V(ClientPreferIPv6ORPort,      AUTOBOOL, "auto"),
   V(ClientPreferIPv6DirPort,     AUTOBOOL, "auto"),
+  OBSOLETE("ClientAutoIPv6ORPort"),
   V(ClientRejectInternalAddresses, BOOL,   "1"),
   V(ClientTransportPlugin,       LINELIST, NULL),
   V(ClientUseIPv6,               BOOL,     "0"),
   V(ClientUseIPv4,               BOOL,     "1"),
-  V(ConsensusParams,             STRING,   NULL),
-  V(ConnLimit,                   UINT,     "1000"),
+  V(ConnLimit,                   POSINT,     "1000"),
   V(ConnDirectionStatistics,     BOOL,     "0"),
   V(ConstrainedSockets,          BOOL,     "0"),
   V(ConstrainedSockSize,         MEMUNIT,  "8192"),
@@ -361,21 +385,26 @@ static config_var_t option_vars_[] = {
   V(UnixSocksGroupWritable,    BOOL,     "0"),
   V(CookieAuthentication,        BOOL,     "0"),
   V(CookieAuthFileGroupReadable, BOOL,     "0"),
-  V(CookieAuthFile,              STRING,   NULL),
+  V(CookieAuthFile,              FILENAME, NULL),
   V(CountPrivateBandwidth,       BOOL,     "0"),
-  VAR("DataDirectory",           FILENAME, DataDirectory_option, NULL),
+  VAR_IMMUTABLE("DataDirectory", FILENAME, DataDirectory_option, NULL),
   V(DataDirectoryGroupReadable,  BOOL,     "0"),
   V(DisableOOSCheck,             BOOL,     "1"),
   V(DisableNetwork,              BOOL,     "0"),
   V(DirAllowPrivateAddresses,    BOOL,     "0"),
-  V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"),
   OBSOLETE("DirListenAddress"),
   V(DirPolicy,                   LINELIST, NULL),
   VPORT(DirPort),
   V(DirPortFrontPage,            FILENAME, NULL),
   VAR("DirReqStatistics",        BOOL,     DirReqStatistics_option, "1"),
   VAR("DirAuthority",            LINELIST, DirAuthorities, NULL),
+#if defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS)
+  /* The unit tests expect the DirCache default to be 1. */
   V(DirCache,                    BOOL,     "1"),
+#else
+  /* We can't be a DirCache if the relay module is disabled. */
+  V(DirCache,                    BOOL,     "0"),
+#endif /* defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS) */
   /* A DirAuthorityFallbackRate of 0.1 means that 0.5% of clients try an
    * authority when all fallbacks are up, and 2% try an authority when 25% of
    * fallbacks are down. (We rebuild the list when 25% of fallbacks are down).
@@ -384,23 +413,27 @@ static config_var_t option_vars_[] = {
    * an order of magnitude, so there isn't too much load shifting to
    * authorities when fallbacks go down. */
   V(DirAuthorityFallbackRate,    DOUBLE,   "0.1"),
-  V(DisableAllSwap,              BOOL,     "0"),
-  V(DisableDebuggerAttachment,   BOOL,     "1"),
+  V_IMMUTABLE(DisableAllSwap,    BOOL,     "0"),
+  V_IMMUTABLE(DisableDebuggerAttachment,   BOOL,     "1"),
   OBSOLETE("DisableIOCP"),
   OBSOLETE("DisableV2DirectoryInfo_"),
   OBSOLETE("DynamicDHGroups"),
   VPORT(DNSPort),
   OBSOLETE("DNSListenAddress"),
+  V(DormantClientTimeout,         INTERVAL, "24 hours"),
+  V(DormantTimeoutDisabledByIdleStreams, BOOL,     "1"),
+  V(DormantOnFirstStartup,       BOOL,      "0"),
+  V(DormantCanceledByStartup,    BOOL,      "0"),
   /* DoS circuit creation options. */
   V(DoSCircuitCreationEnabled,   AUTOBOOL, "auto"),
-  V(DoSCircuitCreationMinConnections,      UINT, "0"),
-  V(DoSCircuitCreationRate,      UINT,     "0"),
-  V(DoSCircuitCreationBurst,     UINT,     "0"),
+  V(DoSCircuitCreationMinConnections,      POSINT, "0"),
+  V(DoSCircuitCreationRate,      POSINT,     "0"),
+  V(DoSCircuitCreationBurst,     POSINT,     "0"),
   V(DoSCircuitCreationDefenseType,         INT,  "0"),
   V(DoSCircuitCreationDefenseTimePeriod,   INTERVAL, "0"),
   /* DoS connection options. */
   V(DoSConnectionEnabled,        AUTOBOOL, "auto"),
-  V(DoSConnectionMaxConcurrentCount,       UINT, "0"),
+  V(DoSConnectionMaxConcurrentCount,       POSINT, "0"),
   V(DoSConnectionDefenseType,    INT,      "0"),
   /* DoS single hop client options. */
   V(DoSRefuseSingleHopClientRendezvous,    AUTOBOOL, "auto"),
@@ -409,13 +442,17 @@ static config_var_t option_vars_[] = {
   V(TestingEnableCellStatsEvent, BOOL,     "0"),
   OBSOLETE("TestingEnableTbEmptyEvent"),
   V(EnforceDistinctSubnets,      BOOL,     "1"),
-  V(EntryNodes,                  ROUTERSET,   NULL),
+  V_D(EntryNodes,                ROUTERSET,   NULL),
   V(EntryStatistics,             BOOL,     "0"),
-  V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"),
-  V(ExcludeNodes,                ROUTERSET, NULL),
-  V(ExcludeExitNodes,            ROUTERSET, NULL),
+  OBSOLETE("TestingEstimatedDescriptorPropagationTime"),
+  V_D(ExcludeNodes,              ROUTERSET, NULL),
+  V_D(ExcludeExitNodes,          ROUTERSET, NULL),
   OBSOLETE("ExcludeSingleHopRelays"),
-  V(ExitNodes,                   ROUTERSET, NULL),
+  V_D(ExitNodes,                 ROUTERSET, NULL),
+  /* Researchers need a way to tell their clients to use specific
+   * middles that they also control, to allow safe live-network
+   * experimentation with new padding machines. */
+  V_D(MiddleNodes,               ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
   V(ExitPolicyRejectLocalInterfaces, BOOL, "0"),
@@ -423,7 +460,7 @@ static config_var_t option_vars_[] = {
   V(ExtendAllowPrivateAddresses, BOOL,     "0"),
   V(ExitRelay,                   AUTOBOOL, "auto"),
   VPORT(ExtORPort),
-  V(ExtORPortCookieAuthFile,     STRING,   NULL),
+  V(ExtORPortCookieAuthFile,     FILENAME,   NULL),
   V(ExtORPortCookieAuthFileGroupReadable, BOOL, "0"),
   V(ExtraInfoStatistics,         BOOL,     "1"),
   V(ExtendByEd25519ID,           AUTOBOOL, "auto"),
@@ -453,11 +490,8 @@ static config_var_t option_vars_[] = {
 #endif /* defined(_WIN32) */
   OBSOLETE("Group"),
   V(GuardLifetime,               INTERVAL, "0 minutes"),
-  V(HardwareAccel,               BOOL,     "0"),
   V(HeartbeatPeriod,             INTERVAL, "6 hours"),
   V(MainloopStats,               BOOL,     "0"),
-  V(AccelName,                   STRING,   NULL),
-  V(AccelDir,                    FILENAME, NULL),
   V(HashedControlPassword,       LINELIST, NULL),
   OBSOLETE("HidServDirectoryV2"),
   VAR("HiddenServiceDir",    LINELIST_S, RendConfigLines,    NULL),
@@ -471,13 +505,20 @@ static config_var_t option_vars_[] = {
   VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceExportCircuitID", LINELIST_S,  RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSDefense", LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSRatePerSec",
+      LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSBurstPerSec",
+      LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceOnionBalanceInstance",
+      LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"),
   V(HidServAuth,                 LINELIST, NULL),
   V(ClientOnionAuthDir,          FILENAME, NULL),
   OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"),
   OBSOLETE("CloseHSServiceRendCircuitsImmediatelyOnTimeout"),
-  V(HiddenServiceSingleHopMode,  BOOL,     "0"),
-  V(HiddenServiceNonAnonymousMode,BOOL,    "0"),
+  V_IMMUTABLE(HiddenServiceSingleHopMode,  BOOL,     "0"),
+  V_IMMUTABLE(HiddenServiceNonAnonymousMode,BOOL,    "0"),
   V(HTTPProxy,                   STRING,   NULL),
   V(HTTPProxyAuthenticator,      STRING,   NULL),
   V(HTTPSProxy,                  STRING,   NULL),
@@ -492,30 +533,30 @@ static config_var_t option_vars_[] = {
   V(Socks5Proxy,                 STRING,   NULL),
   V(Socks5ProxyUsername,         STRING,   NULL),
   V(Socks5ProxyPassword,         STRING,   NULL),
-  VAR("KeyDirectory",            FILENAME, KeyDirectory_option, NULL),
-  V(KeyDirectoryGroupReadable,   BOOL,     "0"),
-  VAR("HSLayer2Nodes",           ROUTERSET,  HSLayer2Nodes,  NULL),
-  VAR("HSLayer3Nodes",           ROUTERSET,  HSLayer3Nodes,  NULL),
+  V(TCPProxy,                    STRING,   NULL),
+  VAR_IMMUTABLE("KeyDirectory",  FILENAME, KeyDirectory_option, NULL),
+  V(KeyDirectoryGroupReadable,   AUTOBOOL, "auto"),
+  VAR_D("HSLayer2Nodes",         ROUTERSET,  HSLayer2Nodes,  NULL),
+  VAR_D("HSLayer3Nodes",         ROUTERSET,  HSLayer3Nodes,  NULL),
   V(KeepalivePeriod,             INTERVAL, "5 minutes"),
-  V(KeepBindCapabilities,            AUTOBOOL, "auto"),
+  V_IMMUTABLE(KeepBindCapabilities,        AUTOBOOL, "auto"),
   VAR("Log",                     LINELIST, Logs,             NULL),
   V(LogMessageDomains,           BOOL,     "0"),
   V(LogTimeGranularity,          MSEC_INTERVAL, "1 second"),
   V(TruncateLogFile,             BOOL,     "0"),
-  V(SyslogIdentityTag,           STRING,   NULL),
-  V(AndroidIdentityTag,          STRING,   NULL),
+  V_IMMUTABLE(SyslogIdentityTag, STRING,   NULL),
+  V_IMMUTABLE(AndroidIdentityTag,STRING,   NULL),
   V(LongLivedPorts,              CSV,
         "21,22,706,1863,5050,5190,5222,5223,6523,6667,6697,8300"),
   VAR("MapAddress",              LINELIST, AddressMap,           NULL),
   V(MaxAdvertisedBandwidth,      MEMUNIT,  "1 GB"),
   V(MaxCircuitDirtiness,         INTERVAL, "10 minutes"),
-  V(MaxClientCircuitsPending,    UINT,     "32"),
+  V(MaxClientCircuitsPending,    POSINT,     "32"),
   V(MaxConsensusAgeForDiffs,     INTERVAL, "0 seconds"),
   VAR("MaxMemInQueues",          MEMUNIT,   MaxMemInQueues_raw, "0"),
   OBSOLETE("MaxOnionsPending"),
   V(MaxOnionQueueDelay,          MSEC_INTERVAL, "1750 msec"),
   V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"),
-  V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"),
   VAR("MyFamily",                LINELIST, MyFamily_lines,       NULL),
   V(NewCircuitPeriod,            INTERVAL, "30 seconds"),
   OBSOLETE("NamingAuthoritativeDirectory"),
@@ -525,11 +566,11 @@ static config_var_t option_vars_[] = {
   OBSOLETE("PredictedPortsRelevanceTime"),
   OBSOLETE("WarnUnsafeSocks"),
   VAR("NodeFamily",              LINELIST, NodeFamilies,         NULL),
-  V(NoExec,                      BOOL,     "0"),
-  V(NumCPUs,                     UINT,     "0"),
-  V(NumDirectoryGuards,          UINT,     "0"),
-  V(NumEntryGuards,              UINT,     "0"),
-  V(NumPrimaryGuards,            UINT,     "0"),
+  V_IMMUTABLE(NoExec,            BOOL,     "0"),
+  V(NumCPUs,                     POSINT,     "0"),
+  V(NumDirectoryGuards,          POSINT,     "0"),
+  V(NumEntryGuards,              POSINT,     "0"),
+  V(NumPrimaryGuards,            POSINT,     "0"),
   V(OfflineMasterKey,            BOOL,     "0"),
   OBSOLETE("ORListenAddress"),
   VPORT(ORPort),
@@ -556,10 +597,8 @@ static config_var_t option_vars_[] = {
   V(PathsNeededToBuildCircuits,  DOUBLE,   "-1"),
   V(PerConnBWBurst,              MEMUNIT,  "0"),
   V(PerConnBWRate,               MEMUNIT,  "0"),
-  V(PidFile,                     STRING,   NULL),
-  V(TestingTorNetwork,           BOOL,     "0"),
-  V(TestingMinExitFlagThreshold, MEMUNIT,  "0"),
-  V(TestingMinFastFlagThreshold, MEMUNIT,  "0"),
+  V_IMMUTABLE(PidFile,           FILENAME,   NULL),
+  V_IMMUTABLE(TestingTorNetwork, BOOL,     "0"),
 
   V(TestingLinkCertLifetime,          INTERVAL, "2 days"),
   V(TestingAuthKeyLifetime,          INTERVAL, "2 days"),
@@ -577,29 +616,28 @@ static config_var_t option_vars_[] = {
   V(ReachableAddresses,          LINELIST, NULL),
   V(ReachableDirAddresses,       LINELIST, NULL),
   V(ReachableORAddresses,        LINELIST, NULL),
-  V(RecommendedVersions,         LINELIST, NULL),
-  V(RecommendedClientVersions,   LINELIST, NULL),
-  V(RecommendedServerVersions,   LINELIST, NULL),
-  V(RecommendedPackages,         LINELIST, NULL),
+  OBSOLETE("RecommendedPackages"),
   V(ReducedConnectionPadding,    BOOL,     "0"),
   V(ConnectionPadding,           AUTOBOOL, "auto"),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
+  V(CircuitPadding,              BOOL,     "1"),
+  V(ReducedCircuitPadding,       BOOL,     "0"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
   V(RelayBandwidthRate,          MEMUNIT,  "0"),
   V(RendPostPeriod,              INTERVAL, "1 hour"),
   V(RephistTrackTime,            INTERVAL, "24 hours"),
-  V(RunAsDaemon,                 BOOL,     "0"),
+  V_IMMUTABLE(RunAsDaemon,       BOOL,     "0"),
   V(ReducedExitPolicy,           BOOL,     "0"),
   OBSOLETE("RunTesting"), // currently unused
-  V(Sandbox,                     BOOL,     "0"),
+  V_IMMUTABLE(Sandbox,           BOOL,     "0"),
   V(SafeLogging,                 STRING,   "1"),
   V(SafeSocks,                   BOOL,     "0"),
   V(ServerDNSAllowBrokenConfig,  BOOL,     "1"),
   V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"),
   V(ServerDNSDetectHijacking,    BOOL,     "1"),
   V(ServerDNSRandomizeCase,      BOOL,     "1"),
-  V(ServerDNSResolvConfFile,     STRING,   NULL),
+  V(ServerDNSResolvConfFile,     FILENAME,   NULL),
   V(ServerDNSSearchDomains,      BOOL,     "0"),
   V(ServerDNSTestAddresses,      CSV,
       "www.google.com,www.mit.edu,www.yahoo.com,www.slashdot.org"),
@@ -620,7 +658,7 @@ static config_var_t option_vars_[] = {
   V(StrictNodes,                 BOOL,     "0"),
   OBSOLETE("Support022HiddenServices"),
   V(TestSocks,                   BOOL,     "0"),
-  V(TokenBucketRefillInterval,   MSEC_INTERVAL, "100 msec"),
+  V_IMMUTABLE(TokenBucketRefillInterval,   MSEC_INTERVAL, "100 msec"),
   OBSOLETE("Tor2webMode"),
   OBSOLETE("Tor2webRendezvousPoints"),
   OBSOLETE("TLSECGroup"),
@@ -637,10 +675,8 @@ static config_var_t option_vars_[] = {
   V(UseGuardFraction,            AUTOBOOL, "auto"),
   V(UseMicrodescriptors,         AUTOBOOL, "auto"),
   OBSOLETE("UseNTorHandshake"),
-  V(User,                        STRING,   NULL),
+  V_IMMUTABLE(User,              STRING,   NULL),
   OBSOLETE("UserspaceIOCPBuffers"),
-  V(AuthDirSharedRandomness,     BOOL,     "1"),
-  V(AuthDirTestEd25519LinkKeys,  BOOL,     "1"),
   OBSOLETE("V1AuthoritativeDirectory"),
   OBSOLETE("V2AuthoritativeDirectory"),
   VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir,   "0"),
@@ -651,27 +687,29 @@ static config_var_t option_vars_[] = {
   V(V3AuthVotingInterval,        INTERVAL, "1 hour"),
   V(V3AuthVoteDelay,             INTERVAL, "5 minutes"),
   V(V3AuthDistDelay,             INTERVAL, "5 minutes"),
-  V(V3AuthNIntervalsValid,       UINT,     "3"),
+  V(V3AuthNIntervalsValid,       POSINT,     "3"),
   V(V3AuthUseLegacyKey,          BOOL,     "0"),
   V(V3BandwidthsFile,            FILENAME, NULL),
   V(GuardfractionFile,           FILENAME, NULL),
-  VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"),
   OBSOLETE("VoteOnHidServDirectoriesV2"),
   V(VirtualAddrNetworkIPv4,      STRING,   "127.192.0.0/10"),
   V(VirtualAddrNetworkIPv6,      STRING,   "[FE80::]/10"),
   V(WarnPlaintextPorts,          CSV,      "23,109,110,143"),
   OBSOLETE("UseFilteringSSLBufferevents"),
   OBSOLETE("__UseFilteringSSLBufferevents"),
-  VAR("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
-  VAR("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
-  VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
-  VAR("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
-  VAR("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
-  VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword,
+  VAR_NODUMP("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
+  VAR_NODUMP("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
+  VAR_NODUMP("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
+  VAR_NODUMP_IMMUTABLE("__DisableSignalHandlers", BOOL,
+                       DisableSignalHandlers,    "0"),
+  VAR_NODUMP("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
+  VAR_NODUMP("__HashedControlSessionPassword", LINELIST,
+             HashedControlSessionPassword,
       NULL),
-  VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
-  VAR("__OwningControllerFD", UINT64, OwningControllerFD, UINT64_MAX_STRING),
-  V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"),
+  VAR_NODUMP("__OwningControllerProcess",STRING,
+                       OwningControllerProcess, NULL),
+  VAR_NODUMP_IMMUTABLE("__OwningControllerFD", UINT64, OwningControllerFD,
+             UINT64_MAX_STRING),
   V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
@@ -700,7 +738,7 @@ static config_var_t option_vars_[] = {
    * blocked), but we also don't want to fail if only some mirrors are
    * blackholed. Clients will try 3 directories simultaneously.
    * (Relays never use simultaneous connections.) */
-  V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"),
+  V(ClientBootstrapConsensusMaxInProgressTries, POSINT, "3"),
   /* When a client has any running bridges, check each bridge occasionally,
     * whether or not that bridge is actually up. */
   V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"),
@@ -717,56 +755,40 @@ static config_var_t option_vars_[] = {
   OBSOLETE("TestingDescriptorMaxDownloadTries"),
   OBSOLETE("TestingMicrodescMaxDownloadTries"),
   OBSOLETE("TestingCertMaxDownloadTries"),
-  V(TestingDirAuthVoteExit, ROUTERSET, NULL),
-  V(TestingDirAuthVoteExitIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteGuard, ROUTERSET, NULL),
-  V(TestingDirAuthVoteGuardIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteHSDir, ROUTERSET, NULL),
-  V(TestingDirAuthVoteHSDirIsStrict,  BOOL,     "0"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"),
+  VAR_INVIS("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_,
+            "0"),
 
   END_OF_CONFIG_VARS
 };
 
+/** List of default directory authorities */
+static const char *default_authorities[] = {
+#ifndef COCCI
+#include "auth_dirs.inc"
+#endif
+  NULL
+};
+
+/** List of fallback directory authorities. The list is generated by opt-in of
+ * relays that meet certain stability criteria.
+ */
+static const char *default_fallbacks[] = {
+#ifndef COCCI
+#include "fallback_dirs.inc"
+#endif
+  NULL
+};
+
 /** Override default values with these if the user sets the TestingTorNetwork
  * option. */
-static const config_var_t testing_tor_network_defaults[] = {
-  V(DirAllowPrivateAddresses,    BOOL,     "1"),
-  V(EnforceDistinctSubnets,      BOOL,     "0"),
-  V(AssumeReachable,             BOOL,     "1"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "0"),
-  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
-    "0"),
-  V(ClientDNSRejectInternalAddresses, BOOL,"0"),
-  V(ClientRejectInternalAddresses, BOOL,   "0"),
-  V(CountPrivateBandwidth,       BOOL,     "1"),
-  V(ExitPolicyRejectPrivate,     BOOL,     "0"),
-  V(ExtendAllowPrivateAddresses, BOOL,     "1"),
-  V(V3AuthVotingInterval,        INTERVAL, "5 minutes"),
-  V(V3AuthVoteDelay,             INTERVAL, "20 seconds"),
-  V(V3AuthDistDelay,             INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"),
-  V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"),
-  V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"),
-  V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"),
-  V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"),
-  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"),
-  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"),
-  V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"),
-  V(TestingEnableConnBwEvent,    BOOL,     "1"),
-  V(TestingEnableCellStatsEvent, BOOL,     "1"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"),
-  V(RendPostPeriod,              INTERVAL, "2 minutes"),
-
-  END_OF_CONFIG_VARS
+static const struct {
+  const char *k;
+  const char *v;
+} testing_tor_network_defaults[] = {
+#ifndef COCCI
+#include "testnet.inc"
+#endif
+  { NULL, NULL }
 };
 
 #undef VAR
@@ -788,35 +810,24 @@ static const config_deprecation_t option_deprecation_notes_[] = {
     "effect on clients since 0.2.8." },
   /* End of options deprecated since 0.3.2.2-alpha. */
 
+  /* Options deprecated since 0.4.3.1-alpha. */
+  { "ClientAutoIPv6ORPort", "This option is unreliable if a connection isn't "
+    "reliably dual-stack."},
+  /* End of options deprecated since 0.4.3.1-alpha. */
+
   { NULL, NULL }
 };
 
 #ifdef _WIN32
 static char *get_windows_conf_root(void);
 #endif
-static int options_act_reversible(const or_options_t *old_options, char **msg);
-static int options_transition_allowed(const or_options_t *old,
-                                      const or_options_t *new,
-                                      char **msg);
-static int options_transition_affects_workers(
-      const or_options_t *old_options, const or_options_t *new_options);
-static int options_transition_affects_descriptor(
-      const or_options_t *old_options, const or_options_t *new_options);
-static int options_transition_affects_dirauth_timing(
-      const or_options_t *old_options, const or_options_t *new_options);
-static int normalize_nickname_list(config_line_t **normalized_out,
-                                   const config_line_t *lst, const char *name,
-                                   char **msg);
-static char *get_bindaddr_from_transport_listen_line(const char *line,
-                                                     const char *transport);
-static int check_server_ports(const smartlist_t *ports,
-                              const or_options_t *options,
-                              int *num_low_ports_out);
+
+static int options_check_transition_cb(const void *old,
+                                       const void *new,
+                                       char **msg);
 static int validate_data_directories(or_options_t *options);
 static int write_configuration_file(const char *fname,
                                     const or_options_t *options);
-static int options_init_logs(const or_options_t *old_options,
-                             or_options_t *options, int validate_only);
 
 static void init_libevent(const or_options_t *options);
 static int opt_streq(const char *s1, const char *s2);
@@ -824,27 +835,37 @@ static int parse_outbound_addresses(or_options_t *options, int validate_only,
                                     char **msg);
 static void config_maybe_load_geoip_files_(const or_options_t *options,
                                            const or_options_t *old_options);
-static int options_validate_cb(void *old_options, void *options,
-                               void *default_options,
-                               int from_setconf, char **msg);
-static void options_free_cb(void *options);
+static int options_validate_cb(const void *old_options, void *options,
+                               char **msg);
 static void cleanup_protocol_warning_severity_level(void);
 static void set_protocol_warning_severity_level(int warning_severity);
+static void options_clear_cb(const config_mgr_t *mgr, void *opts);
+static setopt_err_t options_validate_and_set(const or_options_t *old_options,
+                                             or_options_t *new_options,
+                                             char **msg_out);
+struct listener_transaction_t;
+static void options_rollback_listener_transaction(
+                                           struct listener_transaction_t *xn);
 
 /** Magic value for or_options_t. */
 #define OR_OPTIONS_MAGIC 9090909
 
 /** Configuration format for or_options_t. */
-STATIC config_format_t options_format = {
-  sizeof(or_options_t),
-  OR_OPTIONS_MAGIC,
-  offsetof(or_options_t, magic_),
-  option_abbrevs_,
-  option_deprecation_notes_,
-  option_vars_,
-  options_validate_cb,
-  options_free_cb,
-  NULL
+static const config_format_t options_format = {
+  .size = sizeof(or_options_t),
+  .magic = {
+   "or_options_t",
+   OR_OPTIONS_MAGIC,
+   offsetof(or_options_t, magic_),
+  },
+  .abbrevs = option_abbrevs_,
+  .deprecations = option_deprecation_notes_,
+  .vars = option_vars_,
+  .legacy_validate_fn = options_validate_cb,
+  .check_transition_fn = options_check_transition_cb,
+  .clear_fn = options_clear_cb,
+  .has_config_suite = true,
+  .config_suite_offset = offsetof(or_options_t, subconfigs_),
 };
 
 /*
@@ -860,29 +881,36 @@ static or_options_t *global_default_options = NULL;
 static char *torrc_fname = NULL;
 /** Name of the most recently read torrc-defaults file.*/
 static char *torrc_defaults_fname = NULL;
-/** Configuration options set by command line. */
-static config_line_t *global_cmdline_options = NULL;
-/** Non-configuration options set by the command line */
-static config_line_t *global_cmdline_only_options = NULL;
-/** Boolean: Have we parsed the command line? */
-static int have_parsed_cmdline = 0;
-/** Contents of most recently read DirPortFrontPage file. */
-static char *global_dirfrontpagecontents = NULL;
+/** Result of parsing the command line. */
+static parsed_cmdline_t *global_cmdline = NULL;
 /** List of port_cfg_t for all configured ports. */
 static smartlist_t *configured_ports = NULL;
 /** True iff we're currently validating options, and any calls to
  * get_options() are likely to be bugs. */
 static int in_option_validation = 0;
-/* True iff we've initialized libevent */
-static int libevent_initialized = 0;
+/** True iff we have run options_act_once_on_startup() */
+static bool have_set_startup_options = false;
 
-/** Return the contents of our frontpage string, or NULL if not configured. */
-MOCK_IMPL(const char*,
-get_dirportfrontpage, (void))
+/* A global configuration manager to handle all configuration objects. */
+static config_mgr_t *options_mgr = NULL;
+
+/** Return the global configuration manager object for torrc options. */
+STATIC const config_mgr_t *
+get_options_mgr(void)
 {
-  return global_dirfrontpagecontents;
+  if (PREDICT_UNLIKELY(options_mgr == NULL)) {
+    options_mgr = config_mgr_new(&options_format);
+    int rv = subsystems_register_options_formats(options_mgr);
+    tor_assert(rv == 0);
+    config_mgr_freeze(options_mgr);
+  }
+  return options_mgr;
 }
 
+#define CHECK_OPTIONS_MAGIC(opt) STMT_BEGIN                      \
+    config_check_toplevel_magic(get_options_mgr(), (opt));       \
+  STMT_END
+
 /** Returns the currently configured options. */
 MOCK_IMPL(or_options_t *,
 get_options_mutable, (void))
@@ -899,6 +927,32 @@ get_options,(void))
   return get_options_mutable();
 }
 
+/**
+ * True iff we have noticed that this is a testing tor network, and we
+ * should use the corresponding defaults.
+ **/
+static bool testing_network_configured = false;
+
+/** Return a set of lines for any default options that we want to override
+ * from those set in our config_var_t values. */
+static config_line_t *
+get_options_defaults(void)
+{
+  int i;
+  config_line_t *result = NULL, **next = &result;
+
+  if (testing_network_configured) {
+    for (i = 0; testing_tor_network_defaults[i].k; ++i) {
+      config_line_append(next,
+                         testing_tor_network_defaults[i].k,
+                         testing_tor_network_defaults[i].v);
+      next = &(*next)->next;
+    }
+  }
+
+  return result;
+}
+
 /** Change the current global options to contain <b>new_val</b> instead of
  * their current value; take action based on the new value; free the old value
  * as necessary.  Returns 0 on success, -1 on failure.
@@ -906,9 +960,6 @@ get_options,(void))
 int
 set_options(or_options_t *new_val, char **msg)
 {
-  int i;
-  smartlist_t *elements;
-  config_line_t *line;
   or_options_t *old_options = global_options;
   global_options = new_val;
   /* Note that we pass the *old* options below, for comparison. It
@@ -918,7 +969,8 @@ set_options(or_options_t *new_val, char **msg)
     global_options = old_options;
     return -1;
   }
-  if (options_act(old_options) < 0) { /* acting on the options failed. die. */
+  if (subsystems_set_options(get_options_mgr(), new_val) < 0 ||
+      options_act(old_options) < 0) { /* acting on the options failed. die. */
     if (! tor_event_loop_shutdown_is_pending()) {
       log_err(LD_BUG,
               "Acting on config options left us in a broken state. Dying.");
@@ -930,35 +982,10 @@ set_options(or_options_t *new_val, char **msg)
   /* Issues a CONF_CHANGED event to notify controller of the change. If Tor is
    * just starting up then the old_options will be undefined. */
   if (old_options && old_options != global_options) {
-    elements = smartlist_new();
-    for (i=0; options_format.vars[i].name; ++i) {
-      const config_var_t *var = &options_format.vars[i];
-      const char *var_name = var->name;
-      if (var->type == CONFIG_TYPE_LINELIST_S ||
-          var->type == CONFIG_TYPE_OBSOLETE) {
-        continue;
-      }
-      if (!config_is_same(&options_format, new_val, old_options, var_name)) {
-        line = config_get_assigned_option(&options_format, new_val,
-                                          var_name, 1);
-
-        if (line) {
-          config_line_t *next;
-          for (; line; line = next) {
-            next = line->next;
-            smartlist_add(elements, line->key);
-            smartlist_add(elements, line->value);
-            tor_free(line);
-          }
-        } else {
-          smartlist_add_strdup(elements, options_format.vars[i].name);
-          smartlist_add(elements, NULL);
-        }
-      }
-    }
-    control_event_conf_changed(elements);
-    SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-    smartlist_free(elements);
+    config_line_t *changes =
+      config_get_changes(get_options_mgr(), old_options, new_val);
+    control_event_conf_changed(changes);
+    config_free_lines(changes);
   }
 
   if (old_options != global_options) {
@@ -972,49 +999,14 @@ set_options(or_options_t *new_val, char **msg)
   return 0;
 }
 
-/** The version of this Tor process, as parsed. */
-static char *the_tor_version = NULL;
-/** A shorter version of this Tor process's version, for export in our router
- *  descriptor.  (Does not include the git version, if any.) */
-static char *the_short_tor_version = NULL;
-
-/** Return the current Tor version. */
-const char *
-get_version(void)
-{
-  if (the_tor_version == NULL) {
-    if (strlen(tor_git_revision)) {
-      tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(),
-                   tor_git_revision);
-    } else {
-      the_tor_version = tor_strdup(get_short_version());
-    }
-  }
-  return the_tor_version;
-}
-
-/** Return the current Tor version, without any git tag. */
-const char *
-get_short_version(void)
-{
-
-  if (the_short_tor_version == NULL) {
-#ifdef TOR_BUILD_TAG
-    tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG);
-#else
-    the_short_tor_version = tor_strdup(VERSION);
-#endif
-  }
-  return the_short_tor_version;
-}
-
 /** Release additional memory allocated in options
  */
-STATIC void
-or_options_free_(or_options_t *options)
+static void
+options_clear_cb(const config_mgr_t *mgr, void *opts)
 {
-  if (!options)
-    return;
+  (void)mgr;
+  CHECK_OPTIONS_MAGIC(opts);
+  or_options_t *options = opts;
 
   routerset_free(options->ExcludeExitNodesUnion_);
   if (options->NodeFamilySets) {
@@ -1037,7 +1029,14 @@ or_options_free_(or_options_t *options)
   tor_free(options->command_arg);
   tor_free(options->master_key_fname);
   config_free_lines(options->MyFamily);
-  config_free(&options_format, options);
+}
+
+/** Release all memory allocated in options
+ */
+STATIC void
+or_options_free_(or_options_t *options)
+{
+  config_free(get_options_mgr(), options);
 }
 
 /** Release all memory and resources held by global configuration structures.
@@ -1050,11 +1049,7 @@ config_free_all(void)
   or_options_free(global_default_options);
   global_default_options = NULL;
 
-  config_free_lines(global_cmdline_options);
-  global_cmdline_options = NULL;
-
-  config_free_lines(global_cmdline_only_options);
-  global_cmdline_only_options = NULL;
+  parsed_cmdline_free(global_cmdline);
 
   if (configured_ports) {
     SMARTLIST_FOREACH(configured_ports,
@@ -1065,15 +1060,12 @@ config_free_all(void)
 
   tor_free(torrc_fname);
   tor_free(torrc_defaults_fname);
-  tor_free(global_dirfrontpagecontents);
-
-  tor_free(the_short_tor_version);
-  tor_free(the_tor_version);
 
   cleanup_protocol_warning_severity_level();
 
-  have_parsed_cmdline = 0;
-  libevent_initialized = 0;
+  have_set_startup_options = false;
+
+  config_mgr_free(options_mgr);
 }
 
 /** Make <b>address</b> -- a piece of information related to our operation as
@@ -1184,24 +1176,13 @@ init_protocol_warning_severity_level(void)
 static void
 cleanup_protocol_warning_severity_level(void)
 {
-   atomic_counter_destroy(&protocol_warning_severity_level);
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
+  atomic_counter_destroy(&protocol_warning_severity_level);
 }
 
-/** List of default directory authorities */
-
-static const char *default_authorities[] = {
-#include "auth_dirs.inc"
-  NULL
-};
-
-/** List of fallback directory authorities. The list is generated by opt-in of
- * relays that meet certain stability criteria.
- */
-static const char *default_fallbacks[] = {
-#include "fallback_dirs.inc"
-  NULL
-};
-
 /** Add the default directory authorities directly into the trusted dir list,
  * but only add them insofar as they share bits with <b>type</b>.
  * Each authority's bits are restricted to the bits shared with <b>type</b>.
@@ -1237,7 +1218,8 @@ add_default_fallback_dir_servers,(void))
  * user if we changed any dangerous ones.
  */
 static int
-validate_dir_servers(or_options_t *options, or_options_t *old_options)
+validate_dir_servers(const or_options_t *options,
+                     const or_options_t *old_options)
 {
   config_line_t *cl;
 
@@ -1418,32 +1400,29 @@ create_keys_directory(const or_options_t *options)
 /* Helps determine flags to pass to switch_id. */
 static int have_low_ports = -1;
 
-/** Fetch the active option list, and take actions based on it. All of the
- * things we do should survive being done repeatedly.  If present,
- * <b>old_options</b> contains the previous value of the options.
- *
- * Return 0 if all goes well, return -1 if things went badly.
- */
+/** Take case of initial startup tasks that must occur before any of the
+ * transactional option-related changes are allowed. */
 static int
-options_act_reversible(const or_options_t *old_options, char **msg)
+options_act_once_on_startup(char **msg_out)
 {
-  smartlist_t *new_listeners = smartlist_new();
-  or_options_t *options = get_options_mutable();
-  int running_tor = options->command == CMD_RUN_TOR;
-  int set_conn_limit = 0;
-  int r = -1;
-  int logs_marked = 0, logs_initialized = 0;
-  int old_min_log_level = get_min_log_level();
+  if (have_set_startup_options)
+    return 0;
+
+  const or_options_t *options = get_options();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
+  if (!running_tor)
+    return 0;
 
   /* Daemonize _first_, since we only want to open most of this stuff in
    * the subprocess.  Libevent bases can't be reliably inherited across
    * processes. */
-  if (running_tor && options->RunAsDaemon) {
+  if (options->RunAsDaemon) {
     if (! start_daemon_has_been_called())
-      crypto_prefork();
+      subsystems_prefork();
     /* No need to roll back, since you can't change the value. */
     if (start_daemon())
-      crypto_postfork();
+      subsystems_postfork();
   }
 
 #ifdef HAVE_SYSTEMD
@@ -1451,105 +1430,43 @@ options_act_reversible(const or_options_t *old_options, char **msg)
   sd_notifyf(0, "MAINPID=%ld\n", (long int)getpid());
 #endif
 
-#ifndef HAVE_SYS_UN_H
-  if (options->ControlSocket || options->ControlSocketsGroupWritable) {
-    *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported "
-                      "on this OS/with this build.");
-    goto rollback;
-  }
-#else /* !(!defined(HAVE_SYS_UN_H)) */
-  if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
-    *msg = tor_strdup("Setting ControlSocketGroupWritable without setting"
-                      "a ControlSocket makes no sense.");
-    goto rollback;
-  }
-#endif /* !defined(HAVE_SYS_UN_H) */
-
-  if (running_tor) {
-    int n_ports=0;
-    /* We need to set the connection limit before we can open the listeners. */
-    if (! sandbox_is_active()) {
-      if (set_max_file_descriptors((unsigned)options->ConnLimit,
-                                   &options->ConnLimit_) < 0) {
-        *msg = tor_strdup("Problem with ConnLimit value. "
-                          "See logs for details.");
-        goto rollback;
-      }
-      set_conn_limit = 1;
-    } else {
-      tor_assert(old_options);
-      options->ConnLimit_ = old_options->ConnLimit_;
-    }
-
-    /* Set up libevent.  (We need to do this before we can register the
-     * listeners as listeners.) */
-    if (running_tor && !libevent_initialized) {
-      init_libevent(options);
-      libevent_initialized = 1;
-
-      /* This has to come up after libevent is initialized. */
-      control_initialize_event_queue();
-
-      /*
-       * Initialize the scheduler - this has to come after
-       * options_init_from_torrc() sets up libevent - why yes, that seems
-       * completely sensible to hide the libevent setup in the option parsing
-       * code!  It also needs to happen before init_keys(), so it needs to
-       * happen here too.  How yucky. */
-      scheduler_init();
-    }
-
-    /* Adjust the port configuration so we can launch listeners. */
-    if (parse_ports(options, 0, msg, &n_ports, NULL)) {
-      if (!*msg)
-        *msg = tor_strdup("Unexpected problem parsing port config");
-      goto rollback;
-    }
-
-    /* Set the hibernation state appropriately.*/
-    consider_hibernation(time(NULL));
-
-    /* Launch the listeners.  (We do this before we setuid, so we can bind to
-     * ports under 1024.)  We don't want to rebind if we're hibernating or
-     * shutting down. If networking is disabled, this will close all but the
-     * control listeners, but disable those. */
-    if (!we_are_hibernating()) {
-      if (retry_all_listeners(new_listeners, options->DisableNetwork) < 0) {
-        *msg = tor_strdup("Failed to bind one of the listener ports.");
-        goto rollback;
-      }
-    }
-    if (options->DisableNetwork) {
-      /* Aggressively close non-controller stuff, NOW */
-      log_notice(LD_NET, "DisableNetwork is set. Tor will not make or accept "
-                 "non-control network connections. Shutting down all existing "
-                 "connections.");
-      connection_mark_all_noncontrol_connections();
-      /* We can't complete circuits until the network is re-enabled. */
-      note_that_we_maybe_cant_complete_circuits();
-    }
-  }
+  /* Set up libevent.  (We need to do this before we can register the
+   * listeners as listeners.) */
+  init_libevent(options);
 
-#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
-  /* Open /dev/pf before dropping privileges. */
-  if (options->TransPort_set &&
-      options->TransProxyType_parsed == TPT_DEFAULT) {
-    if (get_pf_socket() < 0) {
-      *msg = tor_strdup("Unable to open /dev/pf for transparent proxy.");
-      goto rollback;
-    }
-  }
-#endif /* defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) */
+  /* This has to come up after libevent is initialized. */
+  control_initialize_event_queue();
 
-  /* Attempt to lock all current and future memory with mlockall() only once */
+  /*
+   * Initialize the scheduler - this has to come after
+   * options_init_from_torrc() sets up libevent - why yes, that seems
+   * completely sensible to hide the libevent setup in the option parsing
+   * code!  It also needs to happen before init_keys(), so it needs to
+   * happen here too.  How yucky. */
+  scheduler_init();
+
+  /* Attempt to lock all current and future memory with mlockall() only once.
+   * This must happen before setuid. */
   if (options->DisableAllSwap) {
     if (tor_mlockall() == -1) {
-      *msg = tor_strdup("DisableAllSwap failure. Do you have proper "
+      *msg_out = tor_strdup("DisableAllSwap failure. Do you have proper "
                         "permissions?");
-      goto done;
+      return -1;
     }
   }
 
+  have_set_startup_options = true;
+  return 0;
+}
+
+/**
+ * Change our user ID if we're configured to do so.
+ **/
+static int
+options_switch_id(char **msg_out)
+{
+  const or_options_t *options = get_options();
+
   /* Setuid/setgid as appropriate */
   if (options->User) {
     tor_assert(have_low_ports != -1);
@@ -1563,11 +1480,52 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     }
     if (switch_id(options->User, switch_id_flags) != 0) {
       /* No need to roll back, since you can't change the value. */
-      *msg = tor_strdup("Problem with User value. See logs for details.");
-      goto done;
+      *msg_out = tor_strdup("Problem with User value. See logs for details.");
+      return -1;
     }
   }
 
+  return 0;
+}
+
+/**
+ * Helper. Given a data directory (<b>datadir</b>) and another directory
+ * (<b>subdir</b>) with respective group-writable permissions
+ * <b>datadir_gr</b> and <b>subdir_gr</b>, compute whether the subdir should
+ * be group-writeable.
+ **/
+static int
+compute_group_readable_flag(const char *datadir,
+                            const char *subdir,
+                            int datadir_gr,
+                            int subdir_gr)
+{
+  if (subdir_gr != -1) {
+    /* The user specified a default for "subdir", so we always obey it. */
+    return subdir_gr;
+  }
+
+  /* The user left the subdir_gr option on "auto." */
+  if (0 == strcmp(subdir, datadir)) {
+    /* The directories are the same, so we use the group-readable flag from
+     * the datadirectory */
+    return datadir_gr;
+  } else {
+    /* The directores are different, so we default to "not group-readable" */
+    return 0;
+  }
+}
+
+/**
+ * Create our DataDirectory, CacheDirectory, and KeyDirectory, and
+ * set their permissions correctly.
+ */
+STATIC int
+options_create_directories(char **msg_out)
+{
+  const or_options_t *options = get_options();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
   /* Ensure data directory is private; create if possible. */
   /* It's okay to do this in "options_act_reversible()" even though it isn't
    * actually reversible, since you can't change the DataDirectory while
@@ -1576,58 +1534,288 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                                       options->DataDirectory,
                                       options->DataDirectoryGroupReadable,
                                       options->User,
-                                      msg) < 0) {
-    goto done;
+                                      msg_out) < 0) {
+    return -1;
   }
+
+  /* We need to handle the group-readable flag for the cache directory and key
+   * directory specially, since they may be the same as the data directory */
+  const int key_dir_group_readable = compute_group_readable_flag(
+                                        options->DataDirectory,
+                                        options->KeyDirectory,
+                                        options->DataDirectoryGroupReadable,
+                                        options->KeyDirectoryGroupReadable);
+
   if (check_and_create_data_directory(running_tor /* create */,
                                       options->KeyDirectory,
-                                      options->KeyDirectoryGroupReadable,
+                                      key_dir_group_readable,
                                       options->User,
-                                      msg) < 0) {
-    goto done;
+                                      msg_out) < 0) {
+    return -1;
   }
 
-  /* We need to handle the group-readable flag for the cache directory
-   * specially, since the directory defaults to being the same as the
-   * DataDirectory. */
-  int cache_dir_group_readable;
-  if (options->CacheDirectoryGroupReadable != -1) {
-    /* If the user specified a value, use their setting */
-    cache_dir_group_readable = options->CacheDirectoryGroupReadable;
-  } else if (!strcmp(options->CacheDirectory, options->DataDirectory)) {
-    /* If the user left the value as "auto", and the cache is the same as the
-     * datadirectory, use the datadirectory setting.
-     */
-    cache_dir_group_readable = options->DataDirectoryGroupReadable;
-  } else {
-    /* Otherwise, "auto" means "not group readable". */
-    cache_dir_group_readable = 0;
-  }
+  const int cache_dir_group_readable = compute_group_readable_flag(
+                                        options->DataDirectory,
+                                        options->CacheDirectory,
+                                        options->DataDirectoryGroupReadable,
+                                        options->CacheDirectoryGroupReadable);
+
   if (check_and_create_data_directory(running_tor /* create */,
                                       options->CacheDirectory,
                                       cache_dir_group_readable,
                                       options->User,
-                                      msg) < 0) {
-    goto done;
+                                      msg_out) < 0) {
+    return -1;
   }
 
-  /* Bail out at this point if we're not going to be a client or server:
-   * we don't run Tor itself. */
-  if (!running_tor)
-    goto commit;
+  return 0;
+}
+
+/** Structure to represent an incomplete configuration of a set of
+ * listeners.
+ *
+ * This structure is generated by options_start_listener_transaction(), and is
+ * either committed by options_commit_listener_transaction() or rolled back by
+ * options_rollback_listener_transaction(). */
+typedef struct listener_transaction_t {
+  bool set_conn_limit; /**< True if we've set the connection limit */
+  unsigned old_conn_limit; /**< If nonzero, previous connlimit value. */
+  smartlist_t *new_listeners; /**< List of new listeners that we opened. */
+} listener_transaction_t;
+
+/**
+ * Start configuring our listeners based on the current value of
+ * get_options().
+ *
+ * The value <b>old_options</b> holds either the previous options object,
+ * or NULL if we're starting for the first time.
+ *
+ * On success, return a listener_transaction_t that we can either roll back or
+ * commit.
+ *
+ * On failure return NULL and write a message into a newly allocated string in
+ * *<b>msg_out</b>.
+ **/
+static listener_transaction_t *
+options_start_listener_transaction(const or_options_t *old_options,
+                                   char **msg_out)
+{
+  listener_transaction_t *xn = tor_malloc_zero(sizeof(listener_transaction_t));
+  xn->new_listeners = smartlist_new();
+  or_options_t *options = get_options_mutable();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
+  if (! running_tor) {
+    return xn;
+  }
+
+  int n_ports=0;
+  /* We need to set the connection limit before we can open the listeners. */
+  if (! sandbox_is_active()) {
+    if (set_max_file_descriptors((unsigned)options->ConnLimit,
+                                 &options->ConnLimit_) < 0) {
+      *msg_out = tor_strdup("Problem with ConnLimit value. "
+                            "See logs for details.");
+      goto rollback;
+    }
+    xn->set_conn_limit = true;
+    if (old_options)
+      xn->old_conn_limit = (unsigned)old_options->ConnLimit;
+  } else {
+    tor_assert(old_options);
+    options->ConnLimit_ = old_options->ConnLimit_;
+  }
+
+  /* Adjust the port configuration so we can launch listeners. */
+  /* 31851: some ports are relay-only */
+  if (parse_ports(options, 0, msg_out, &n_ports, NULL)) {
+    if (!*msg_out)
+      *msg_out = tor_strdup("Unexpected problem parsing port config");
+    goto rollback;
+  }
+
+  /* Set the hibernation state appropriately.*/
+  consider_hibernation(time(NULL));
+
+  /* Launch the listeners.  (We do this before we setuid, so we can bind to
+   * ports under 1024.)  We don't want to rebind if we're hibernating or
+   * shutting down. If networking is disabled, this will close all but the
+   * control listeners, but disable those. */
+  /* 31851: some listeners are relay-only */
+  if (!we_are_hibernating()) {
+    if (retry_all_listeners(xn->new_listeners,
+                            options->DisableNetwork) < 0) {
+      *msg_out = tor_strdup("Failed to bind one of the listener ports.");
+      goto rollback;
+    }
+  }
+  if (options->DisableNetwork) {
+    /* Aggressively close non-controller stuff, NOW */
+    log_notice(LD_NET, "DisableNetwork is set. Tor will not make or accept "
+               "non-control network connections. Shutting down all existing "
+               "connections.");
+    connection_mark_all_noncontrol_connections();
+    /* We can't complete circuits until the network is re-enabled. */
+    note_that_we_maybe_cant_complete_circuits();
+  }
+
+#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
+  /* Open /dev/pf before (possibly) dropping privileges. */
+  if (options->TransPort_set &&
+      options->TransProxyType_parsed == TPT_DEFAULT) {
+    if (get_pf_socket() < 0) {
+      *msg_out = tor_strdup("Unable to open /dev/pf for transparent proxy.");
+      goto rollback;
+    }
+  }
+#endif /* defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) */
+
+  return xn;
+
+ rollback:
+  options_rollback_listener_transaction(xn);
+  return NULL;
+}
+
+/**
+ * Finish configuring the listeners that started to get configured with
+ * <b>xn</b>.  Frees <b>xn</b>.
+ **/
+static void
+options_commit_listener_transaction(listener_transaction_t *xn)
+{
+  tor_assert(xn);
+  if (xn->set_conn_limit) {
+    or_options_t *options = get_options_mutable();
+    /*
+     * If we adjusted the conn limit, recompute the OOS threshold too
+     *
+     * How many possible sockets to keep in reserve?  If we have lots of
+     * possible sockets, keep this below a limit and set ConnLimit_high_thresh
+     * very close to ConnLimit_, but if ConnLimit_ is low, shrink it in
+     * proportion.
+     *
+     * Somewhat arbitrarily, set socks_in_reserve to 5% of ConnLimit_, but
+     * cap it at 64.
+     */
+    int socks_in_reserve = options->ConnLimit_ / 20;
+    if (socks_in_reserve > 64) socks_in_reserve = 64;
+
+    options->ConnLimit_high_thresh = options->ConnLimit_ - socks_in_reserve;
+    options->ConnLimit_low_thresh = (options->ConnLimit_ / 4) * 3;
+    log_info(LD_GENERAL,
+             "Recomputed OOS thresholds: ConnLimit %d, ConnLimit_ %d, "
+             "ConnLimit_high_thresh %d, ConnLimit_low_thresh %d",
+             options->ConnLimit, options->ConnLimit_,
+             options->ConnLimit_high_thresh,
+             options->ConnLimit_low_thresh);
+
+    /* Give the OOS handler a chance with the new thresholds */
+    connection_check_oos(get_n_open_sockets(), 0);
+  }
+
+  smartlist_free(xn->new_listeners);
+  tor_free(xn);
+}
+
+/**
+ * Revert the listener configuration changes that that started to get
+ * configured with <b>xn</b>.  Frees <b>xn</b>.
+ **/
+static void
+options_rollback_listener_transaction(listener_transaction_t *xn)
+{
+  if (! xn)
+    return;
+
+  or_options_t *options = get_options_mutable();
+
+  if (xn->set_conn_limit && xn->old_conn_limit)
+    set_max_file_descriptors(xn->old_conn_limit, &options->ConnLimit_);
+
+  SMARTLIST_FOREACH(xn->new_listeners, connection_t *, conn,
+  {
+    log_notice(LD_NET, "Closing partially-constructed %s on %s:%d",
+               conn_type_to_string(conn->type), conn->address, conn->port);
+    connection_close_immediate(conn);
+    connection_mark_for_close(conn);
+  });
+
+  smartlist_free(xn->new_listeners);
+  tor_free(xn);
+}
+
+/** Structure to represent an incomplete configuration of a set of logs.
+ *
+ * This structure is generated by options_start_log_transaction(), and is
+ * either committed by options_commit_log_transaction() or rolled back by
+ * options_rollback_log_transaction(). */
+typedef struct log_transaction_t {
+  /** Previous lowest severity of any configured log. */
+  int old_min_log_level;
+  /** True if we have marked the previous logs to be closed */
+  bool logs_marked;
+  /** True if we initialized the new set of logs */
+  bool logs_initialized;
+  /** True if our safelogging configuration is different from what it was
+   * previously (or if we are starting for the first time). */
+  bool safelogging_changed;
+} log_transaction_t;
+
+/**
+ * Start configuring our logs based on the current value of get_options().
+ *
+ * The value <b>old_options</b> holds either the previous options object,
+ * or NULL if we're starting for the first time.
+ *
+ * On success, return a log_transaction_t that we can either roll back or
+ * commit.
+ *
+ * On failure return NULL and write a message into a newly allocated string in
+ * *<b>msg_out</b>.
+ **/
+STATIC log_transaction_t *
+options_start_log_transaction(const or_options_t *old_options,
+                              char **msg_out)
+{
+  const or_options_t *options = get_options();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
+  log_transaction_t *xn = tor_malloc_zero(sizeof(log_transaction_t));
+  xn->old_min_log_level = get_min_log_level();
+  xn->safelogging_changed = !old_options ||
+    old_options->SafeLogging_ != options->SafeLogging_;
+
+  if (! running_tor)
+    goto done;
 
   mark_logs_temp(); /* Close current logs once new logs are open. */
-  logs_marked = 1;
+  xn->logs_marked = true;
   /* Configure the tor_log(s) */
   if (options_init_logs(old_options, options, 0)<0) {
-    *msg = tor_strdup("Failed to init Log options. See logs for details.");
-    goto rollback;
+    *msg_out = tor_strdup("Failed to init Log options. See logs for details.");
+    options_rollback_log_transaction(xn);
+    xn = NULL;
+    goto done;
   }
-  logs_initialized = 1;
 
- commit:
-  r = 0;
-  if (logs_marked) {
+  xn->logs_initialized = true;
+
+ done:
+  return xn;
+}
+
+/**
+ * Finish configuring the logs that started to get configured with <b>xn</b>.
+ * Frees <b>xn</b>.
+ **/
+STATIC void
+options_commit_log_transaction(log_transaction_t *xn)
+{
+  const or_options_t *options = get_options();
+  tor_assert(xn);
+
+  if (xn->logs_marked) {
     log_severity_list_t *severity =
       tor_malloc_zero(sizeof(log_severity_list_t));
     close_temp_logs();
@@ -1637,7 +1825,8 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     tor_free(severity);
     tor_log_update_sigsafe_err_fds();
   }
-  if (logs_initialized) {
+
+  if (xn->logs_initialized) {
     flush_log_messages_from_startup();
   }
 
@@ -1646,12 +1835,12 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     int bad_safelog = 0, bad_severity = 0, new_badness = 0;
     if (options->SafeLogging_ != SAFELOG_SCRUB_ALL) {
       bad_safelog = 1;
-      if (!old_options || old_options->SafeLogging_ != options->SafeLogging_)
+      if (xn->safelogging_changed)
         new_badness = 1;
     }
     if (get_min_log_level() >= LOG_INFO) {
       bad_severity = 1;
-      if (get_min_log_level() != old_min_log_level)
+      if (get_min_log_level() != xn->old_min_log_level)
         new_badness = 1;
     }
     if (bad_safelog && bad_severity)
@@ -1667,59 +1856,105 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                "Overwrite the log afterwards.", badness);
   }
 
-  if (set_conn_limit) {
-    /*
-     * If we adjusted the conn limit, recompute the OOS threshold too
-     *
-     * How many possible sockets to keep in reserve?  If we have lots of
-     * possible sockets, keep this below a limit and set ConnLimit_high_thresh
-     * very close to ConnLimit_, but if ConnLimit_ is low, shrink it in
-     * proportion.
-     *
-     * Somewhat arbitrarily, set socks_in_reserve to 5% of ConnLimit_, but
-     * cap it at 64.
-     */
-    int socks_in_reserve = options->ConnLimit_ / 20;
-    if (socks_in_reserve > 64) socks_in_reserve = 64;
+  tor_free(xn);
+}
 
-    options->ConnLimit_high_thresh = options->ConnLimit_ - socks_in_reserve;
-    options->ConnLimit_low_thresh = (options->ConnLimit_ / 4) * 3;
-    log_info(LD_GENERAL,
-             "Recomputed OOS thresholds: ConnLimit %d, ConnLimit_ %d, "
-             "ConnLimit_high_thresh %d, ConnLimit_low_thresh %d",
-             options->ConnLimit, options->ConnLimit_,
-             options->ConnLimit_high_thresh,
-             options->ConnLimit_low_thresh);
+/**
+ * Revert the log configuration changes that that started to get configured
+ * with <b>xn</b>.  Frees <b>xn</b>.
+ **/
+STATIC void
+options_rollback_log_transaction(log_transaction_t *xn)
+{
+  if (!xn)
+    return;
 
-    /* Give the OOS handler a chance with the new thresholds */
-    connection_check_oos(get_n_open_sockets(), 0);
+  if (xn->logs_marked) {
+    rollback_log_changes();
+    control_adjust_event_log_severity();
   }
 
+  tor_free(xn);
+}
+
+/**
+ * Fetch the active option list, and take actions based on it. All of
+ * the things we do in this function should survive being done
+ * repeatedly, OR be done only once when starting Tor.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * This function is only truly "reversible" _after_ the first time it
+ * is run.  The first time that it runs, it performs some irreversible
+ * tasks in the correct sequence between the reversible option changes.
+ *
+ * Option changes should only be marked as "reversible" if they cannot
+ * be validated before switching them, but they can be switched back if
+ * some other validation fails.
+ *
+ * Return 0 if all goes well, return -1 if things went badly.
+ */
+MOCK_IMPL(STATIC int,
+options_act_reversible,(const or_options_t *old_options, char **msg))
+{
+  const bool first_time = ! have_set_startup_options;
+  log_transaction_t *log_transaction = NULL;
+  listener_transaction_t *listener_transaction = NULL;
+  int r = -1;
+
+  /* The ordering of actions in this function is not free, sadly.
+   *
+   * First of all, we _must_ daemonize before we take all kinds of
+   * initialization actions, since they need to happen in the
+   * subprocess.
+   */
+  if (options_act_once_on_startup(msg) < 0)
+    goto rollback;
+
+  /* Once we've handled most of once-off initialization, we need to
+   * open our listeners before we switch IDs.  (If we open listeners first,
+   * we might not be able to bind to low ports.)
+   */
+  listener_transaction = options_start_listener_transaction(old_options, msg);
+  if (listener_transaction == NULL)
+    goto rollback;
+
+  if (first_time) {
+    if (options_switch_id(msg) < 0)
+      goto rollback;
+  }
+
+  /* On the other hand, we need to touch the file system _after_ we
+   * switch IDs: otherwise, we'll be making directories and opening files
+   * with the wrong permissions.
+   */
+  if (first_time) {
+    if (options_create_directories(msg) < 0)
+      goto rollback;
+  }
+
+  /* Bail out at this point if we're not going to be a client or server:
+   * we don't run Tor itself. */
+  log_transaction = options_start_log_transaction(old_options, msg);
+  if (log_transaction == NULL)
+    goto rollback;
+
+  // Commit!
+  r = 0;
+
+  options_commit_log_transaction(log_transaction);
+
+  options_commit_listener_transaction(listener_transaction);
+
   goto done;
 
  rollback:
   r = -1;
   tor_assert(*msg);
 
-  if (logs_marked) {
-    rollback_log_changes();
-    control_adjust_event_log_severity();
-  }
-
-  if (set_conn_limit && old_options)
-    set_max_file_descriptors((unsigned)old_options->ConnLimit,
-                             &options->ConnLimit_);
-
-  SMARTLIST_FOREACH(new_listeners, connection_t *, conn,
-  {
-    log_notice(LD_NET, "Closing partially-constructed %s on %s:%d",
-               conn_type_to_string(conn->type), conn->address, conn->port);
-    connection_close_immediate(conn);
-    connection_mark_for_close(conn);
-  });
+  options_rollback_log_transaction(log_transaction);
+  options_rollback_listener_transaction(listener_transaction);
 
  done:
-  smartlist_free(new_listeners);
   return r;
 }
 
@@ -1732,6 +1967,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out)
   int routerset_usage =
     routerset_needs_geoip(options->EntryNodes) ||
     routerset_needs_geoip(options->ExitNodes) ||
+    routerset_needs_geoip(options->MiddleNodes) ||
     routerset_needs_geoip(options->ExcludeExitNodes) ||
     routerset_needs_geoip(options->ExcludeNodes) ||
     routerset_needs_geoip(options->HSLayer2Nodes) ||
@@ -1749,32 +1985,6 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out)
   return bridge_usage || routerset_usage;
 }
 
-/** Return the bandwidthrate that we are going to report to the authorities
- * based on the config options. */
-uint32_t
-get_effective_bwrate(const or_options_t *options)
-{
-  uint64_t bw = options->BandwidthRate;
-  if (bw > options->MaxAdvertisedBandwidth)
-    bw = options->MaxAdvertisedBandwidth;
-  if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate)
-    bw = options->RelayBandwidthRate;
-  /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */
-  return (uint32_t)bw;
-}
-
-/** Return the bandwidthburst that we are going to report to the authorities
- * based on the config options. */
-uint32_t
-get_effective_bwburst(const or_options_t *options)
-{
-  uint64_t bw = options->BandwidthBurst;
-  if (options->RelayBandwidthBurst > 0 && bw > options->RelayBandwidthBurst)
-    bw = options->RelayBandwidthBurst;
-  /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */
-  return (uint32_t)bw;
-}
-
 /* Used in the various options_transition_affects* functions. */
 #define YES_IF_CHANGED_BOOL(opt) \
   if (!CFG_EQ_BOOL(old_options, new_options, opt)) return 1;
@@ -1817,32 +2027,6 @@ options_transition_affects_guards(const or_options_t *old_options,
   return 0;
 }
 
-/**
- * Return true if changing the configuration from <b>old</b> to <b>new</b>
- * affects the timing of the voting subsystem
- */
-static int
-options_transition_affects_dirauth_timing(const or_options_t *old_options,
-                                          const or_options_t *new_options)
-{
-  tor_assert(old_options);
-  tor_assert(new_options);
-
-  if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options))
-    return 1;
-  if (! authdir_mode_v3(new_options))
-    return 0;
-  YES_IF_CHANGED_INT(V3AuthVotingInterval);
-  YES_IF_CHANGED_INT(V3AuthVoteDelay);
-  YES_IF_CHANGED_INT(V3AuthDistDelay);
-  YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval);
-  YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay);
-  YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay);
-  YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset);
-
-  return 0;
-}
-
 /** Fetch the active option list, and take actions based on it. All of the
  * things we do should survive being done repeatedly.  If present,
  * <b>old_options</b> contains the previous value of the options.
@@ -1850,17 +2034,16 @@ options_transition_affects_dirauth_timing(const or_options_t *old_options,
  * Return 0 if all goes well, return -1 if it's time to die.
  *
  * Note: We haven't moved all the "act on new configuration" logic
- * here yet.  Some is still in do_hup() and other places.
+ * the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
  */
-STATIC int
-options_act(const or_options_t *old_options)
+MOCK_IMPL(STATIC int,
+options_act,(const or_options_t *old_options))
 {
   config_line_t *cl;
   or_options_t *options = get_options_mutable();
   int running_tor = options->command == CMD_RUN_TOR;
   char *msg=NULL;
-  const int transition_affects_workers =
-    old_options && options_transition_affects_workers(old_options, options);
   const int transition_affects_guards =
     old_options && options_transition_affects_guards(old_options, options);
 
@@ -1918,19 +2101,6 @@ options_act(const or_options_t *old_options)
              "in a non-anonymous mode. It will provide NO ANONYMITY.");
   }
 
-  /* If we are a bridge with a pluggable transport proxy but no
-     Extended ORPort, inform the user that they are missing out. */
-  if (server_mode(options) && options->ServerTransportPlugin &&
-      !options->ExtORPort_lines) {
-    log_notice(LD_CONFIG, "We use pluggable transports but the Extended "
-               "ORPort is disabled. Tor and your pluggable transports proxy "
-               "communicate with each other via the Extended ORPort so it "
-               "is suggested you enable it: it will also allow your Bridge "
-               "to collect statistics about its clients that use pluggable "
-               "transports. Please enable it using the ExtORPort torrc option "
-               "(e.g. set 'ExtORPort auto').");
-  }
-
   if (options->Bridges) {
     mark_bridge_list();
     for (cl = options->Bridges; cl; cl = cl->next) {
@@ -1980,22 +2150,17 @@ options_act(const or_options_t *old_options)
   if (! or_state_loaded() && running_tor) {
     if (or_state_load())
       return -1;
-    rep_hist_load_mtbf_data(time(NULL));
-  }
-
-  /* If we have an ExtORPort, initialize its auth cookie. */
-  if (running_tor &&
-      init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) {
-    log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file.");
-    return -1;
+    if (options_act_dirauth_mtbf(options) < 0)
+      return -1;
   }
 
+  /* 31851: some of the code in these functions is relay-only */
   mark_transport_list();
   pt_prepare_proxy_list_for_config_read();
   if (!options->DisableNetwork) {
     if (options->ClientTransportPlugin) {
       for (cl = options->ClientTransportPlugin; cl; cl = cl->next) {
-        if (parse_transport_line(options, cl->value, 0, 0) < 0) {
+        if (pt_parse_transport_line(options, cl->value, 0, 0) < 0) {
           // LCOV_EXCL_START
           log_warn(LD_BUG,
                    "Previously validated ClientTransportPlugin line "
@@ -2005,20 +2170,11 @@ options_act(const or_options_t *old_options)
         }
       }
     }
-
-    if (options->ServerTransportPlugin && server_mode(options)) {
-      for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
-        if (parse_transport_line(options, cl->value, 0, 1) < 0) {
-          // LCOV_EXCL_START
-          log_warn(LD_BUG,
-                   "Previously validated ServerTransportPlugin line "
-                   "could not be added!");
-          return -1;
-          // LCOV_EXCL_STOP
-        }
-      }
-    }
   }
+
+  if (options_act_server_transport(old_options) < 0)
+    return -1;
+
   sweep_transport_list();
   sweep_proxy_list();
 
@@ -2039,19 +2195,8 @@ options_act(const or_options_t *old_options)
     finish_daemon(options->DataDirectory);
   }
 
-  /* See whether we need to enable/disable our once-a-second timer. */
-  reschedule_per_second_timer();
-
-  /* We want to reinit keys as needed before we do much of anything else:
-     keys are important, and other things can depend on them. */
-  if (transition_affects_workers ||
-      (options->V3AuthoritativeDir && (!old_options ||
-                                       !old_options->V3AuthoritativeDir))) {
-    if (init_keys() < 0) {
-      log_warn(LD_BUG,"Error initializing keys; exiting");
-      return -1;
-    }
-  }
+  if (options_act_relay(old_options) < 0)
+    return -1;
 
   /* Write our PID to the PID file. If we do not have write permissions we
    * will log a warning and exit. */
@@ -2075,15 +2220,6 @@ options_act(const or_options_t *old_options)
     return -1;
   }
 
-  if (server_mode(options)) {
-    static int cdm_initialized = 0;
-    if (cdm_initialized == 0) {
-      cdm_initialized = 1;
-      consdiffmgr_configure(NULL);
-      consdiffmgr_validate();
-    }
-  }
-
   if (init_control_cookie_authentication(options->CookieAuthentication) < 0) {
     log_warn(LD_CONFIG,"Error creating control cookie authentication file.");
     return -1;
@@ -2101,15 +2237,8 @@ options_act(const or_options_t *old_options)
    * might be a change of scheduler or parameter. */
   scheduler_conf_changed();
 
-  /* Set up accounting */
-  if (accounting_parse_options(options, 0)<0) {
-    // LCOV_EXCL_START
-    log_warn(LD_BUG,"Error in previously validated accounting options");
+  if (options_act_relay_accounting(old_options) < 0)
     return -1;
-    // LCOV_EXCL_STOP
-  }
-  if (accounting_is_enabled(options))
-    configure_accounting(time(NULL));
 
   /* Change the cell EWMA settings */
   cmux_ewma_set_options(options, networkstatus_get_latest_consensus());
@@ -2133,6 +2262,7 @@ options_act(const or_options_t *old_options)
     tor_free(http_authenticator);
   }
 
+  /* 31851: OutboundBindAddressExit is relay-only */
   if (parse_outbound_addresses(options, 0, &msg) < 0) {
     // LCOV_EXCL_START
     log_warn(LD_BUG, "Failed parsing previously validated outbound "
@@ -2174,6 +2304,7 @@ options_act(const or_options_t *old_options)
                          options->HSLayer2Nodes) ||
         !routerset_equal(old_options->HSLayer3Nodes,
                          options->HSLayer3Nodes) ||
+        !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) ||
         options->StrictNodes != old_options->StrictNodes) {
       log_info(LD_CIRC,
                "Changed to using entry guards or bridges, or changed "
@@ -2218,65 +2349,17 @@ options_act(const or_options_t *old_options)
     if (revise_automap_entries)
       addressmap_clear_invalid_automaps(options);
 
-/* How long should we delay counting bridge stats after becoming a bridge?
- * We use this so we don't count clients who used our bridge thinking it is
- * a relay. If you change this, don't forget to change the log message
- * below. It's 4 hours (the time it takes to stop being used by clients)
- * plus some extra time for clock skew. */
-#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60)
-
-    if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) {
-      int was_relay = 0;
-      if (options->BridgeRelay) {
-        time_t int_start = time(NULL);
-        if (config_lines_eq(old_options->ORPort_lines,options->ORPort_lines)) {
-          int_start += RELAY_BRIDGE_STATS_DELAY;
-          was_relay = 1;
-        }
-        geoip_bridge_stats_init(int_start);
-        log_info(LD_CONFIG, "We are acting as a bridge now.  Starting new "
-                 "GeoIP stats interval%s.", was_relay ? " in 6 "
-                 "hours from now" : "");
-      } else {
-        geoip_bridge_stats_term();
-        log_info(LD_GENERAL, "We are no longer acting as a bridge.  "
-                 "Forgetting GeoIP stats.");
-      }
-    }
-
-    if (transition_affects_workers) {
-      log_info(LD_GENERAL,
-               "Worker-related options changed. Rotating workers.");
-      const int server_mode_turned_on =
-        server_mode(options) && !server_mode(old_options);
-      const int dir_server_mode_turned_on =
-        dir_server_mode(options) && !dir_server_mode(old_options);
-
-      if (server_mode_turned_on || dir_server_mode_turned_on) {
-        cpu_init();
-      }
+    if (options_act_bridge_stats(old_options) < 0)
+      return -1;
 
-      if (server_mode_turned_on) {
-        ip_address_changed(0);
-        if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL)))
-          inform_testing_reachability();
-      }
-      cpuworkers_rotate_keyinfo();
-      if (dns_reset())
-        return -1;
-    } else {
-      if (dns_reset())
-        return -1;
-    }
+    if (dns_reset())
+      return -1;
 
-    if (options->PerConnBWRate != old_options->PerConnBWRate ||
-        options->PerConnBWBurst != old_options->PerConnBWBurst)
-      connection_or_update_token_buckets(get_connection_array(), options);
+    if (options_act_relay_bandwidth(old_options) < 0)
+      return -1;
 
     if (options->BandwidthRate != old_options->BandwidthRate ||
-        options->BandwidthBurst != old_options->BandwidthBurst ||
-        options->RelayBandwidthRate != old_options->RelayBandwidthRate ||
-        options->RelayBandwidthBurst != old_options->RelayBandwidthBurst)
+        options->BandwidthBurst != old_options->BandwidthBurst)
       connection_bucket_adjust(options);
 
     if (options->MainloopStats != old_options->MainloopStats) {
@@ -2284,132 +2367,43 @@ options_act(const or_options_t *old_options)
     }
   }
 
+  /* 31851: These options are relay-only, but we need to disable them if we
+   * are in client mode. In 29211, we will disable all relay options in
+   * client mode. */
   /* Only collect directory-request statistics on relays and bridges. */
   options->DirReqStatistics = options->DirReqStatistics_option &&
     server_mode(options);
   options->HiddenServiceStatistics =
     options->HiddenServiceStatistics_option && server_mode(options);
 
-  if (options->CellStatistics || options->DirReqStatistics ||
-      options->EntryStatistics || options->ExitPortStatistics ||
-      options->ConnDirectionStatistics ||
-      options->HiddenServiceStatistics ||
-      options->BridgeAuthoritativeDir) {
-    time_t now = time(NULL);
-    int print_notice = 0;
-
-    /* Only collect other relay-only statistics on relays. */
-    if (!public_server_mode(options)) {
-      options->CellStatistics = 0;
-      options->EntryStatistics = 0;
-      options->ConnDirectionStatistics = 0;
-      options->ExitPortStatistics = 0;
-    }
-
-    if ((!old_options || !old_options->CellStatistics) &&
-        options->CellStatistics) {
-      rep_hist_buffer_stats_init(now);
-      print_notice = 1;
-    }
-    if ((!old_options || !old_options->DirReqStatistics) &&
-        options->DirReqStatistics) {
-      if (geoip_is_loaded(AF_INET)) {
-        geoip_dirreq_stats_init(now);
-        print_notice = 1;
-      } else {
-        /* disable statistics collection since we have no geoip file */
-        options->DirReqStatistics = 0;
-        if (options->ORPort_set)
-          log_notice(LD_CONFIG, "Configured to measure directory request "
-                                "statistics, but no GeoIP database found. "
-                                "Please specify a GeoIP database using the "
-                                "GeoIPFile option.");
-      }
-    }
-    if ((!old_options || !old_options->EntryStatistics) &&
-        options->EntryStatistics && !should_record_bridge_info(options)) {
-      /* If we get here, we've started recording bridge info when we didn't
-       * do so before.  Note that "should_record_bridge_info()" will
-       * always be false at this point, because of the earlier block
-       * that cleared EntryStatistics when public_server_mode() was false.
-       * We're leaving it in as defensive programming. */
-      if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) {
-        geoip_entry_stats_init(now);
-        print_notice = 1;
-      } else {
-        options->EntryStatistics = 0;
-        log_notice(LD_CONFIG, "Configured to measure entry node "
-                              "statistics, but no GeoIP database found. "
-                              "Please specify a GeoIP database using the "
-                              "GeoIPFile option.");
-      }
-    }
-    if ((!old_options || !old_options->ExitPortStatistics) &&
-        options->ExitPortStatistics) {
-      rep_hist_exit_stats_init(now);
-      print_notice = 1;
-    }
-    if ((!old_options || !old_options->ConnDirectionStatistics) &&
-        options->ConnDirectionStatistics) {
-      rep_hist_conn_stats_init(now);
-    }
-    if ((!old_options || !old_options->HiddenServiceStatistics) &&
-        options->HiddenServiceStatistics) {
-      log_info(LD_CONFIG, "Configured to measure hidden service statistics.");
-      rep_hist_hs_stats_init(now);
-    }
-    if ((!old_options || !old_options->BridgeAuthoritativeDir) &&
-        options->BridgeAuthoritativeDir) {
-      rep_hist_desc_stats_init(now);
-      print_notice = 1;
-    }
-    if (print_notice)
-        log_notice(LD_CONFIG, "Configured to measure statistics. Look for "
-                "the *-stats files that will first be written to the "
-                 "data directory in 24 hours from now.");
-  }
-
-  /* If we used to have statistics enabled but we just disabled them,
-     stop gathering them.  */
-  if (old_options && old_options->CellStatistics &&
-      !options->CellStatistics)
-    rep_hist_buffer_stats_term();
-  if (old_options && old_options->DirReqStatistics &&
-      !options->DirReqStatistics)
-    geoip_dirreq_stats_term();
-  if (old_options && old_options->EntryStatistics &&
-      !options->EntryStatistics)
-    geoip_entry_stats_term();
-  if (old_options && old_options->HiddenServiceStatistics &&
-      !options->HiddenServiceStatistics)
-    rep_hist_hs_stats_term();
-  if (old_options && old_options->ExitPortStatistics &&
-      !options->ExitPortStatistics)
-    rep_hist_exit_stats_term();
-  if (old_options && old_options->ConnDirectionStatistics &&
-      !options->ConnDirectionStatistics)
-    rep_hist_conn_stats_term();
-  if (old_options && old_options->BridgeAuthoritativeDir &&
-      !options->BridgeAuthoritativeDir)
-    rep_hist_desc_stats_term();
-
-  /* Since our options changed, we might need to regenerate and upload our
-   * server descriptor.
-   */
-  if (!old_options ||
-      options_transition_affects_descriptor(old_options, options))
-    mark_my_descriptor_dirty("config change");
+  /* Only collect other relay-only statistics on relays. */
+  if (!public_server_mode(options)) {
+    options->CellStatistics = 0;
+    options->EntryStatistics = 0;
+    options->ConnDirectionStatistics = 0;
+    options->ExitPortStatistics = 0;
+  }
+
+  bool print_notice = 0;
+  if (options_act_relay_stats(old_options, &print_notice) < 0)
+    return -1;
+  if (options_act_dirauth_stats(old_options, &print_notice) < 0)
+    return -1;
+  if (print_notice)
+    options_act_relay_stats_msg();
+
+  if (options_act_relay_desc(old_options) < 0)
+    return -1;
+
+  if (options_act_dirauth(old_options) < 0)
+    return -1;
 
   /* We may need to reschedule some directory stuff if our status changed. */
   if (old_options) {
-    if (options_transition_affects_dirauth_timing(old_options, options)) {
-      voting_schedule_recalculate_timing(options, time(NULL));
-      reschedule_dirvote(options);
-    }
-    if (!bool_eq(directory_fetches_dir_info_early(options),
-                 directory_fetches_dir_info_early(old_options)) ||
-        !bool_eq(directory_fetches_dir_info_later(options),
-                 directory_fetches_dir_info_later(old_options)) ||
+    if (!bool_eq(dirclient_fetches_dir_info_early(options),
+                 dirclient_fetches_dir_info_early(old_options)) ||
+        !bool_eq(dirclient_fetches_dir_info_later(options),
+                 dirclient_fetches_dir_info_later(old_options)) ||
         !config_lines_eq(old_options->Bridges, options->Bridges)) {
       /* Make sure update_router_have_minimum_dir_info() gets called. */
       router_dir_info_changed();
@@ -2419,87 +2413,113 @@ options_act(const or_options_t *old_options)
     }
   }
 
-  /* DoS mitigation subsystem only applies to public relay. */
-  if (public_server_mode(options)) {
-    /* If we are configured as a relay, initialize the subsystem. Even on HUP,
-     * this is safe to call as it will load data from the current options
-     * or/and the consensus. */
-    dos_init();
-  } else if (old_options && public_server_mode(old_options)) {
-    /* Going from relay to non relay, clean it up. */
-    dos_free_all();
-  }
-
-  /* Load the webpage we're going to serve every time someone asks for '/' on
-     our DirPort. */
-  tor_free(global_dirfrontpagecontents);
-  if (options->DirPortFrontPage) {
-    global_dirfrontpagecontents =
-      read_file_to_str(options->DirPortFrontPage, 0, NULL);
-    if (!global_dirfrontpagecontents) {
-      log_warn(LD_CONFIG,
-               "DirPortFrontPage file '%s' not found. Continuing anyway.",
-               options->DirPortFrontPage);
-    }
-  }
+  if (options_act_relay_dos(old_options) < 0)
+    return -1;
+  if (options_act_relay_dir(old_options) < 0)
+    return -1;
 
   return 0;
 }
 
+/**
+ * Enumeration to describe the syntax for a command-line option.
+ **/
 typedef enum {
-  TAKES_NO_ARGUMENT = 0,
+  /** Describe an option that does not take an argument. */
+  ARGUMENT_NONE = 0,
+  /** Describes an option that takes a single argument. */
   ARGUMENT_NECESSARY = 1,
+  /** Describes an option that takes a single optional argument. */
   ARGUMENT_OPTIONAL = 2
 } takes_argument_t;
 
+/** Table describing arguments that Tor accepts on the command line,
+ * other than those that are the same as in torrc. */
 static const struct {
+  /** The string that the user has to provide. */
   const char *name;
+  /** Does this option accept an argument? */
   takes_argument_t takes_argument;
+  /** If not CMD_RUN_TOR, what should Tor do when it starts? */
+  tor_cmdline_mode_t command;
+  /** If nonzero, set the quiet level to this. 1 is "hush", 2 is "quiet" */
+  int quiet;
 } CMDLINE_ONLY_OPTIONS[] = {
-  { "-f",                     ARGUMENT_NECESSARY },
-  { "--allow-missing-torrc",  TAKES_NO_ARGUMENT },
-  { "--defaults-torrc",       ARGUMENT_NECESSARY },
-  { "--hash-password",        ARGUMENT_NECESSARY },
-  { "--dump-config",          ARGUMENT_OPTIONAL },
-  { "--list-fingerprint",     TAKES_NO_ARGUMENT },
-  { "--keygen",               TAKES_NO_ARGUMENT },
-  { "--key-expiration",       ARGUMENT_OPTIONAL },
-  { "--newpass",              TAKES_NO_ARGUMENT },
-  { "--no-passphrase",        TAKES_NO_ARGUMENT },
-  { "--passphrase-fd",        ARGUMENT_NECESSARY },
-  { "--verify-config",        TAKES_NO_ARGUMENT },
-  { "--ignore-missing-torrc", TAKES_NO_ARGUMENT },
-  { "--quiet",                TAKES_NO_ARGUMENT },
-  { "--hush",                 TAKES_NO_ARGUMENT },
-  { "--version",              TAKES_NO_ARGUMENT },
-  { "--list-modules",         TAKES_NO_ARGUMENT },
-  { "--library-versions",     TAKES_NO_ARGUMENT },
-  { "-h",                     TAKES_NO_ARGUMENT },
-  { "--help",                 TAKES_NO_ARGUMENT },
-  { "--list-torrc-options",   TAKES_NO_ARGUMENT },
-  { "--list-deprecated-options",TAKES_NO_ARGUMENT },
-  { "--nt-service",           TAKES_NO_ARGUMENT },
-  { "-nt-service",            TAKES_NO_ARGUMENT },
-  { NULL, 0 },
+  { .name="-f",
+    .takes_argument=ARGUMENT_NECESSARY },
+  { .name="--allow-missing-torrc" },
+  { .name="--defaults-torrc",
+    .takes_argument=ARGUMENT_NECESSARY },
+  { .name="--hash-password",
+    .takes_argument=ARGUMENT_NECESSARY,
+    .command=CMD_HASH_PASSWORD,
+    .quiet=QUIET_HUSH  },
+  { .name="--dump-config",
+    .takes_argument=ARGUMENT_OPTIONAL,
+    .command=CMD_DUMP_CONFIG,
+    .quiet=QUIET_SILENT },
+  { .name="--list-fingerprint",
+    .command=CMD_LIST_FINGERPRINT },
+  { .name="--keygen",
+    .command=CMD_KEYGEN },
+  { .name="--key-expiration",
+    .takes_argument=ARGUMENT_OPTIONAL,
+    .command=CMD_KEY_EXPIRATION },
+  { .name="--newpass" },
+  { .name="--no-passphrase" },
+  { .name="--passphrase-fd",
+    .takes_argument=ARGUMENT_NECESSARY },
+  { .name="--verify-config",
+    .command=CMD_VERIFY_CONFIG },
+  { .name="--ignore-missing-torrc" },
+  { .name="--quiet",
+    .quiet=QUIET_SILENT },
+  { .name="--hush",
+    .quiet=QUIET_HUSH },
+  { .name="--version",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--list-modules",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--library-versions",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="-h",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--help",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH  },
+  { .name="--list-torrc-options",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--list-deprecated-options",
+    .command=CMD_IMMEDIATE },
+  { .name="--nt-service" },
+  { .name="-nt-service" },
+  { .name="--dbg-dump-subsystem-list",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name=NULL },
 };
 
 /** Helper: Read a list of configuration options from the command line.  If
- * successful, or if ignore_errors is set, put them in *<b>result</b>, put the
- * commandline-only options in *<b>cmdline_result</b>, and return 0;
- * otherwise, return -1 and leave *<b>result</b> and <b>cmdline_result</b>
- * alone. */
-int
-config_parse_commandline(int argc, char **argv, int ignore_errors,
-                         config_line_t **result,
-                         config_line_t **cmdline_result)
+ * successful, return a newly allocated parsed_cmdline_t; otherwise return
+ * NULL.
+ *
+ * If <b>ignore_errors</b> is set, try to recover from all recoverable
+ * errors and return the best command line we can.
+ */
+parsed_cmdline_t *
+config_parse_commandline(int argc, char **argv, int ignore_errors)
 {
+  parsed_cmdline_t *result = tor_malloc_zero(sizeof(parsed_cmdline_t));
+  result->command = CMD_RUN_TOR;
   config_line_t *param = NULL;
 
-  config_line_t *front = NULL;
-  config_line_t **new = &front;
-
-  config_line_t *front_cmdline = NULL;
-  config_line_t **new_cmdline = &front_cmdline;
+  config_line_t **new_cmdline = &result->cmdline_opts;
+  config_line_t **new = &result->other_opts;
 
   char *s, *arg;
   int i = 1;
@@ -2509,11 +2529,19 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
     takes_argument_t want_arg = ARGUMENT_NECESSARY;
     int is_cmdline = 0;
     int j;
+    bool is_a_command = false;
 
     for (j = 0; CMDLINE_ONLY_OPTIONS[j].name != NULL; ++j) {
       if (!strcmp(argv[i], CMDLINE_ONLY_OPTIONS[j].name)) {
         is_cmdline = 1;
         want_arg = CMDLINE_ONLY_OPTIONS[j].takes_argument;
+        if (CMDLINE_ONLY_OPTIONS[j].command != CMD_RUN_TOR) {
+          is_a_command = true;
+          result->command = CMDLINE_ONLY_OPTIONS[j].command;
+        }
+        quiet_level_t quiet = CMDLINE_ONLY_OPTIONS[j].quiet;
+        if (quiet > result->quiet_level)
+          result->quiet_level = quiet;
         break;
       }
     }
@@ -2544,26 +2572,29 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
       } else {
         log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.",
             argv[i]);
-        config_free_lines(front);
-        config_free_lines(front_cmdline);
-        return -1;
+        parsed_cmdline_free(result);
+        return NULL;
       }
     } else if (want_arg == ARGUMENT_OPTIONAL && is_last) {
       arg = tor_strdup("");
     } else {
-      arg = (want_arg != TAKES_NO_ARGUMENT) ? tor_strdup(argv[i+1]) :
+      arg = (want_arg != ARGUMENT_NONE) ? tor_strdup(argv[i+1]) :
                                               tor_strdup("");
     }
 
     param = tor_malloc_zero(sizeof(config_line_t));
     param->key = is_cmdline ? tor_strdup(argv[i]) :
-                   tor_strdup(config_expand_abbrev(&options_format, s, 1, 1));
+                 tor_strdup(config_expand_abbrev(get_options_mgr(), s, 1, 1));
     param->value = arg;
     param->command = command;
     param->next = NULL;
     log_debug(LD_CONFIG, "command line: parsed keyword '%s', value '%s'",
         param->key, param->value);
 
+    if (is_a_command) {
+      result->command_arg = param->value;
+    }
+
     if (is_cmdline) {
       *new_cmdline = param;
       new_cmdline = &((*new_cmdline)->next);
@@ -2574,17 +2605,26 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
 
     i += want_arg ? 2 : 1;
   }
-  *cmdline_result = front_cmdline;
-  *result = front;
-  return 0;
+
+  return result;
+}
+
+/** Release all storage held by <b>cmdline</b>. */
+void
+parsed_cmdline_free_(parsed_cmdline_t *cmdline)
+{
+  if (!cmdline)
+    return;
+  config_free_lines(cmdline->cmdline_opts);
+  config_free_lines(cmdline->other_opts);
+  tor_free(cmdline);
 }
 
 /** Return true iff key is a valid configuration option. */
 int
 option_is_recognized(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return (var != NULL);
+  return config_find_option_name(get_options_mgr(), key) != NULL;
 }
 
 /** Return the canonical name of a configuration option, or NULL
@@ -2592,8 +2632,7 @@ option_is_recognized(const char *key)
 const char *
 option_get_canonical_name(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return var ? var->name : NULL;
+  return config_find_option_name(get_options_mgr(), key);
 }
 
 /** Return a canonical list of the options assigned for key.
@@ -2601,7 +2640,7 @@ option_get_canonical_name(const char *key)
 config_line_t *
 option_get_assignment(const or_options_t *options, const char *key)
 {
-  return config_get_assigned_option(&options_format, options, key, 1);
+  return config_get_assigned_option(get_options_mgr(), options, key, 1);
 }
 
 /** Try assigning <b>list</b> to the global options. You do this by duping
@@ -2617,44 +2656,16 @@ setopt_err_t
 options_trial_assign(config_line_t *list, unsigned flags, char **msg)
 {
   int r;
-  or_options_t *trial_options = config_dup(&options_format, get_options());
+  or_options_t *trial_options = config_dup(get_options_mgr(), get_options());
 
-  if ((r=config_assign(&options_format, trial_options,
+  if ((r=config_assign(get_options_mgr(), trial_options,
                        list, flags, msg)) < 0) {
     or_options_free(trial_options);
     return r;
   }
+  const or_options_t *cur_options = get_options();
 
-  setopt_err_t rv;
-  or_options_t *cur_options = get_options_mutable();
-
-  in_option_validation = 1;
-
-  if (options_validate(cur_options, trial_options,
-                       global_default_options, 1, msg) < 0) {
-    or_options_free(trial_options);
-    rv = SETOPT_ERR_PARSE; /*XXX make this a separate return value. */
-    goto done;
-  }
-
-  if (options_transition_allowed(cur_options, trial_options, msg) < 0) {
-    or_options_free(trial_options);
-    rv = SETOPT_ERR_TRANSITION;
-    goto done;
-  }
-  in_option_validation = 0;
-
-  if (set_options(trial_options, msg)<0) {
-    or_options_free(trial_options);
-    rv = SETOPT_ERR_SETTING;
-    goto done;
-  }
-
-  /* we liked it. put it in place. */
-  rv = SETOPT_OK;
- done:
-  in_option_validation = 0;
-  return rv;
+  return options_validate_and_set(cur_options, trial_options, msg);
 }
 
 /** Print a usage message for tor. */
@@ -2664,7 +2675,7 @@ print_usage(void)
   printf(
 "Copyright (c) 2001-2004, Roger Dingledine\n"
 "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n"
-"Copyright (c) 2007-2019, The Tor Project, Inc.\n\n"
+"Copyright (c) 2007-2020, The Tor Project, Inc.\n\n"
 "tor -f <torrc> [args]\n"
 "See man page for options, or https://www.torproject.org/ for "
 "documentation.\n");
@@ -2674,48 +2685,41 @@ print_usage(void)
 static void
 list_torrc_options(void)
 {
-  int i;
-  for (i = 0; option_vars_[i].name; ++i) {
-    const config_var_t *var = &option_vars_[i];
-    if (var->type == CONFIG_TYPE_OBSOLETE ||
-        var->type == CONFIG_TYPE_LINELIST_V)
+  smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+  SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+    /* Possibly this should check listable, rather than (or in addition to)
+     * settable. See ticket 31654.
+     */
+    if (! config_var_is_settable(var)) {
+      /* This variable cannot be set, or cannot be set by this name. */
       continue;
-    printf("%s\n", var->name);
-  }
+    }
+    printf("%s\n", var->member.name);
+  } SMARTLIST_FOREACH_END(var);
+  smartlist_free(vars);
 }
 
 /** Print all deprecated but non-obsolete torrc options. */
 static void
 list_deprecated_options(void)
 {
-  const config_deprecation_t *d;
-  for (d = option_deprecation_notes_; d->name; ++d) {
-    printf("%s\n", d->name);
-  }
+  smartlist_t *deps = config_mgr_list_deprecated_vars(get_options_mgr());
+  /* Possibly this should check whether the variables are listable,
+   * but currently it does not.  See ticket 31654. */
+  SMARTLIST_FOREACH(deps, const char *, name,
+                    printf("%s\n", name));
+  smartlist_free(deps);
 }
 
 /** Print all compile-time modules and their enabled/disabled status. */
 static void
 list_enabled_modules(void)
 {
+  printf("%s: %s\n", "relay", have_module_relay() ? "yes" : "no");
   printf("%s: %s\n", "dirauth", have_module_dirauth() ? "yes" : "no");
-}
-
-/** Last value actually set by resolve_my_address. */
-static uint32_t last_resolved_addr = 0;
-
-/** Accessor for last_resolved_addr from outside this file. */
-uint32_t
-get_last_resolved_addr(void)
-{
-  return last_resolved_addr;
-}
-
-/** Reset last_resolved_addr from outside this file. */
-void
-reset_last_resolved_addr(void)
-{
-  last_resolved_addr = 0;
+  // We don't list dircache, because it cannot be enabled or disabled
+  // independently from relay.  Listing it here would proliferate
+  // test variants in test_parseconf.sh to no useful purpose.
 }
 
 /* Return true if <b>options</b> is using the default authorities, and false
@@ -2726,282 +2730,13 @@ using_default_dir_authorities(const or_options_t *options)
   return (!options->DirAuthorities && !options->AlternateDirAuthority);
 }
 
-/**
- * Attempt getting our non-local (as judged by tor_addr_is_internal()
- * function) IP address using following techniques, listed in
- * order from best (most desirable, try first) to worst (least
- * desirable, try if everything else fails).
- *
- * First, attempt using <b>options-\>Address</b> to get our
- * non-local IP address.
- *
- * If <b>options-\>Address</b> represents a non-local IP address,
- * consider it ours.
- *
- * If <b>options-\>Address</b> is a DNS name that resolves to
- * a non-local IP address, consider this IP address ours.
- *
- * If <b>options-\>Address</b> is NULL, fall back to getting local
- * hostname and using it in above-described ways to try and
- * get our IP address.
- *
- * In case local hostname cannot be resolved to a non-local IP
- * address, try getting an IP address of network interface
- * in hopes it will be non-local one.
- *
- * Fail if one or more of the following is true:
- *   - DNS name in <b>options-\>Address</b> cannot be resolved.
- *   - <b>options-\>Address</b> is a local host address.
- *   - Attempt at getting local hostname fails.
- *   - Attempt at getting network interface address fails.
- *
- * Return 0 if all is well, or -1 if we can't find a suitable
- * public IP address.
- *
- * If we are returning 0:
- *   - Put our public IP address (in host order) into *<b>addr_out</b>.
- *   - If <b>method_out</b> is non-NULL, set *<b>method_out</b> to a static
- *     string describing how we arrived at our answer.
- *      - "CONFIGURED" - parsed from IP address string in
- *        <b>options-\>Address</b>
- *      - "RESOLVED" - resolved from DNS name in <b>options-\>Address</b>
- *      - "GETHOSTNAME" - resolved from a local hostname.
- *      - "INTERFACE" - retrieved from a network interface.
- *   - If <b>hostname_out</b> is non-NULL, and we resolved a hostname to
- *     get our address, set *<b>hostname_out</b> to a newly allocated string
- *     holding that hostname. (If we didn't get our address by resolving a
- *     hostname, set *<b>hostname_out</b> to NULL.)
- *
- * XXXX ipv6
- */
-int
-resolve_my_address(int warn_severity, const or_options_t *options,
-                   uint32_t *addr_out,
-                   const char **method_out, char **hostname_out)
-{
-  struct in_addr in;
-  uint32_t addr; /* host order */
-  char hostname[256];
-  const char *method_used;
-  const char *hostname_used;
-  int explicit_ip=1;
-  int explicit_hostname=1;
-  int from_interface=0;
-  char *addr_string = NULL;
-  const char *address = options->Address;
-  int notice_severity = warn_severity <= LOG_NOTICE ?
-                          LOG_NOTICE : warn_severity;
-
-  tor_addr_t myaddr;
-  tor_assert(addr_out);
-
-  /*
-   * Step one: Fill in 'hostname' to be our best guess.
-   */
-
-  if (address && *address) {
-    strlcpy(hostname, address, sizeof(hostname));
-  } else { /* then we need to guess our address */
-    explicit_ip = 0; /* it's implicit */
-    explicit_hostname = 0; /* it's implicit */
-
-    if (tor_gethostname(hostname, sizeof(hostname)) < 0) {
-      log_fn(warn_severity, LD_NET,"Error obtaining local hostname");
-      return -1;
-    }
-    log_debug(LD_CONFIG, "Guessed local host name as '%s'", hostname);
-  }
-
-  /*
-   * Step two: Now that we know 'hostname', parse it or resolve it. If
-   * it doesn't parse or resolve, look at the interface address. Set 'addr'
-   * to be our (host-order) 32-bit answer.
-   */
-
-  if (tor_inet_aton(hostname, &in) == 0) {
-    /* then we have to resolve it */
-    explicit_ip = 0;
-    if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */
-      uint32_t interface_ip; /* host order */
-
-      if (explicit_hostname) {
-        log_fn(warn_severity, LD_CONFIG,
-               "Could not resolve local Address '%s'. Failing.", hostname);
-        return -1;
-      }
-      log_fn(notice_severity, LD_CONFIG,
-             "Could not resolve guessed local hostname '%s'. "
-             "Trying something else.", hostname);
-      if (get_interface_address(warn_severity, &interface_ip)) {
-        log_fn(warn_severity, LD_CONFIG,
-               "Could not get local interface IP address. Failing.");
-        return -1;
-      }
-      from_interface = 1;
-      addr = interface_ip;
-      log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for "
-             "local interface. Using that.", fmt_addr32(addr));
-      strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
-    } else { /* resolved hostname into addr */
-      tor_addr_from_ipv4h(&myaddr, addr);
-
-      if (!explicit_hostname &&
-          tor_addr_is_internal(&myaddr, 0)) {
-        tor_addr_t interface_ip;
-
-        log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' "
-               "resolves to a private IP address (%s). Trying something "
-               "else.", hostname, fmt_addr32(addr));
-
-        if (get_interface_address6(warn_severity, AF_INET, &interface_ip)<0) {
-          log_fn(warn_severity, LD_CONFIG,
-                 "Could not get local interface IP address. Too bad.");
-        } else if (tor_addr_is_internal(&interface_ip, 0)) {
-          log_fn(notice_severity, LD_CONFIG,
-                 "Interface IP address '%s' is a private address too. "
-                 "Ignoring.", fmt_addr(&interface_ip));
-        } else {
-          from_interface = 1;
-          addr = tor_addr_to_ipv4h(&interface_ip);
-          log_fn(notice_severity, LD_CONFIG,
-                 "Learned IP address '%s' for local interface."
-                 " Using that.", fmt_addr32(addr));
-          strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
-        }
-      }
-    }
-  } else {
-    addr = ntohl(in.s_addr); /* set addr so that addr_string is not
-                              * illformed */
-  }
-
-  /*
-   * Step three: Check whether 'addr' is an internal IP address, and error
-   * out if it is and we don't want that.
-   */
-
-  tor_addr_from_ipv4h(&myaddr,addr);
-
-  addr_string = tor_dup_ip(addr);
-  if (tor_addr_is_internal(&myaddr, 0)) {
-    /* make sure we're ok with publishing an internal IP */
-    if (using_default_dir_authorities(options)) {
-      /* if they are using the default authorities, disallow internal IPs
-       * always. */
-      log_fn(warn_severity, LD_CONFIG,
-             "Address '%s' resolves to private IP address '%s'. "
-             "Tor servers that use the default DirAuthorities must have "
-             "public IP addresses.", hostname, addr_string);
-      tor_free(addr_string);
-      return -1;
-    }
-    if (!explicit_ip) {
-      /* even if they've set their own authorities, require an explicit IP if
-       * they're using an internal address. */
-      log_fn(warn_severity, LD_CONFIG, "Address '%s' resolves to private "
-             "IP address '%s'. Please set the Address config option to be "
-             "the IP address you want to use.", hostname, addr_string);
-      tor_free(addr_string);
-      return -1;
-    }
-  }
-
-  /*
-   * Step four: We have a winner! 'addr' is our answer for sure, and
-   * 'addr_string' is its string form. Fill out the various fields to
-   * say how we decided it.
-   */
-
-  log_debug(LD_CONFIG, "Resolved Address to '%s'.", addr_string);
-
-  if (explicit_ip) {
-    method_used = "CONFIGURED";
-    hostname_used = NULL;
-  } else if (explicit_hostname) {
-    method_used = "RESOLVED";
-    hostname_used = hostname;
-  } else if (from_interface) {
-    method_used = "INTERFACE";
-    hostname_used = NULL;
-  } else {
-    method_used = "GETHOSTNAME";
-    hostname_used = hostname;
-  }
-
-  *addr_out = addr;
-  if (method_out)
-    *method_out = method_used;
-  if (hostname_out)
-    *hostname_out = hostname_used ? tor_strdup(hostname_used) : NULL;
-
-  /*
-   * Step five: Check if the answer has changed since last time (or if
-   * there was no last time), and if so call various functions to keep
-   * us up-to-date.
-   */
-
-  if (last_resolved_addr && last_resolved_addr != *addr_out) {
-    /* Leave this as a notice, regardless of the requested severity,
-     * at least until dynamic IP address support becomes bulletproof. */
-    log_notice(LD_NET,
-               "Your IP address seems to have changed to %s "
-               "(METHOD=%s%s%s). Updating.",
-               addr_string, method_used,
-               hostname_used ? " HOSTNAME=" : "",
-               hostname_used ? hostname_used : "");
-    ip_address_changed(0);
-  }
-
-  if (last_resolved_addr != *addr_out) {
-    control_event_server_status(LOG_NOTICE,
-                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=%s%s%s",
-                                addr_string, method_used,
-                                hostname_used ? " HOSTNAME=" : "",
-                                hostname_used ? hostname_used : "");
-  }
-  last_resolved_addr = *addr_out;
-
-  /*
-   * And finally, clean up and return success.
-   */
-
-  tor_free(addr_string);
-  return 0;
-}
-
-/** Return true iff <b>addr</b> is judged to be on the same network as us, or
- * on a private network.
- */
-MOCK_IMPL(int,
-is_local_addr, (const tor_addr_t *addr))
-{
-  if (tor_addr_is_internal(addr, 0))
-    return 1;
-  /* Check whether ip is on the same /24 as we are. */
-  if (get_options()->EnforceDistinctSubnets == 0)
-    return 0;
-  if (tor_addr_family(addr) == AF_INET) {
-    uint32_t ip = tor_addr_to_ipv4h(addr);
-
-    /* It's possible that this next check will hit before the first time
-     * resolve_my_address actually succeeds.  (For clients, it is likely that
-     * resolve_my_address will never be called at all).  In those cases,
-     * last_resolved_addr will be 0, and so checking to see whether ip is on
-     * the same /24 as last_resolved_addr will be the same as checking whether
-     * it was on net 0, which is already done by tor_addr_is_internal.
-     */
-    if ((last_resolved_addr & (uint32_t)0xffffff00ul)
-        == (ip & (uint32_t)0xffffff00ul))
-      return 1;
-  }
-  return 0;
-}
-
 /** Return a new empty or_options_t.  Used for testing. */
 or_options_t *
 options_new(void)
 {
-  return config_new(&options_format);
+  or_options_t *options = config_new(get_options_mgr());
+  options->command = CMD_RUN_TOR;
+  return options;
 }
 
 /** Set <b>options</b> to hold reasonable defaults for most options.
@@ -3009,7 +2744,17 @@ options_new(void)
 void
 options_init(or_options_t *options)
 {
-  config_init(&options_format, options);
+  config_init(get_options_mgr(), options);
+  config_line_t *dflts = get_options_defaults();
+  char *msg=NULL;
+  if (config_assign(get_options_mgr(), options, dflts,
+                    CAL_WARN_DEPRECATIONS, &msg)<0) {
+    log_err(LD_BUG, "Unable to set default options: %s", msg);
+    tor_free(msg);
+    tor_assert_unreached();
+  }
+  config_free_lines(dflts);
+  tor_free(msg);
 }
 
 /** Return a string containing a possible configuration file that would give
@@ -3039,7 +2784,7 @@ options_dump(const or_options_t *options, int how_to_dump)
       return NULL;
   }
 
-  return config_dump(&options_format, use_defaults, options, minimal, 0);
+  return config_dump(get_options_mgr(), use_defaults, options, minimal, 0);
 }
 
 /** Return 0 if every element of sl is a string holding a decimal
@@ -3069,8 +2814,8 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg)
  * a complaint into *<b>msg</b> using string <b>desc</b>, and return -1.
  * Else return 0.
  */
-static int
-ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg)
+int
+config_ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg)
 {
   if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) {
     /* This handles an understandable special case where somebody says "2gb"
@@ -3086,48 +2831,6 @@ ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg)
   return 0;
 }
 
-/** Parse an authority type from <b>options</b>-\>PublishServerDescriptor
- * and write it to <b>options</b>-\>PublishServerDescriptor_. Treat "1"
- * as "v3" unless BridgeRelay is 1, in which case treat it as "bridge".
- * Treat "0" as "".
- * Return 0 on success or -1 if not a recognized authority type (in which
- * case the value of PublishServerDescriptor_ is undefined). */
-static int
-compute_publishserverdescriptor(or_options_t *options)
-{
-  smartlist_t *list = options->PublishServerDescriptor;
-  dirinfo_type_t *auth = &options->PublishServerDescriptor_;
-  *auth = NO_DIRINFO;
-  if (!list) /* empty list, answer is none */
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(list, const char *, string) {
-    if (!strcasecmp(string, "v1"))
-      log_warn(LD_CONFIG, "PublishServerDescriptor v1 has no effect, because "
-                          "there are no v1 directory authorities anymore.");
-    else if (!strcmp(string, "1"))
-      if (options->BridgeRelay)
-        *auth |= BRIDGE_DIRINFO;
-      else
-        *auth |= V3_DIRINFO;
-    else if (!strcasecmp(string, "v2"))
-      log_warn(LD_CONFIG, "PublishServerDescriptor v2 has no effect, because "
-                          "there are no v2 directory authorities anymore.");
-    else if (!strcasecmp(string, "v3"))
-      *auth |= V3_DIRINFO;
-    else if (!strcasecmp(string, "bridge"))
-      *auth |= BRIDGE_DIRINFO;
-    else if (!strcasecmp(string, "hidserv"))
-      log_warn(LD_CONFIG,
-               "PublishServerDescriptor hidserv is invalid. See "
-               "PublishHidServDescriptors.");
-    else if (!strcasecmp(string, "") || !strcmp(string, "0"))
-      /* no authority */;
-    else
-      return -1;
-  } SMARTLIST_FOREACH_END(string);
-  return 0;
-}
-
 /** Lowest allowable value for RendPostPeriod; if this is too low, hidden
  * services can overload the directory system. */
 #define MIN_REND_POST_PERIOD (10*60)
@@ -3160,23 +2863,67 @@ compute_publishserverdescriptor(or_options_t *options)
  * */
 #define RECOMMENDED_MIN_CIRCUIT_BUILD_TIMEOUT (10)
 
-static int
-options_validate_cb(void *old_options, void *options, void *default_options,
-                    int from_setconf, char **msg)
+/**
+ * Validate <b>new_options</b>.  If it is valid, and it is a reasonable
+ * replacement for <b>old_options</b>, replace the previous value of the
+ * global options, and return return SETOPT_OK.
+ *
+ * If it is not valid, then free <b>new_options</b>, set *<b>msg_out</b> to a
+ * newly allocated error message, and return an error code.
+ */
+static setopt_err_t
+options_validate_and_set(const or_options_t *old_options,
+                         or_options_t *new_options,
+                         char **msg_out)
 {
+  setopt_err_t rv;
+  validation_status_t vs;
+
   in_option_validation = 1;
-  int rv = options_validate(old_options, options, default_options,
-                          from_setconf, msg);
+  vs = config_validate(get_options_mgr(), old_options, new_options, msg_out);
+
+  if (vs == VSTAT_TRANSITION_ERR) {
+    rv = SETOPT_ERR_TRANSITION;
+    goto err;
+  } else if (vs < 0) {
+    rv = SETOPT_ERR_PARSE;
+    goto err;
+  }
+  in_option_validation = 0;
+
+  if (set_options(new_options, msg_out)) {
+    rv = SETOPT_ERR_SETTING;
+    goto err;
+  }
+
+  rv = SETOPT_OK;
+  new_options = NULL; /* prevent free */
+ err:
   in_option_validation = 0;
+  tor_assert(new_options == NULL || rv != SETOPT_OK);
+  or_options_free(new_options);
   return rv;
 }
 
-/** Callback to free an or_options_t */
-static void
-options_free_cb(void *options)
+#ifdef TOR_UNIT_TESTS
+/**
+ * Return 0 if every setting in <b>options</b> is reasonable, is a
+ * permissible transition from <b>old_options</b>, and none of the
+ * testing-only settings differ from <b>default_options</b> unless in
+ * testing mode.  Else return -1.  Should have no side effects, except for
+ * normalizing the contents of <b>options</b>.
+ *
+ * On error, tor_strdup an error explanation into *<b>msg</b>.
+ */
+int
+options_validate(const or_options_t *old_options, or_options_t *options,
+                 char **msg)
 {
-  or_options_free_(options);
+  validation_status_t vs;
+  vs = config_validate(get_options_mgr(), old_options, options, msg);
+  return vs < 0 ? -1 : 0;
 }
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
@@ -3198,7 +2945,7 @@ options_free_cb(void *options)
  */
 static int
 warn_if_option_path_is_relative(const char *option,
-                                char *filepath)
+                                const char *filepath)
 {
   if (filepath && path_is_relative(filepath)) {
     char *abs_path = make_path_absolute(filepath);
@@ -3211,34 +2958,29 @@ warn_if_option_path_is_relative(const char *option,
 }
 
 /** Scan <b>options</b> for occurrences of relative file/directory
- * path and log a warning whenever it is found.
+ * paths and log a warning whenever one is found.
  *
  * Return 1 if there were relative paths; 0 otherwise.
  */
 static int
-warn_about_relative_paths(or_options_t *options)
+warn_about_relative_paths(const or_options_t *options)
 {
   tor_assert(options);
   int n = 0;
+  const config_mgr_t *mgr = get_options_mgr();
 
-  n += warn_if_option_path_is_relative("CookieAuthFile",
-                                       options->CookieAuthFile);
-  n += warn_if_option_path_is_relative("ExtORPortCookieAuthFile",
-                                       options->ExtORPortCookieAuthFile);
-  n += warn_if_option_path_is_relative("DirPortFrontPage",
-                                       options->DirPortFrontPage);
-  n += warn_if_option_path_is_relative("V3BandwidthsFile",
-                                       options->V3BandwidthsFile);
-  n += warn_if_option_path_is_relative("ControlPortWriteToFile",
-                                       options->ControlPortWriteToFile);
-  n += warn_if_option_path_is_relative("GeoIPFile",options->GeoIPFile);
-  n += warn_if_option_path_is_relative("GeoIPv6File",options->GeoIPv6File);
-  n += warn_if_option_path_is_relative("Log",options->DebugLogFile);
-  n += warn_if_option_path_is_relative("AccelDir",options->AccelDir);
-  n += warn_if_option_path_is_relative("DataDirectory",options->DataDirectory);
-  n += warn_if_option_path_is_relative("PidFile",options->PidFile);
-  n += warn_if_option_path_is_relative("ClientOnionAuthDir",
-                                        options->ClientOnionAuthDir);
+  smartlist_t *vars = config_mgr_list_vars(mgr);
+  SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, cv) {
+    config_line_t *line;
+    if (cv->member.type != CONFIG_TYPE_FILENAME)
+      continue;
+    const char *name = cv->member.name;
+    line = config_get_assigned_option(mgr, options, name, 0);
+    if (line)
+      n += warn_if_option_path_is_relative(name, line->value);
+    config_free_lines(line);
+  } SMARTLIST_FOREACH_END(cv);
+  smartlist_free(vars);
 
   for (config_line_t *hs_line = options->RendConfigLines; hs_line;
        hs_line = hs_line->next) {
@@ -3265,6 +3007,10 @@ options_validate_scheduler(or_options_t *options, char **msg)
            "can be used or set at least one value.");
   }
   /* Ok, we do have scheduler types, validate them. */
+  if (options->SchedulerTypes_) {
+    SMARTLIST_FOREACH(options->SchedulerTypes_, int *, iptr, tor_free(iptr));
+    smartlist_free(options->SchedulerTypes_);
+  }
   options->SchedulerTypes_ = smartlist_new();
   SMARTLIST_FOREACH_BEGIN(options->Schedulers, const char *, type) {
     int *sched_type;
@@ -3357,25 +3103,20 @@ options_validate_single_onion(or_options_t *options, char **msg)
   return 0;
 }
 
-/** Return 0 if every setting in <b>options</b> is reasonable, is a
- * permissible transition from <b>old_options</b>, and none of the
- * testing-only settings differ from <b>default_options</b> unless in
- * testing mode.  Else return -1.  Should have no side effects, except for
- * normalizing the contents of <b>options</b>.
- *
- * On error, tor_strdup an error explanation into *<b>msg</b>.
- *
- * XXX
- * If <b>from_setconf</b>, we were called by the controller, and our
- * Log line should stay empty. If it's 0, then give us a default log
- * if there are no logs defined.
+/**
+ * Legacy validation/normalization callback for or_options_t.  See
+ * legacy_validate_fn_t for more information.
  */
-STATIC int
-options_validate(or_options_t *old_options, or_options_t *options,
-                 or_options_t *default_options, int from_setconf, char **msg)
+static int
+options_validate_cb(const void *old_options_, void *options_, char **msg)
 {
+  if (old_options_)
+    CHECK_OPTIONS_MAGIC(old_options_);
+  CHECK_OPTIONS_MAGIC(options_);
+  const or_options_t *old_options = old_options_;
+  or_options_t *options = options_;
+
   config_line_t *cl;
-  const char *uname = get_uname();
   int n_ports=0;
   int world_writable_control_socket=0;
 
@@ -3386,22 +3127,30 @@ options_validate(or_options_t *old_options, or_options_t *options,
                   &world_writable_control_socket) < 0)
     return -1;
 
+#ifndef HAVE_SYS_UN_H
+  if (options->ControlSocket || options->ControlSocketsGroupWritable) {
+    *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported "
+                      "on this OS/with this build.");
+    return -1;
+  }
+#else /* defined(HAVE_SYS_UN_H) */
+  if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
+    *msg = tor_strdup("Setting ControlSocketGroupWritable without setting "
+                      "a ControlSocket makes no sense.");
+    return -1;
+  }
+#endif /* !defined(HAVE_SYS_UN_H) */
+
   /* Set UseEntryGuards from the configured value, before we check it below.
    * We change UseEntryGuards when it's incompatible with other options,
    * but leave UseEntryGuards_option with the original value.
    * Always use the value of UseEntryGuards, not UseEntryGuards_option. */
   options->UseEntryGuards = options->UseEntryGuards_option;
 
-  if (server_mode(options) &&
-      (!strcmpstart(uname, "Windows 95") ||
-       !strcmpstart(uname, "Windows 98") ||
-       !strcmpstart(uname, "Windows Me"))) {
-    log_warn(LD_CONFIG, "Tor is running as a server, but you are "
-        "running %s; this probably won't work. See "
-        "https://www.torproject.org/docs/faq.html#BestOSForRelay "
-        "for details.", uname);
-  }
+  if (options_validate_relay_os(old_options, options, msg) < 0)
+    return -1;
 
+  /* 31851: OutboundBindAddressExit is unused in client mode */
   if (parse_outbound_addresses(options, 1, msg) < 0)
     return -1;
 
@@ -3416,59 +3165,16 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "with relative paths.");
   }
 
-  if (options->Nickname == NULL) {
-    if (server_mode(options)) {
-      options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
-    }
-  } else {
-    if (!is_legal_nickname(options->Nickname)) {
-      tor_asprintf(msg,
-          "Nickname '%s', nicknames must be between 1 and 19 characters "
-          "inclusive, and must contain only the characters [a-zA-Z0-9].",
-          options->Nickname);
-      return -1;
-    }
-  }
-
-  if (server_mode(options) && !options->ContactInfo) {
-    log_warn(LD_CONFIG,
-             "Your ContactInfo config option is not set. Please strongly "
-             "consider setting it, so we can contact you if your relay is "
-             "misconfigured, end-of-life, or something else goes wrong. "
-             "It is also possible that your relay might get rejected from "
-             "the network due to a missing valid contact address.");
-  }
-
-  const char *ContactInfo = options->ContactInfo;
-  if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo)))
-    REJECT("ContactInfo config option must be UTF-8.");
+  if (options_validate_relay_info(old_options, options, msg) < 0)
+    return -1;
 
-  /* Special case on first boot if no Log options are given. */
-  if (!options->Logs && !options->RunAsDaemon && !from_setconf) {
-    if (quiet_level == 0)
-      config_line_append(&options->Logs, "Log", "notice stdout");
-    else if (quiet_level == 1)
-      config_line_append(&options->Logs, "Log", "warn stdout");
-  }
+  /* 31851: this function is currently a no-op in client mode */
+  check_network_configuration(server_mode(options));
 
   /* Validate the tor_log(s) */
   if (options_init_logs(old_options, options, 1)<0)
     REJECT("Failed to validate Log options. See logs for details.");
 
-  if (authdir_mode(options)) {
-    /* confirm that our address isn't broken, so we can complain now */
-    uint32_t tmp;
-    if (resolve_my_address(LOG_WARN, options, &tmp, NULL, NULL) < 0)
-      REJECT("Failed to resolve/guess local address. See logs for details.");
-  }
-
-  if (server_mode(options) && options->RendConfigLines)
-    log_warn(LD_CONFIG,
-        "Tor is currently configured as a relay and a hidden service. "
-        "That's not very secure: you should probably run your hidden service "
-        "in a separate Tor process, at least -- see "
-        "https://trac.torproject.org/8742");
-
   /* XXXX require that the only port not be DirPort? */
   /* XXXX require that at least one port be listened-upon. */
   if (n_ports == 0 && !options->RendConfigLines)
@@ -3483,13 +3189,13 @@ options_validate(or_options_t *old_options, or_options_t *options,
     if (!strcasecmp(options->TransProxyType, "default")) {
       options->TransProxyType_parsed = TPT_DEFAULT;
     } else if (!strcasecmp(options->TransProxyType, "pf-divert")) {
-#if !defined(OpenBSD) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined(DARWIN)
       /* Later versions of OS X have pf */
       REJECT("pf-divert is a OpenBSD-specific "
              "and OS X/Darwin-specific feature.");
 #else
       options->TransProxyType_parsed = TPT_PF_DIVERT;
-#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
+#endif /* !defined(OpenBSD) && !defined(DARWIN) */
     } else if (!strcasecmp(options->TransProxyType, "tproxy")) {
 #if !defined(__linux__)
       REJECT("TPROXY is a Linux-specific feature.");
@@ -3513,7 +3219,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("Cannot use TransProxyType without any valid TransPort.");
     }
   }
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   if (options->TransPort_set)
     REJECT("TransPort is disabled in this build.");
 #endif /* defined(USE_TRANSPARENT) */
@@ -3547,69 +3253,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "features to be broken in unpredictable ways.");
   }
 
-  for (cl = options->RecommendedPackages; cl; cl = cl->next) {
-    if (! validate_recommended_package_line(cl->value)) {
-      log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored",
-               escaped(cl->value));
-    }
-  }
-
-  if (options->AuthoritativeDir) {
-    if (!options->ContactInfo && !options->TestingTorNetwork)
-      REJECT("Authoritative directory servers must set ContactInfo");
-    if (!options->RecommendedClientVersions)
-      options->RecommendedClientVersions =
-        config_lines_dup(options->RecommendedVersions);
-    if (!options->RecommendedServerVersions)
-      options->RecommendedServerVersions =
-        config_lines_dup(options->RecommendedVersions);
-    if (options->VersioningAuthoritativeDir &&
-        (!options->RecommendedClientVersions ||
-         !options->RecommendedServerVersions))
-      REJECT("Versioning authoritative dir servers must set "
-             "Recommended*Versions.");
-
-#ifdef HAVE_MODULE_DIRAUTH
-    char *t;
-    /* Call these functions to produce warnings only. */
-    t = format_recommended_version_list(options->RecommendedClientVersions, 1);
-    tor_free(t);
-    t = format_recommended_version_list(options->RecommendedServerVersions, 1);
-    tor_free(t);
-#endif
-
-    if (options->UseEntryGuards) {
-      log_info(LD_CONFIG, "Authoritative directory servers can't set "
-               "UseEntryGuards. Disabling.");
-      options->UseEntryGuards = 0;
-    }
-    if (!options->DownloadExtraInfo && authdir_mode_v3(options)) {
-      log_info(LD_CONFIG, "Authoritative directories always try to download "
-               "extra-info documents. Setting DownloadExtraInfo.");
-      options->DownloadExtraInfo = 1;
-    }
-    if (!(options->BridgeAuthoritativeDir ||
-          options->V3AuthoritativeDir))
-      REJECT("AuthoritativeDir is set, but none of "
-             "(Bridge/V3)AuthoritativeDir is set.");
-    /* If we have a v3bandwidthsfile and it's broken, complain on startup */
-    if (options->V3BandwidthsFile && !old_options) {
-      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
-    }
-    /* same for guardfraction file */
-    if (options->GuardfractionFile && !old_options) {
-      dirserv_read_guardfraction_file(options->GuardfractionFile, NULL);
-    }
-  }
-
-  if (options->AuthoritativeDir && !options->DirPort_set)
-    REJECT("Running as authoritative directory, but no DirPort set.");
-
-  if (options->AuthoritativeDir && !options->ORPort_set)
-    REJECT("Running as authoritative directory, but no ORPort set.");
-
-  if (options->AuthoritativeDir && options->ClientOnly)
-    REJECT("Running as authoritative directory, but ClientOnly also set.");
+  if (options_validate_dirauth_mode(old_options, options, msg) < 0)
+    return -1;
 
   if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly)
     REJECT("FetchDirInfoExtraEarly requires that you also set "
@@ -3748,49 +3393,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
     return -1;
   }
 
-  if (compute_publishserverdescriptor(options) < 0) {
-    tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor");
+  if (options_validate_publish_server(old_options, options, msg) < 0)
     return -1;
-  }
-
-  if ((options->BridgeRelay
-        || options->PublishServerDescriptor_ & BRIDGE_DIRINFO)
-      && (options->PublishServerDescriptor_ & V3_DIRINFO)) {
-    REJECT("Bridges are not supposed to publish router descriptors to the "
-           "directory authorities. Please correct your "
-           "PublishServerDescriptor line.");
-  }
-
-  if (options->BridgeRelay && options->DirPort_set) {
-    log_warn(LD_CONFIG, "Can't set a DirPort on a bridge relay; disabling "
-             "DirPort");
-    config_free_lines(options->DirPort_lines);
-    options->DirPort_lines = NULL;
-    options->DirPort_set = 0;
-  }
 
-  if (server_mode(options) && options->ConnectionPadding != -1) {
-    REJECT("Relays must use 'auto' for the ConnectionPadding setting.");
-  }
-
-  if (server_mode(options) && options->ReducedConnectionPadding != 0) {
-    REJECT("Relays cannot set ReducedConnectionPadding. ");
-  }
-
-  if (options->BridgeDistribution) {
-    if (!options->BridgeRelay) {
-      REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
-    }
-    if (check_bridge_distribution_setting(options->BridgeDistribution) < 0) {
-      REJECT("Invalid BridgeDistribution value.");
-    }
-  }
-
-  if (options->MinUptimeHidServDirectoryV2 < 0) {
-    log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at "
-                        "least 0 seconds. Changing to 0.");
-    options->MinUptimeHidServDirectoryV2 = 0;
-  }
+  if (options_validate_relay_padding(old_options, options, msg) < 0)
+    return -1;
 
   const int min_rendpostperiod =
     options->TestingTorNetwork ?
@@ -3829,7 +3436,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "UseEntryGuards is disabled, but you have configured one or more "
              "hidden services on this Tor instance.  Your hidden services "
              "will be very easy to locate using a well-known attack -- see "
-             "http://freehaven.net/anonbib/#hs-attack06 for details.");
+             "https://freehaven.net/anonbib/#hs-attack06 for details.");
   }
 
   if (options->NumPrimaryGuards && options->NumEntryGuards &&
@@ -3899,6 +3506,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "default.");
   }
 
+  if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) {
+    REJECT("DormantClientTimeout is too low. It must be at least 10 minutes.");
+  }
+
   if (options->PathBiasNoticeRate > 1.0) {
     tor_asprintf(msg,
               "PathBiasNoticeRate is too high. "
@@ -3950,7 +3561,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   if (options->HeartbeatPeriod &&
-      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) {
+      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD &&
+      !options->TestingTorNetwork) {
     log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; "
              "raising to %d seconds.", MIN_HEARTBEAT_PERIOD);
     options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD;
@@ -3959,134 +3571,24 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->KeepalivePeriod < 1)
     REJECT("KeepalivePeriod option must be positive.");
 
-  if (ensure_bandwidth_cap(&options->BandwidthRate,
+  if (config_ensure_bandwidth_cap(&options->BandwidthRate,
                            "BandwidthRate", msg) < 0)
     return -1;
-  if (ensure_bandwidth_cap(&options->BandwidthBurst,
+  if (config_ensure_bandwidth_cap(&options->BandwidthBurst,
                            "BandwidthBurst", msg) < 0)
     return -1;
-  if (ensure_bandwidth_cap(&options->MaxAdvertisedBandwidth,
-                           "MaxAdvertisedBandwidth", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->RelayBandwidthRate,
-                           "RelayBandwidthRate", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->RelayBandwidthBurst,
-                           "RelayBandwidthBurst", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->PerConnBWRate,
-                           "PerConnBWRate", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->PerConnBWBurst,
-                           "PerConnBWBurst", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->AuthDirFastGuarantee,
-                           "AuthDirFastGuarantee", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->AuthDirGuardBWGuarantee,
-                           "AuthDirGuardBWGuarantee", msg) < 0)
-    return -1;
 
-  if (options->RelayBandwidthRate && !options->RelayBandwidthBurst)
-    options->RelayBandwidthBurst = options->RelayBandwidthRate;
-  if (options->RelayBandwidthBurst && !options->RelayBandwidthRate)
-    options->RelayBandwidthRate = options->RelayBandwidthBurst;
-
-  if (server_mode(options)) {
-    const unsigned required_min_bw =
-      public_server_mode(options) ?
-       RELAY_REQUIRED_MIN_BANDWIDTH : BRIDGE_REQUIRED_MIN_BANDWIDTH;
-    const char * const optbridge =
-      public_server_mode(options) ? "" : "bridge ";
-    if (options->BandwidthRate < required_min_bw) {
-      tor_asprintf(msg,
-                       "BandwidthRate is set to %d bytes/second. "
-                       "For %sservers, it must be at least %u.",
-                       (int)options->BandwidthRate, optbridge,
-                       required_min_bw);
-      return -1;
-    } else if (options->MaxAdvertisedBandwidth <
-               required_min_bw/2) {
-      tor_asprintf(msg,
-                       "MaxAdvertisedBandwidth is set to %d bytes/second. "
-                       "For %sservers, it must be at least %u.",
-                       (int)options->MaxAdvertisedBandwidth, optbridge,
-                       required_min_bw/2);
-      return -1;
-    }
-    if (options->RelayBandwidthRate &&
-      options->RelayBandwidthRate < required_min_bw) {
-      tor_asprintf(msg,
-                       "RelayBandwidthRate is set to %d bytes/second. "
-                       "For %sservers, it must be at least %u.",
-                       (int)options->RelayBandwidthRate, optbridge,
-                       required_min_bw);
-      return -1;
-    }
-  }
-
-  if (options->RelayBandwidthRate > options->RelayBandwidthBurst)
-    REJECT("RelayBandwidthBurst must be at least equal "
-           "to RelayBandwidthRate.");
+  if (options_validate_relay_bandwidth(old_options, options, msg) < 0)
+    return -1;
 
   if (options->BandwidthRate > options->BandwidthBurst)
     REJECT("BandwidthBurst must be at least equal to BandwidthRate.");
 
-  /* if they set relaybandwidth* really high but left bandwidth*
-   * at the default, raise the defaults. */
-  if (options->RelayBandwidthRate > options->BandwidthRate)
-    options->BandwidthRate = options->RelayBandwidthRate;
-  if (options->RelayBandwidthBurst > options->BandwidthBurst)
-    options->BandwidthBurst = options->RelayBandwidthBurst;
-
-  if (accounting_parse_options(options, 1)<0)
-    REJECT("Failed to parse accounting options. See logs for details.");
-
-  if (options->AccountingMax) {
-    if (options->RendConfigLines && server_mode(options)) {
-      log_warn(LD_CONFIG, "Using accounting with a hidden service and an "
-               "ORPort is risky: your hidden service(s) and your public "
-               "address will all turn off at the same time, which may alert "
-               "observers that they are being run by the same party.");
-    } else if (config_count_key(options->RendConfigLines,
-                                "HiddenServiceDir") > 1) {
-      log_warn(LD_CONFIG, "Using accounting with multiple hidden services is "
-               "risky: they will all turn off at the same time, which may "
-               "alert observers that they are being run by the same party.");
-    }
-  }
-
-  options->AccountingRule = ACCT_MAX;
-  if (options->AccountingRule_option) {
-    if (!strcmp(options->AccountingRule_option, "sum"))
-      options->AccountingRule = ACCT_SUM;
-    else if (!strcmp(options->AccountingRule_option, "max"))
-      options->AccountingRule = ACCT_MAX;
-    else if (!strcmp(options->AccountingRule_option, "in"))
-      options->AccountingRule = ACCT_IN;
-    else if (!strcmp(options->AccountingRule_option, "out"))
-      options->AccountingRule = ACCT_OUT;
-    else
-      REJECT("AccountingRule must be 'sum', 'max', 'in', or 'out'");
-  }
-
-  if (options->DirPort_set && !options->DirCache) {
-    REJECT("DirPort configured but DirCache disabled. DirPort requires "
-           "DirCache.");
-  }
-
-  if (options->BridgeRelay && !options->DirCache) {
-    REJECT("We're a bridge but DirCache is disabled. BridgeRelay requires "
-           "DirCache.");
-  }
+  if (options_validate_relay_accounting(old_options, options, msg) < 0)
+    return -1;
 
-  if (server_mode(options)) {
-    char *dircache_msg = NULL;
-    if (have_enough_mem_for_dircache(options, 0, &dircache_msg)) {
-      log_warn(LD_CONFIG, "%s", dircache_msg);
-      tor_free(dircache_msg);
-    }
-  }
+  if (options_validate_relay_mode(old_options, options, msg) < 0)
+    return -1;
 
   if (options->HTTPProxy) { /* parse it now */
     if (tor_addr_port_lookup(options->HTTPProxy,
@@ -4136,19 +3638,28 @@ options_validate(or_options_t *old_options, or_options_t *options,
     }
   }
 
+  if (options->TCPProxy) {
+    int res = parse_tcp_proxy_line(options->TCPProxy, options, msg);
+    if (res < 0) {
+      return res;
+    }
+  }
+
   /* Check if more than one exclusive proxy type has been enabled. */
   if (!!options->Socks4Proxy + !!options->Socks5Proxy +
-      !!options->HTTPSProxy > 1)
+      !!options->HTTPSProxy + !!options->TCPProxy > 1)
     REJECT("You have configured more than one proxy type. "
-           "(Socks4Proxy|Socks5Proxy|HTTPSProxy)");
+           "(Socks4Proxy|Socks5Proxy|HTTPSProxy|TCPProxy)");
 
   /* Check if the proxies will give surprising behavior. */
   if (options->HTTPProxy && !(options->Socks4Proxy ||
                               options->Socks5Proxy ||
-                              options->HTTPSProxy)) {
-    log_warn(LD_CONFIG, "HTTPProxy configured, but no SOCKS proxy or "
-             "HTTPS proxy configured. Watch out: this configuration will "
-             "proxy unencrypted directory connections only.");
+                              options->HTTPSProxy ||
+                              options->TCPProxy)) {
+    log_warn(LD_CONFIG, "HTTPProxy configured, but no SOCKS proxy, "
+             "HTTPS proxy, or any other TCP proxy configured. Watch out: "
+             "this configuration will proxy unencrypted directory "
+             "connections only.");
   }
 
   if (options->Socks5ProxyUsername) {
@@ -4216,15 +3727,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "have it group-readable.");
   }
 
-  if (options->MyFamily_lines && options->BridgeRelay) {
-    log_warn(LD_CONFIG, "Listing a family for a bridge relay is not "
-             "supported: it can reveal bridge fingerprints to censors. "
-             "You should also make sure you aren't listing this bridge's "
-             "fingerprint in any other MyFamily.");
-  }
-  if (normalize_nickname_list(&options->MyFamily,
-                              options->MyFamily_lines, "MyFamily", msg))
-    return -1;
   for (cl = options->NodeFamilies; cl; cl = cl->next) {
     routerset_t *rs = routerset_new();
     if (routerset_parse(rs, cl->value, cl->key)) {
@@ -4259,50 +3761,12 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   for (cl = options->ClientTransportPlugin; cl; cl = cl->next) {
-    if (parse_transport_line(options, cl->value, 1, 0) < 0)
+    if (pt_parse_transport_line(options, cl->value, 1, 0) < 0)
       REJECT("Invalid client transport line. See logs for details.");
   }
 
-  for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
-    if (parse_transport_line(options, cl->value, 1, 1) < 0)
-      REJECT("Invalid server transport line. See logs for details.");
-  }
-
-  if (options->ServerTransportPlugin && !server_mode(options)) {
-    log_notice(LD_GENERAL, "Tor is not configured as a relay but you specified"
-               " a ServerTransportPlugin line (%s). The ServerTransportPlugin "
-               "line will be ignored.",
-               escaped(options->ServerTransportPlugin->value));
-  }
-
-  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
-    /** If get_bindaddr_from_transport_listen_line() fails with
-        'transport' being NULL, it means that something went wrong
-        while parsing the ServerTransportListenAddr line. */
-    char *bindaddr = get_bindaddr_from_transport_listen_line(cl->value, NULL);
-    if (!bindaddr)
-      REJECT("ServerTransportListenAddr did not parse. See logs for details.");
-    tor_free(bindaddr);
-  }
-
-  if (options->ServerTransportListenAddr && !options->ServerTransportPlugin) {
-    log_notice(LD_GENERAL, "You need at least a single managed-proxy to "
-               "specify a transport listen address. The "
-               "ServerTransportListenAddr line will be ignored.");
-  }
-
-  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
-    /** If get_options_from_transport_options_line() fails with
-        'transport' being NULL, it means that something went wrong
-        while parsing the ServerTransportOptions line. */
-    smartlist_t *options_sl =
-      get_options_from_transport_options_line(cl->value, NULL);
-    if (!options_sl)
-      REJECT("ServerTransportOptions did not parse. See logs for details.");
-
-    SMARTLIST_FOREACH(options_sl, char *, cp, tor_free(cp));
-    smartlist_free(options_sl);
-  }
+  if (options_validate_server_transport(old_options, options, msg) < 0)
+    return -1;
 
   if (options->ConstrainedSockets) {
     /* If the user wants to constrain socket buffer use, make sure the desired
@@ -4316,85 +3780,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
           MIN_CONSTRAINED_TCP_BUFFER, MAX_CONSTRAINED_TCP_BUFFER);
       return -1;
     }
-    if (options->DirPort_set) {
-      /* Providing cached directory entries while system TCP buffers are scarce
-       * will exacerbate the socket errors.  Suggest that this be disabled. */
-      COMPLAIN("You have requested constrained socket buffers while also "
-               "serving directory entries via DirPort.  It is strongly "
-               "suggested that you disable serving directory requests when "
-               "system TCP buffer resources are scarce.");
-    }
   }
 
-  if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
-      options->V3AuthVotingInterval/2) {
-    /*
-    This doesn't work, but it seems like it should:
-     what code is preventing the interval being less than twice the lead-up?
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
-          options->V3AuthVotingInterval) {
-        REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than "
-               "V3AuthVotingInterval");
-      } else {
-        COMPLAIN("V3AuthVoteDelay plus V3AuthDistDelay is more than half "
-                 "V3AuthVotingInterval. This may lead to "
-                 "consensus instability, particularly if clocks drift.");
-      }
-    } else {
-     */
-      REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
-             "V3AuthVotingInterval");
-    /*
-    }
-     */
-  }
-
-  if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) {
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) {
-        REJECT("V3AuthVoteDelay is way too low.");
-      } else {
-        COMPLAIN("V3AuthVoteDelay is very low. "
-                 "This may lead to failure to vote for a consensus.");
-      }
-    } else {
-      REJECT("V3AuthVoteDelay is way too low.");
-    }
-  }
-
-  if (options->V3AuthDistDelay < MIN_DIST_SECONDS) {
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) {
-        REJECT("V3AuthDistDelay is way too low.");
-      } else {
-        COMPLAIN("V3AuthDistDelay is very low. "
-                 "This may lead to missing votes in a consensus.");
-      }
-    } else {
-      REJECT("V3AuthDistDelay is way too low.");
-    }
-  }
-
-  if (options->V3AuthNIntervalsValid < 2)
-    REJECT("V3AuthNIntervalsValid must be at least 2.");
-
-  if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) {
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) {
-        REJECT("V3AuthVotingInterval is insanely low.");
-      } else {
-        COMPLAIN("V3AuthVotingInterval is very low. "
-                 "This may lead to failure to synchronise for a consensus.");
-      }
-    } else {
-      REJECT("V3AuthVotingInterval is insanely low.");
-    }
-  } else if (options->V3AuthVotingInterval > 24*60*60) {
-    REJECT("V3AuthVotingInterval is insanely high.");
-  } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) {
-    COMPLAIN("V3AuthVotingInterval does not divide evenly into 24 hours.");
-  }
+  if (options_validate_dirauth_schedule(old_options, options, msg) < 0)
+    return -1;
 
   if (hs_config_service_all(options, 1) < 0)
     REJECT("Failed to configure rendezvous options. See logs for details.");
@@ -4422,88 +3811,51 @@ options_validate(or_options_t *old_options, or_options_t *options,
 
 #define CHECK_DEFAULT(arg)                                              \
   STMT_BEGIN                                                            \
-    if (!options->TestingTorNetwork &&                                  \
-        !options->UsingTestNetworkDefaults_ &&                          \
-        !config_is_same(&options_format,options,                        \
-                        default_options,#arg)) {                        \
+    if (!config_is_same(get_options_mgr(),options,                      \
+                        dflt_options,#arg)) {                           \
+      or_options_free(dflt_options);                                    \
       REJECT(#arg " may only be changed in testing Tor "                \
              "networks!");                                              \
-    } STMT_END
-  CHECK_DEFAULT(TestingV3AuthInitialVotingInterval);
-  CHECK_DEFAULT(TestingV3AuthInitialVoteDelay);
-  CHECK_DEFAULT(TestingV3AuthInitialDistDelay);
-  CHECK_DEFAULT(TestingV3AuthVotingStartOffset);
-  CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability);
-  CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime);
-  CHECK_DEFAULT(TestingServerDownloadInitialDelay);
-  CHECK_DEFAULT(TestingClientDownloadInitialDelay);
-  CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay);
-  CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay);
-  CHECK_DEFAULT(TestingBridgeDownloadInitialDelay);
-  CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay);
-  CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest);
-  CHECK_DEFAULT(TestingDirConnectionMaxStall);
-  CHECK_DEFAULT(TestingAuthKeyLifetime);
-  CHECK_DEFAULT(TestingLinkCertLifetime);
-  CHECK_DEFAULT(TestingSigningKeySlop);
-  CHECK_DEFAULT(TestingAuthKeySlop);
-  CHECK_DEFAULT(TestingLinkKeySlop);
+    }                                                                   \
+  STMT_END
+
+  /* Check for options that can only be changed from the defaults in testing
+     networks.  */
+  if (! options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) {
+    or_options_t *dflt_options = options_new();
+    options_init(dflt_options);
+    /* 31851: some of these options are dirauth or relay only */
+    CHECK_DEFAULT(TestingV3AuthInitialVotingInterval);
+    CHECK_DEFAULT(TestingV3AuthInitialVoteDelay);
+    CHECK_DEFAULT(TestingV3AuthInitialDistDelay);
+    CHECK_DEFAULT(TestingV3AuthVotingStartOffset);
+    CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability);
+    CHECK_DEFAULT(TestingServerDownloadInitialDelay);
+    CHECK_DEFAULT(TestingClientDownloadInitialDelay);
+    CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay);
+    CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay);
+    CHECK_DEFAULT(TestingBridgeDownloadInitialDelay);
+    CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay);
+    CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest);
+    CHECK_DEFAULT(TestingDirConnectionMaxStall);
+    CHECK_DEFAULT(TestingAuthKeyLifetime);
+    CHECK_DEFAULT(TestingLinkCertLifetime);
+    CHECK_DEFAULT(TestingSigningKeySlop);
+    CHECK_DEFAULT(TestingAuthKeySlop);
+    CHECK_DEFAULT(TestingLinkKeySlop);
+    or_options_free(dflt_options);
+  }
 #undef CHECK_DEFAULT
 
   if (!options->ClientDNSRejectInternalAddresses &&
       !(options->DirAuthorities ||
         (options->AlternateDirAuthority && options->AlternateBridgeAuthority)))
     REJECT("ClientDNSRejectInternalAddresses used for default network.");
-  if (options->SigningKeyLifetime < options->TestingSigningKeySlop*2)
-    REJECT("SigningKeyLifetime is too short.");
-  if (options->TestingLinkCertLifetime < options->TestingAuthKeySlop*2)
-    REJECT("LinkCertLifetime is too short.");
-  if (options->TestingAuthKeyLifetime < options->TestingLinkKeySlop*2)
-    REJECT("TestingAuthKeyLifetime is too short.");
-
-  if (options->TestingV3AuthInitialVotingInterval
-      < MIN_VOTE_INTERVAL_TESTING_INITIAL) {
-    REJECT("TestingV3AuthInitialVotingInterval is insanely low.");
-  } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) {
-    REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into "
-           "30 minutes.");
-  }
-
-  if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) {
-    REJECT("TestingV3AuthInitialVoteDelay is way too low.");
-  }
-
-  if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) {
-    REJECT("TestingV3AuthInitialDistDelay is way too low.");
-  }
 
-  if (options->TestingV3AuthInitialVoteDelay +
-      options->TestingV3AuthInitialDistDelay >=
-      options->TestingV3AuthInitialVotingInterval) {
-    REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay "
-           "must be less than TestingV3AuthInitialVotingInterval");
-  }
-
-  if (options->TestingV3AuthVotingStartOffset >
-      MIN(options->TestingV3AuthInitialVotingInterval,
-          options->V3AuthVotingInterval)) {
-    REJECT("TestingV3AuthVotingStartOffset is higher than the voting "
-           "interval.");
-  } else if (options->TestingV3AuthVotingStartOffset < 0) {
-    REJECT("TestingV3AuthVotingStartOffset must be non-negative.");
-  }
-
-  if (options->TestingAuthDirTimeToLearnReachability < 0) {
-    REJECT("TestingAuthDirTimeToLearnReachability must be non-negative.");
-  } else if (options->TestingAuthDirTimeToLearnReachability > 2*60*60) {
-    COMPLAIN("TestingAuthDirTimeToLearnReachability is insanely high.");
-  }
-
-  if (options->TestingEstimatedDescriptorPropagationTime < 0) {
-    REJECT("TestingEstimatedDescriptorPropagationTime must be non-negative.");
-  } else if (options->TestingEstimatedDescriptorPropagationTime > 60*60) {
-    COMPLAIN("TestingEstimatedDescriptorPropagationTime is insanely high.");
-  }
+  if (options_validate_relay_testing(old_options, options, msg) < 0)
+    return -1;
+  if (options_validate_dirauth_testing(old_options, options, msg) < 0)
+    return -1;
 
   if (options->TestingClientMaxIntervalWithoutRequest < 1) {
     REJECT("TestingClientMaxIntervalWithoutRequest is way too low.");
@@ -4545,27 +3897,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
                         "testing Tor network!");
   }
 
-  if (options->AccelName && !options->HardwareAccel)
-    options->HardwareAccel = 1;
-  if (options->AccelDir && !options->AccelName)
-    REJECT("Can't use hardware crypto accelerator dir without engine name.");
-
-  if (options->PublishServerDescriptor)
-    SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, {
-      if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0"))
-        if (smartlist_len(options->PublishServerDescriptor) > 1) {
-          COMPLAIN("You have passed a list of multiple arguments to the "
-                   "PublishServerDescriptor option that includes 0 or 1. "
-                   "0 or 1 should only be used as the sole argument. "
-                   "This configuration will be rejected in a future release.");
-          break;
-        }
-    });
-
-  if (options->BridgeRelay == 1 && ! options->ORPort_set)
-      REJECT("BridgeRelay is 1, ORPort is not set. This is an invalid "
-             "combination.");
-
   if (options_validate_scheduler(options, msg) < 0) {
     return -1;
   }
@@ -4580,8 +3911,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
  * actual maximum value.  We clip this value if it's too low, and autodetect
  * it if it's set to 0. */
 STATIC uint64_t
-compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
+compute_real_max_mem_in_queues(const uint64_t val, bool is_server)
 {
+#define MIN_SERVER_MB 64
+#define MIN_UNWARNED_SERVER_MB 256
+#define MIN_UNWARNED_CLIENT_MB 64
   uint64_t result;
 
   if (val == 0) {
@@ -4610,7 +3944,7 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 #else
 /* On a 32-bit platform, we can't have 8GB of ram. */
 #define RAM_IS_VERY_LARGE(x) (0)
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
       if (RAM_IS_VERY_LARGE(ram)) {
         /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues
@@ -4639,7 +3973,7 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
         result = avail;
       }
     }
-    if (log_guess && ! notice_sent) {
+    if (is_server && ! notice_sent) {
       log_notice(LD_CONFIG, "%sMaxMemInQueues is set to %"PRIu64" MB. "
                  "You can override this by setting MaxMemInQueues by hand.",
                  ram ? "Based on detected system memory, " : "",
@@ -4647,60 +3981,30 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
       notice_sent = 1;
     }
     return result;
-  } else if (val < ONE_GIGABYTE / 4) {
-    log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. "
-             "Ideally, have it as large as you can afford.");
-    return ONE_GIGABYTE / 4;
+  } else if (is_server && val < ONE_MEGABYTE * MIN_SERVER_MB) {
+    /* We can't configure less than this much on a server.  */
+    log_warn(LD_CONFIG, "MaxMemInQueues must be at least %d MB on servers "
+             "for now. Ideally, have it as large as you can afford.",
+             MIN_SERVER_MB);
+    return MIN_SERVER_MB * ONE_MEGABYTE;
+  } else if (is_server && val < ONE_MEGABYTE * MIN_UNWARNED_SERVER_MB) {
+    /* On a server, if it's less than this much, we warn that things
+     * may go badly. */
+    log_warn(LD_CONFIG, "MaxMemInQueues is set to a low value; if your "
+             "relay doesn't work, this may be the reason why.");
+    return val;
+  } else if (! is_server && val < ONE_MEGABYTE * MIN_UNWARNED_CLIENT_MB) {
+    /* On a client, if it's less than this much, we warn that things
+     * may go badly. */
+    log_warn(LD_CONFIG, "MaxMemInQueues is set to a low value; if your "
+             "client doesn't work, this may be the reason why.");
+    return val;
   } else {
     /* The value was fine all along */
     return val;
   }
 }
 
-/* If we have less than 300 MB suggest disabling dircache */
-#define DIRCACHE_MIN_MEM_MB 300
-#define DIRCACHE_MIN_MEM_BYTES (DIRCACHE_MIN_MEM_MB*ONE_MEGABYTE)
-#define STRINGIFY(val) #val
-
-/** Create a warning message for emitting if we are a dircache but may not have
- * enough system memory, or if we are not a dircache but probably should be.
- * Return -1 when a message is returned in *msg*, else return 0. */
-STATIC int
-have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
-                             char **msg)
-{
-  *msg = NULL;
-  /* XXX We should possibly be looking at MaxMemInQueues here
-   * unconditionally.  Or we should believe total_mem unconditionally. */
-  if (total_mem == 0) {
-    if (get_total_system_memory(&total_mem) < 0) {
-      total_mem = options->MaxMemInQueues >= SIZE_MAX ?
-        SIZE_MAX : (size_t)options->MaxMemInQueues;
-    }
-  }
-  if (options->DirCache) {
-    if (total_mem < DIRCACHE_MIN_MEM_BYTES) {
-      if (options->BridgeRelay) {
-        tor_asprintf(msg, "Running a Bridge with less than %d MB of memory "
-                       "is not recommended.", DIRCACHE_MIN_MEM_MB);
-      } else {
-        tor_asprintf(msg, "Being a directory cache (default) with less than "
-                       "%d MB of memory is not recommended and may consume "
-                       "most of the available resources. Consider disabling "
-                       "this functionality by setting the DirCache option "
-                       "to 0.", DIRCACHE_MIN_MEM_MB);
-      }
-    }
-  } else {
-    if (total_mem >= DIRCACHE_MIN_MEM_BYTES) {
-      *msg = tor_strdup("DirCache is disabled and we are configured as a "
-               "relay. We will not become a Guard.");
-    }
-  }
-  return *msg == NULL ? 0 : -1;
-}
-#undef STRINGIFY
-
 /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL
  * equal strings. */
 static int
@@ -4709,13 +4013,19 @@ opt_streq(const char *s1, const char *s2)
   return 0 == strcmp_opt(s1, s2);
 }
 
-/** Check if any of the previous options have changed but aren't allowed to. */
+/** Check if any config options have changed but aren't allowed to. */
 static int
-options_transition_allowed(const or_options_t *old,
-                           const or_options_t *new_val,
-                           char **msg)
+options_check_transition_cb(const void *old_,
+                            const void *new_val_,
+                            char **msg)
 {
-  if (!old)
+  CHECK_OPTIONS_MAGIC(old_);
+  CHECK_OPTIONS_MAGIC(new_val_);
+
+  const or_options_t *old = old_;
+  const or_options_t *new_val = new_val_;
+
+  if (BUG(!old))
     return 0;
 
 #define BAD_CHANGE_TO(opt, how) do {                                    \
@@ -4724,36 +4034,6 @@ options_transition_allowed(const or_options_t *old,
     return -1;                                                          \
   } while (0)
 
-#define NO_CHANGE_BOOL(opt) \
-  if (! CFG_EQ_BOOL(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
-#define NO_CHANGE_INT(opt) \
-  if (! CFG_EQ_INT(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
-#define NO_CHANGE_STRING(opt) \
-  if (! CFG_EQ_STRING(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
-
-  NO_CHANGE_STRING(PidFile);
-  NO_CHANGE_BOOL(RunAsDaemon);
-  NO_CHANGE_BOOL(Sandbox);
-  NO_CHANGE_STRING(DataDirectory);
-  NO_CHANGE_STRING(KeyDirectory);
-  NO_CHANGE_STRING(CacheDirectory);
-  NO_CHANGE_STRING(User);
-  NO_CHANGE_BOOL(KeepBindCapabilities);
-  NO_CHANGE_STRING(SyslogIdentityTag);
-  NO_CHANGE_STRING(AndroidIdentityTag);
-  NO_CHANGE_BOOL(HardwareAccel);
-  NO_CHANGE_STRING(AccelName);
-  NO_CHANGE_STRING(AccelDir);
-  NO_CHANGE_BOOL(TestingTorNetwork);
-  NO_CHANGE_BOOL(DisableAllSwap);
-  NO_CHANGE_INT(TokenBucketRefillInterval);
-  NO_CHANGE_BOOL(HiddenServiceSingleHopMode);
-  NO_CHANGE_BOOL(HiddenServiceNonAnonymousMode);
-  NO_CHANGE_BOOL(DisableDebuggerAttachment);
-  NO_CHANGE_BOOL(NoExec);
-  NO_CHANGE_INT(OwningControllerFD);
-  NO_CHANGE_BOOL(DisableSignalHandlers);
-
   if (sandbox_is_active()) {
 #define SB_NOCHANGE_STR(opt)                      \
     if (! CFG_EQ_STRING(old, new_val, opt))       \
@@ -4790,71 +4070,6 @@ options_transition_allowed(const or_options_t *old,
   return 0;
 }
 
-/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
- * will require us to rotate the CPU and DNS workers; else return 0. */
-static int
-options_transition_affects_workers(const or_options_t *old_options,
-                                   const or_options_t *new_options)
-{
-  YES_IF_CHANGED_STRING(DataDirectory);
-  YES_IF_CHANGED_INT(NumCPUs);
-  YES_IF_CHANGED_LINELIST(ORPort_lines);
-  YES_IF_CHANGED_BOOL(ServerDNSSearchDomains);
-  YES_IF_CHANGED_BOOL(SafeLogging_);
-  YES_IF_CHANGED_BOOL(ClientOnly);
-  YES_IF_CHANGED_BOOL(LogMessageDomains);
-  YES_IF_CHANGED_LINELIST(Logs);
-
-  if (server_mode(old_options) != server_mode(new_options) ||
-      public_server_mode(old_options) != public_server_mode(new_options) ||
-      dir_server_mode(old_options) != dir_server_mode(new_options))
-    return 1;
-
-  /* Nothing that changed matters. */
-  return 0;
-}
-
-/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
- * will require us to generate a new descriptor; else return 0. */
-static int
-options_transition_affects_descriptor(const or_options_t *old_options,
-                                      const or_options_t *new_options)
-{
-  /* XXX We can be smarter here. If your DirPort isn't being
-   * published and you just turned it off, no need to republish. Etc. */
-
-  YES_IF_CHANGED_STRING(DataDirectory);
-  YES_IF_CHANGED_STRING(Nickname);
-  YES_IF_CHANGED_STRING(Address);
-  YES_IF_CHANGED_LINELIST(ExitPolicy);
-  YES_IF_CHANGED_BOOL(ExitRelay);
-  YES_IF_CHANGED_BOOL(ExitPolicyRejectPrivate);
-  YES_IF_CHANGED_BOOL(ExitPolicyRejectLocalInterfaces);
-  YES_IF_CHANGED_BOOL(IPv6Exit);
-  YES_IF_CHANGED_LINELIST(ORPort_lines);
-  YES_IF_CHANGED_LINELIST(DirPort_lines);
-  YES_IF_CHANGED_LINELIST(DirPort_lines);
-  YES_IF_CHANGED_BOOL(ClientOnly);
-  YES_IF_CHANGED_BOOL(DisableNetwork);
-  YES_IF_CHANGED_BOOL(PublishServerDescriptor_);
-  YES_IF_CHANGED_STRING(ContactInfo);
-  YES_IF_CHANGED_STRING(BridgeDistribution);
-  YES_IF_CHANGED_LINELIST(MyFamily);
-  YES_IF_CHANGED_STRING(AccountingStart);
-  YES_IF_CHANGED_INT(AccountingMax);
-  YES_IF_CHANGED_INT(AccountingRule);
-  YES_IF_CHANGED_BOOL(DirCache);
-  YES_IF_CHANGED_BOOL(AssumeReachable);
-
-  if (get_effective_bwrate(old_options) != get_effective_bwrate(new_options) ||
-      get_effective_bwburst(old_options) !=
-        get_effective_bwburst(new_options) ||
-      public_server_mode(old_options) != public_server_mode(new_options))
-    return 1;
-
-  return 0;
-}
-
 #ifdef _WIN32
 /** Return the directory on windows where we expect to find our application
  * data. */
@@ -4939,85 +4154,6 @@ get_default_conf_file(int defaults_file)
 #endif /* defined(DISABLE_SYSTEM_TORRC) || ... */
 }
 
-/** Verify whether lst is a list of strings containing valid-looking
- * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$'
- * to any nickname or fingerprint that needs it. Also splits comma-separated
- * list elements into multiple elements. Return 0 on success.
- * Warn and return -1 on failure.
- */
-static int
-normalize_nickname_list(config_line_t **normalized_out,
-                        const config_line_t *lst, const char *name,
-                        char **msg)
-{
-  if (!lst)
-    return 0;
-
-  config_line_t *new_nicknames = NULL;
-  config_line_t **new_nicknames_next = &new_nicknames;
-
-  const config_line_t *cl;
-  for (cl = lst; cl; cl = cl->next) {
-    const char *line = cl->value;
-    if (!line)
-      continue;
-
-    int valid_line = 1;
-    smartlist_t *sl = smartlist_new();
-    smartlist_split_string(sl, line, ",",
-      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
-    SMARTLIST_FOREACH_BEGIN(sl, char *, s)
-    {
-      char *normalized = NULL;
-      if (!is_legal_nickname_or_hexdigest(s)) {
-        // check if first char is dollar
-        if (s[0] != '$') {
-          // Try again but with a dollar symbol prepended
-          char *prepended;
-          tor_asprintf(&prepended, "$%s", s);
-
-          if (is_legal_nickname_or_hexdigest(prepended)) {
-            // The nickname is valid when it's prepended, set it as the
-            // normalized version
-            normalized = prepended;
-          } else {
-            // Still not valid, free and fallback to error message
-            tor_free(prepended);
-          }
-        }
-
-        if (!normalized) {
-          tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
-          valid_line = 0;
-          break;
-        }
-      } else {
-        normalized = tor_strdup(s);
-      }
-
-      config_line_t *next = tor_malloc_zero(sizeof(*next));
-      next->key = tor_strdup(cl->key);
-      next->value = normalized;
-      next->next = NULL;
-
-      *new_nicknames_next = next;
-      new_nicknames_next = &next->next;
-    } SMARTLIST_FOREACH_END(s);
-
-    SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
-    smartlist_free(sl);
-
-    if (!valid_line) {
-      config_free_lines(new_nicknames);
-      return -1;
-    }
-  }
-
-  *normalized_out = new_nicknames;
-
-  return 0;
-}
-
 /** Learn config file name from command line arguments, or use the default.
  *
  * If <b>defaults_file</b> is true, we're looking for torrc-defaults;
@@ -5030,12 +4166,12 @@ normalize_nickname_list(config_line_t **normalized_out,
  * filename if it doesn't exist.
  */
 static char *
-find_torrc_filename(config_line_t *cmd_arg,
+find_torrc_filename(const config_line_t *cmd_arg,
                     int defaults_file,
                     int *using_default_fname, int *ignore_missing_torrc)
 {
   char *fname=NULL;
-  config_line_t *p_index;
+  const config_line_t *p_index;
   const char *fname_opt = defaults_file ? "--defaults-torrc" : "-f";
   const char *ignore_opt = defaults_file ? NULL : "--ignore-missing-torrc";
 
@@ -5087,7 +4223,7 @@ find_torrc_filename(config_line_t *cmd_arg,
       } else {
         fname = dflt ? tor_strdup(dflt) : NULL;
       }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
       fname = dflt ? tor_strdup(dflt) : NULL;
 #endif /* !defined(_WIN32) */
     }
@@ -5114,7 +4250,7 @@ load_torrc_from_stdin(void)
  * Return the contents of the file on success, and NULL on failure.
  */
 static char *
-load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file)
+load_torrc_from_disk(const config_line_t *cmd_arg, int defaults_file)
 {
   char *fname=NULL;
   char *cf = NULL;
@@ -5169,24 +4305,20 @@ int
 options_init_from_torrc(int argc, char **argv)
 {
   char *cf=NULL, *cf_defaults=NULL;
-  int command;
   int retval = -1;
-  char *command_arg = NULL;
   char *errmsg=NULL;
-  config_line_t *p_index = NULL;
-  config_line_t *cmdline_only_options = NULL;
+  const config_line_t *cmdline_only_options;
 
   /* Go through command-line variables */
-  if (! have_parsed_cmdline) {
+  if (global_cmdline == NULL) {
     /* Or we could redo the list every time we pass this place.
      * It does not really matter */
-    if (config_parse_commandline(argc, argv, 0, &global_cmdline_options,
-                                 &global_cmdline_only_options) < 0) {
+    global_cmdline = config_parse_commandline(argc, argv, 0);
+    if (global_cmdline == NULL) {
       goto err;
     }
-    have_parsed_cmdline = 1;
   }
-  cmdline_only_options = global_cmdline_only_options;
+  cmdline_only_options = global_cmdline->cmdline_opts;
 
   if (config_line_find(cmdline_only_options, "-h") ||
       config_line_find(cmdline_only_options, "--help")) {
@@ -5203,6 +4335,10 @@ options_init_from_torrc(int argc, char **argv)
     list_deprecated_options();
     return 1;
   }
+  if (config_line_find(cmdline_only_options, "--dbg-dump-subsystem-list")) {
+    subsystems_dump_list();
+    return 1;
+  }
 
   if (config_line_find(cmdline_only_options, "--version")) {
     printf("Tor version %s.\n",get_version());
@@ -5249,25 +4385,10 @@ options_init_from_torrc(int argc, char **argv)
     return 1;
   }
 
-  command = CMD_RUN_TOR;
-  for (p_index = cmdline_only_options; p_index; p_index = p_index->next) {
-    if (!strcmp(p_index->key,"--keygen")) {
-      command = CMD_KEYGEN;
-    } else if (!strcmp(p_index->key, "--key-expiration")) {
-      command = CMD_KEY_EXPIRATION;
-      command_arg = p_index->value;
-    } else if (!strcmp(p_index->key,"--list-fingerprint")) {
-      command = CMD_LIST_FINGERPRINT;
-    } else if (!strcmp(p_index->key, "--hash-password")) {
-      command = CMD_HASH_PASSWORD;
-      command_arg = p_index->value;
-    } else if (!strcmp(p_index->key, "--dump-config")) {
-      command = CMD_DUMP_CONFIG;
-      command_arg = p_index->value;
-    } else if (!strcmp(p_index->key, "--verify-config")) {
-      command = CMD_VERIFY_CONFIG;
-    }
-  }
+  int command = global_cmdline->command;
+  const char *command_arg = global_cmdline->command_arg;
+  /* "immediate" has already been handled by this point. */
+  tor_assert(command != CMD_IMMEDIATE);
 
   if (command == CMD_HASH_PASSWORD) {
     cf_defaults = tor_strdup("");
@@ -5389,6 +4510,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
                          int command, const char *command_arg,
                          char **msg)
 {
+  bool retry = false;
   or_options_t *oldoptions, *newoptions, *newdefaultoptions=NULL;
   config_line_t *cl;
   int retval;
@@ -5399,8 +4521,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   oldoptions = global_options; /* get_options unfortunately asserts if
                                   this is the first time we run*/
 
-  newoptions = tor_malloc_zero(sizeof(or_options_t));
-  newoptions->magic_ = OR_OPTIONS_MAGIC;
+  newoptions = options_new();
   options_init(newoptions);
   newoptions->command = command;
   newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
@@ -5419,7 +4540,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       err = SETOPT_ERR_PARSE;
       goto err;
     }
-    retval = config_assign(&options_format, newoptions, cl,
+    retval = config_assign(get_options_mgr(), newoptions, cl,
                            CAL_WARN_DEPRECATIONS, msg);
     config_free_lines(cl);
     if (retval < 0) {
@@ -5427,16 +4548,23 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       goto err;
     }
     if (i==0)
-      newdefaultoptions = config_dup(&options_format, newoptions);
+      newdefaultoptions = config_dup(get_options_mgr(), newoptions);
   }
 
   if (newdefaultoptions == NULL) {
-    newdefaultoptions = config_dup(&options_format, global_default_options);
+    newdefaultoptions = config_dup(get_options_mgr(), global_default_options);
   }
 
   /* Go through command-line variables too */
-  retval = config_assign(&options_format, newoptions,
-                         global_cmdline_options, CAL_WARN_DEPRECATIONS, msg);
+  {
+    config_line_t *other_opts = NULL;
+    if (global_cmdline) {
+      other_opts = global_cmdline->other_opts;
+    }
+    retval = config_assign(get_options_mgr(), newoptions,
+                           other_opts,
+                           CAL_WARN_DEPRECATIONS, msg);
+  }
   if (retval < 0) {
     err = SETOPT_ERR_PARSE;
     goto err;
@@ -5444,98 +4572,22 @@ options_init_from_string(const char *cf_defaults, const char *cf,
 
   newoptions->IncludeUsed = cf_has_include;
   newoptions->FilesOpenedByIncludes = opened_files;
+  opened_files = NULL; // prevent double-free.
 
   /* If this is a testing network configuration, change defaults
-   * for a list of dependent config options, re-initialize newoptions
-   * with the new defaults, and assign all options to it second time. */
-  if (newoptions->TestingTorNetwork) {
-    /* XXXX this is a bit of a kludge.  perhaps there's a better way to do
-     * this?  We could, for example, make the parsing algorithm do two passes
-     * over the configuration.  If it finds any "suite" options like
-     * TestingTorNetwork, it could change the defaults before its second pass.
-     * Not urgent so long as this seems to work, but at any sign of trouble,
-     * let's clean it up.  -NM */
-
-    /* Change defaults. */
-    for (int i = 0; testing_tor_network_defaults[i].name; ++i) {
-      const config_var_t *new_var = &testing_tor_network_defaults[i];
-      config_var_t *old_var =
-          config_find_option_mutable(&options_format, new_var->name);
-      tor_assert(new_var);
-      tor_assert(old_var);
-      old_var->initvalue = new_var->initvalue;
-
-      if ((config_find_deprecation(&options_format, new_var->name))) {
-        log_warn(LD_GENERAL, "Testing options override the deprecated "
-                 "option %s. Is that intentional?",
-                 new_var->name);
-      }
-    }
-
-    /* Clear newoptions and re-initialize them with new defaults. */
-    or_options_free(newoptions);
-    or_options_free(newdefaultoptions);
-    newdefaultoptions = NULL;
-    newoptions = tor_malloc_zero(sizeof(or_options_t));
-    newoptions->magic_ = OR_OPTIONS_MAGIC;
-    options_init(newoptions);
-    newoptions->command = command;
-    newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
-
-    /* Assign all options a second time. */
-    opened_files = smartlist_new();
-    for (int i = 0; i < 2; ++i) {
-      const char *body = i==0 ? cf_defaults : cf;
-      if (!body)
-        continue;
-
-      /* get config lines, assign them */
-      retval = config_get_lines_include(body, &cl, 1,
-                                        body == cf ? &cf_has_include : NULL,
-                                        opened_files);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      retval = config_assign(&options_format, newoptions, cl, 0, msg);
-      config_free_lines(cl);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      if (i==0)
-        newdefaultoptions = config_dup(&options_format, newoptions);
-    }
-    /* Assign command-line variables a second time too */
-    retval = config_assign(&options_format, newoptions,
-                           global_cmdline_options, 0, msg);
-    if (retval < 0) {
-      err = SETOPT_ERR_PARSE;
-      goto err;
-    }
-  }
-
-  newoptions->IncludeUsed = cf_has_include;
-  in_option_validation = 1;
-  newoptions->FilesOpenedByIncludes = opened_files;
-
-  /* Validate newoptions */
-  if (options_validate(oldoptions, newoptions, newdefaultoptions,
-                       0, msg) < 0) {
-    err = SETOPT_ERR_PARSE; /*XXX make this a separate return value.*/
+   * for a list of dependent config options, and try this function again. */
+  if (newoptions->TestingTorNetwork && ! testing_network_configured) {
+    // retry with the testing defaults.
+    testing_network_configured = true;
+    retry = true;
     goto err;
   }
 
-  if (options_transition_allowed(oldoptions, newoptions, msg) < 0) {
-    err = SETOPT_ERR_TRANSITION;
+  err = options_validate_and_set(oldoptions, newoptions, msg);
+  if (err < 0) {
+    newoptions = NULL; // This was already freed in options_validate_and_set.
     goto err;
   }
-  in_option_validation = 0;
-
-  if (set_options(newoptions, msg)) {
-    err = SETOPT_ERR_SETTING;
-    goto err; /* frees and replaces old options */
-  }
 
   or_options_free(global_default_options);
   global_default_options = newdefaultoptions;
@@ -5548,15 +4600,16 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
     smartlist_free(opened_files);
   }
-  // may have been set to opened_files, avoid double free
-  newoptions->FilesOpenedByIncludes = NULL;
-  or_options_free(newoptions);
   or_options_free(newdefaultoptions);
+  or_options_free(newoptions);
   if (*msg) {
     char *old_msg = *msg;
     tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg);
     tor_free(old_msg);
   }
+  if (retry)
+    return options_init_from_string(cf_defaults, cf, command, command_arg,
+                                    msg);
   return err;
 }
 
@@ -5681,22 +4734,14 @@ open_and_add_file_log(const log_severity_list_t *severity,
 }
 
 /**
- * Initialize the logs based on the configuration file.
- */
+ * Try to set our global log granularity from `options->LogGranularity`,
+ * adjusting it as needed so that we are an even divisor of a second, or an
+ * even multiple of seconds. Return 0 on success, -1 on failure.
+ **/
 static int
-options_init_logs(const or_options_t *old_options, or_options_t *options,
-                  int validate_only)
+options_init_log_granularity(const or_options_t *options,
+                             int validate_only)
 {
-  config_line_t *opt;
-  int ok;
-  smartlist_t *elts;
-  int run_as_daemon =
-#ifdef _WIN32
-               0;
-#else
-               options->RunAsDaemon;
-#endif
-
   if (options->LogTimeGranularity <= 0) {
     log_warn(LD_CONFIG, "Log time granularity '%d' has to be positive.",
              options->LogTimeGranularity);
@@ -5726,9 +4771,38 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
       set_log_time_granularity(options->LogTimeGranularity);
   }
 
+  return 0;
+}
+
+/**
+ * Initialize the logs based on the configuration file.
+ */
+STATIC int
+options_init_logs(const or_options_t *old_options, const or_options_t *options,
+                  int validate_only)
+{
+  config_line_t *opt;
+  int ok;
+  smartlist_t *elts;
+  int run_as_daemon =
+#ifdef _WIN32
+               0;
+#else
+               options->RunAsDaemon;
+#endif
+
+  if (options_init_log_granularity(options, validate_only) < 0)
+    return -1;
+
   ok = 1;
   elts = smartlist_new();
 
+  if (options->Logs == NULL && !run_as_daemon && !validate_only) {
+    /* When no logs are given, the default behavior is to log nothing (if
+       RunAsDaemon is set) or to log based on the quiet level otherwise. */
+    add_default_log_for_quiet_level(quiet_level);
+  }
+
   for (opt = options->Logs; opt; opt = opt->next) {
     log_severity_list_t *severity;
     const char *cfg = opt->value;
@@ -5782,7 +4856,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
 #else
         log_warn(LD_CONFIG, "Android logging is not supported"
                             " on this system. Sorry.");
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
         goto cleanup;
       }
     }
@@ -6002,6 +5076,68 @@ parse_bridge_line(const char *line)
   return bridge_line;
 }
 
+/** Parse the contents of a TCPProxy line from <b>line</b> and put it
+ * in <b>options</b>. Return 0 if the line is well-formed, and -1 if it
+ * isn't.
+ *
+ * This will mutate only options->TCPProxyProtocol, options->TCPProxyAddr,
+ * and options->TCPProxyPort.
+ *
+ * On error, tor_strdup an error explanation into *<b>msg</b>.
+ */
+STATIC int
+parse_tcp_proxy_line(const char *line, or_options_t *options, char **msg)
+{
+  int ret = 0;
+  tor_assert(line);
+  tor_assert(options);
+  tor_assert(msg);
+
+  smartlist_t *sl = smartlist_new();
+  /* Split between the protocol and the address/port. */
+  smartlist_split_string(sl, line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
+
+  /* The address/port is not specified. */
+  if (smartlist_len(sl) < 2) {
+    *msg = tor_strdup("TCPProxy has no address/port. Please fix.");
+    goto err;
+  }
+
+  char *protocol_string = smartlist_get(sl, 0);
+  char *addrport_string = smartlist_get(sl, 1);
+
+  /* The only currently supported protocol is 'haproxy'. */
+  if (strcasecmp(protocol_string, "haproxy")) {
+    *msg = tor_strdup("TCPProxy protocol is not supported. Currently "
+                      "the only supported protocol is 'haproxy'. "
+                      "Please fix.");
+    goto err;
+  } else {
+    /* Otherwise, set the correct protocol. */
+    options->TCPProxyProtocol = TCP_PROXY_PROTOCOL_HAPROXY;
+  }
+
+  /* Parse the address/port. */
+  if (tor_addr_port_lookup(addrport_string, &options->TCPProxyAddr,
+                           &options->TCPProxyPort) < 0) {
+    *msg = tor_strdup("TCPProxy address/port failed to parse or resolve. "
+                      "Please fix.");
+    goto err;
+  }
+
+  /* Success. */
+  ret = 0;
+  goto end;
+
+ err:
+  ret = -1;
+ end:
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return ret;
+}
+
 /** Read the contents of a ClientTransportPlugin or ServerTransportPlugin
  * line from <b>line</b>, depending on the value of <b>server</b>. Return 0
  * if the line is well-formed, and -1 if it isn't.
@@ -6012,9 +5148,8 @@ parse_bridge_line(const char *line)
  * our internal transport list.
  * - If it's a managed proxy line, launch the managed proxy.
  */
-
-STATIC int
-parse_transport_line(const or_options_t *options,
+int
+pt_parse_transport_line(const or_options_t *options,
                      const char *line, int validate_only,
                      int server)
 {
@@ -6150,9 +5285,10 @@ parse_transport_line(const or_options_t *options,
 
     /* ClientTransportPlugins connecting through a proxy is managed only. */
     if (!server && (options->Socks4Proxy || options->Socks5Proxy ||
-                    options->HTTPSProxy)) {
+                    options->HTTPSProxy || options->TCPProxy)) {
       log_warn(LD_CONFIG, "You have configured an external proxy with another "
-                          "proxy type. (Socks4Proxy|Socks5Proxy|HTTPSProxy)");
+                          "proxy type. (Socks4Proxy|Socks5Proxy|HTTPSProxy|"
+                          "TCPProxy)");
       goto err;
     }
 
@@ -6207,157 +5343,6 @@ parse_transport_line(const or_options_t *options,
   return r;
 }
 
-/** Given a ServerTransportListenAddr <b>line</b>, return its
- *  <address:port> string. Return NULL if the line was not
- *  well-formed.
- *
- *  If <b>transport</b> is set, return NULL if the line is not
- *  referring to <b>transport</b>.
- *
- *  The returned string is allocated on the heap and it's the
- *  responsibility of the caller to free it. */
-static char *
-get_bindaddr_from_transport_listen_line(const char *line,const char *transport)
-{
-  smartlist_t *items = NULL;
-  const char *parsed_transport = NULL;
-  char *addrport = NULL;
-  tor_addr_t addr;
-  uint16_t port = 0;
-
-  items = smartlist_new();
-  smartlist_split_string(items, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-
-  if (smartlist_len(items) < 2) {
-    log_warn(LD_CONFIG,"Too few arguments on ServerTransportListenAddr line.");
-    goto err;
-  }
-
-  parsed_transport = smartlist_get(items, 0);
-  addrport = tor_strdup(smartlist_get(items, 1));
-
-  /* If 'transport' is given, check if it matches the one on the line */
-  if (transport && strcmp(transport, parsed_transport))
-    goto err;
-
-  /* Validate addrport */
-  if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port, -1)<0) {
-    log_warn(LD_CONFIG, "Error parsing ServerTransportListenAddr "
-             "address '%s'", addrport);
-    goto err;
-  }
-
-  goto done;
-
- err:
-  tor_free(addrport);
-  addrport = NULL;
-
- done:
-  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
-  smartlist_free(items);
-
-  return addrport;
-}
-
-/** Given a ServerTransportOptions <b>line</b>, return a smartlist
- *  with the options. Return NULL if the line was not well-formed.
- *
- *  If <b>transport</b> is set, return NULL if the line is not
- *  referring to <b>transport</b>.
- *
- *  The returned smartlist and its strings are allocated on the heap
- *  and it's the responsibility of the caller to free it. */
-smartlist_t *
-get_options_from_transport_options_line(const char *line,const char *transport)
-{
-  smartlist_t *items = smartlist_new();
-  smartlist_t *options = smartlist_new();
-  const char *parsed_transport = NULL;
-
-  smartlist_split_string(items, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-
-  if (smartlist_len(items) < 2) {
-    log_warn(LD_CONFIG,"Too few arguments on ServerTransportOptions line.");
-    goto err;
-  }
-
-  parsed_transport = smartlist_get(items, 0);
-  /* If 'transport' is given, check if it matches the one on the line */
-  if (transport && strcmp(transport, parsed_transport))
-    goto err;
-
-  SMARTLIST_FOREACH_BEGIN(items, const char *, option) {
-    if (option_sl_idx == 0) /* skip the transport field (first field)*/
-      continue;
-
-    /* validate that it's a k=v value */
-    if (!string_is_key_value(LOG_WARN, option)) {
-      log_warn(LD_CONFIG, "%s is not a k=v value.", escaped(option));
-      goto err;
-    }
-
-    /* add it to the options smartlist */
-    smartlist_add_strdup(options, option);
-    log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option));
-  } SMARTLIST_FOREACH_END(option);
-
-  goto done;
-
- err:
-  SMARTLIST_FOREACH(options, char*, s, tor_free(s));
-  smartlist_free(options);
-  options = NULL;
-
- done:
-  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
-  smartlist_free(items);
-
-  return options;
-}
-
-/** Given the name of a pluggable transport in <b>transport</b>, check
- *  the configuration file to see if the user has explicitly asked for
- *  it to listen on a specific port. Return a <address:port> string if
- *  so, otherwise NULL. */
-char *
-get_transport_bindaddr_from_config(const char *transport)
-{
-  config_line_t *cl;
-  const or_options_t *options = get_options();
-
-  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
-    char *bindaddr =
-      get_bindaddr_from_transport_listen_line(cl->value, transport);
-    if (bindaddr)
-      return bindaddr;
-  }
-
-  return NULL;
-}
-
-/** Given the name of a pluggable transport in <b>transport</b>, check
- *  the configuration file to see if the user has asked us to pass any
- *  parameters to the pluggable transport. Return a smartlist
- *  containing the parameters, otherwise NULL. */
-smartlist_t *
-get_options_for_server_transport(const char *transport)
-{
-  config_line_t *cl;
-  const or_options_t *options = get_options();
-
-  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
-    smartlist_t *options_sl =
-      get_options_from_transport_options_line(cl->value, transport);
-    if (options_sl)
-      return options_sl;
-  }
-
-  return NULL;
-}
-
 /** Read the contents of a DirAuthority line from <b>line</b>. If
  * <b>validate_only</b> is 0, and the line is well-formed, and it
  * shares any bits with <b>required_type</b> or <b>required_type</b>
@@ -6628,22 +5613,33 @@ parse_dir_fallback_line(const char *line,
   return r;
 }
 
-/** Allocate and return a new port_cfg_t with reasonable defaults. */
-STATIC port_cfg_t *
+/** Allocate and return a new port_cfg_t with reasonable defaults.
+ *
+ * <b>namelen</b> is the length of the unix socket name
+ * (typically the filesystem path), not including the trailing NUL.
+ * It should be 0 for ports that are not zunix sockets. */
+port_cfg_t *
 port_cfg_new(size_t namelen)
 {
   tor_assert(namelen <= SIZE_T_CEILING - sizeof(port_cfg_t) - 1);
   port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t) + namelen + 1);
+
+  /* entry_cfg flags */
   cfg->entry_cfg.ipv4_traffic = 1;
   cfg->entry_cfg.ipv6_traffic = 1;
+  cfg->entry_cfg.prefer_ipv6 = 0;
   cfg->entry_cfg.dns_request = 1;
   cfg->entry_cfg.onion_traffic = 1;
   cfg->entry_cfg.prefer_ipv6_virtaddr = 1;
+  cfg->entry_cfg.session_group = SESSION_GROUP_UNSET;
+  cfg->entry_cfg.isolation_flags = ISO_DEFAULT;
+
+  /* Other flags default to 0 due to tor_malloc_zero */
   return cfg;
 }
 
 /** Free all storage held in <b>port</b> */
-STATIC void
+void
 port_cfg_free_(port_cfg_t *port)
 {
   tor_free(port);
@@ -6677,27 +5673,6 @@ warn_nonlocal_client_ports(const smartlist_t *ports,
   } SMARTLIST_FOREACH_END(port);
 }
 
-/** Warn for every Extended ORPort port in <b>ports</b> that is on a
- *  publicly routable address. */
-static void
-warn_nonlocal_ext_orports(const smartlist_t *ports, const char *portname)
-{
-  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
-    if (port->type != CONN_TYPE_EXT_OR_LISTENER)
-      continue;
-    if (port->is_unix_addr)
-      continue;
-    /* XXX maybe warn even if address is RFC1918? */
-    if (!tor_addr_is_internal(&port->addr, 1)) {
-      log_warn(LD_CONFIG, "You specified a public address '%s' for %sPort. "
-               "This is not advised; this address is supposed to only be "
-               "exposed on localhost so that your pluggable transport "
-               "proxies can connect to it.",
-               fmt_addrport(&port->addr, port->port), portname);
-    }
-  } SMARTLIST_FOREACH_END(port);
-}
-
 /** Given a list of port_cfg_t in <b>ports</b>, warn if any controller port
  * there is listening on any non-loopback address.  If <b>forbid_nonlocal</b>
  * is true, then emit a stronger warning and remove the port from the list.
@@ -6768,7 +5743,7 @@ port_cfg_line_extract_addrport(const char *line,
     size_t sz;
     *is_unix_out = 1;
     *addrport_out = NULL;
-    line += strlen(unix_socket_prefix); /*No q: Keep the quote */
+    line += strlen(unix_socket_prefix); /* No 'unix:', but keep the quote */
     *rest_out = unescape_string(line, addrport_out, &sz);
     if (!*rest_out || (*addrport_out && sz != strlen(*addrport_out))) {
       tor_free(*addrport_out);
@@ -6811,55 +5786,6 @@ warn_client_dns_cache(const char *option, int disabling)
 }
 
 /**
- * Validate the configured bridge distribution method from a BridgeDistribution
- * config line.
- *
- * The input <b>bd</b>, is a string taken from the BridgeDistribution config
- * line (if present).  If the option wasn't set, return 0 immediately.  The
- * BridgeDistribution option is then validated.  Currently valid, recognised
- * options are:
- *
- * - "none"
- * - "any"
- * - "https"
- * - "email"
- * - "moat"
- * - "hyphae"
- *
- * If the option string is unrecognised, a warning will be logged and 0 is
- * returned.  If the option string contains an invalid character, -1 is
- * returned.
- **/
-STATIC int
-check_bridge_distribution_setting(const char *bd)
-{
-  if (bd == NULL)
-    return 0;
-
-  const char *RECOGNIZED[] = {
-    "none", "any", "https", "email", "moat", "hyphae"
-  };
-  unsigned i;
-  for (i = 0; i < ARRAY_LENGTH(RECOGNIZED); ++i) {
-    if (!strcasecmp(bd, RECOGNIZED[i]))
-      return 0;
-  }
-
-  const char *cp = bd;
-  //  Method = (KeywordChar | "_") +
-  while (TOR_ISALNUM(*cp) || *cp == '-' || *cp == '_')
-    ++cp;
-
-  if (*cp == 0) {
-    log_warn(LD_CONFIG, "Unrecognized BridgeDistribution value %s. I'll "
-           "assume you know what you are doing...", escaped(bd));
-    return 0; // we reached the end of the string; all is well
-  } else {
-    return -1; // we found a bad character in the string.
-  }
-}
-
-/**
  * Parse port configuration for a single port type.
  *
  * Read entries of the "FooPort" type from the list <b>ports</b>.  Syntax is
@@ -6889,8 +5815,8 @@ check_bridge_distribution_setting(const char *bd)
  * <b>out</b> for every port that the client should listen on.  Return 0
  * on success, -1 on failure.
  */
-STATIC int
-parse_port_config(smartlist_t *out,
+int
+port_parse_config(smartlist_t *out,
                   const config_line_t *ports,
                   const char *portname,
                   int listener_type,
@@ -6912,11 +5838,12 @@ parse_port_config(smartlist_t *out,
   const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET;
   int got_zero_port=0, got_nonzero_port=0;
   char *unix_socket_path = NULL;
+  port_cfg_t *cfg = NULL;
 
   /* If there's no FooPort, then maybe make a default one. */
   if (! ports) {
     if (defaultport && defaultaddr && out) {
-      port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0);
+       cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0);
        cfg->type = listener_type;
        if (is_unix_socket) {
          tor_addr_make_unspec(&cfg->addr);
@@ -6926,8 +5853,6 @@ parse_port_config(smartlist_t *out,
          cfg->port = defaultport;
          tor_addr_parse(&cfg->addr, defaultaddr);
        }
-       cfg->entry_cfg.session_group = SESSION_GROUP_UNSET;
-       cfg->entry_cfg.isolation_flags = ISO_DEFAULT;
        smartlist_add(out, cfg);
     }
     return 0;
@@ -6941,28 +5866,12 @@ parse_port_config(smartlist_t *out,
   for (; ports; ports = ports->next) {
     tor_addr_t addr;
     tor_addr_make_unspec(&addr);
-
-    int port;
-    int sessiongroup = SESSION_GROUP_UNSET;
-    unsigned isolation = ISO_DEFAULT;
-    int prefer_no_auth = 0;
-    int socks_iso_keep_alive = 0;
-
+    int port, ok,
+        has_used_unix_socket_only_option = 0,
+        is_unix_tagged_addr = 0;
     uint16_t ptmp=0;
-    int ok;
-    /* This must be kept in sync with port_cfg_new's defaults */
-    int no_listen = 0, no_advertise = 0, all_addrs = 0,
-      bind_ipv4_only = 0, bind_ipv6_only = 0,
-      ipv4_traffic = 1, ipv6_traffic = 1, prefer_ipv6 = 0, dns_request = 1,
-      onion_traffic = 1,
-      cache_ipv4 = 0, use_cached_ipv4 = 0,
-      cache_ipv6 = 0, use_cached_ipv6 = 0,
-      prefer_ipv6_automap = 1, world_writable = 0, group_writable = 0,
-      relax_dirmode_check = 0,
-      has_used_unix_socket_only_option = 0;
-
-    int is_unix_tagged_addr = 0;
     const char *rest_of_line = NULL;
+
     if (port_cfg_line_extract_addrport(ports->value,
                           &addrport, &is_unix_tagged_addr, &rest_of_line)<0) {
       log_warn(LD_CONFIG, "Invalid %sPort line with unparsable address",
@@ -7038,17 +5947,20 @@ parse_port_config(smartlist_t *out,
       }
     }
 
+    /* Default port_cfg_t object initialization */
+    cfg = port_cfg_new(unix_socket_path ? strlen(unix_socket_path) : 0);
+
     if (unix_socket_path && default_to_group_writable)
-      group_writable = 1;
+      cfg->is_group_writable = 1;
 
     /* Now parse the rest of the options, if any. */
     if (use_server_options) {
       /* This is a server port; parse advertising options */
       SMARTLIST_FOREACH_BEGIN(elts, char *, elt) {
         if (!strcasecmp(elt, "NoAdvertise")) {
-          no_advertise = 1;
+          cfg->server_cfg.no_advertise = 1;
         } else if (!strcasecmp(elt, "NoListen")) {
-          no_listen = 1;
+          cfg->server_cfg.no_listen = 1;
 #if 0
         /* not implemented yet. */
         } else if (!strcasecmp(elt, "AllAddrs")) {
@@ -7056,33 +5968,36 @@ parse_port_config(smartlist_t *out,
           all_addrs = 1;
 #endif /* 0 */
         } else if (!strcasecmp(elt, "IPv4Only")) {
-          bind_ipv4_only = 1;
+          cfg->server_cfg.bind_ipv4_only = 1;
         } else if (!strcasecmp(elt, "IPv6Only")) {
-          bind_ipv6_only = 1;
+          cfg->server_cfg.bind_ipv6_only = 1;
         } else {
           log_warn(LD_CONFIG, "Unrecognized %sPort option '%s'",
                    portname, escaped(elt));
         }
       } SMARTLIST_FOREACH_END(elt);
 
-      if (no_advertise && no_listen) {
+      if (cfg->server_cfg.no_advertise && cfg->server_cfg.no_listen) {
         log_warn(LD_CONFIG, "Tried to set both NoListen and NoAdvertise "
                  "on %sPort line '%s'",
                  portname, escaped(ports->value));
         goto err;
       }
-      if (bind_ipv4_only && bind_ipv6_only) {
+      if (cfg->server_cfg.bind_ipv4_only &&
+          cfg->server_cfg.bind_ipv6_only) {
         log_warn(LD_CONFIG, "Tried to set both IPv4Only and IPv6Only "
                  "on %sPort line '%s'",
                  portname, escaped(ports->value));
         goto err;
       }
-      if (bind_ipv4_only && tor_addr_family(&addr) != AF_INET) {
+      if (cfg->server_cfg.bind_ipv4_only &&
+          tor_addr_family(&addr) != AF_INET) {
         log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv4",
                  portname);
         goto err;
       }
-      if (bind_ipv6_only && tor_addr_family(&addr) != AF_INET6) {
+      if (cfg->server_cfg.bind_ipv6_only &&
+          tor_addr_family(&addr) != AF_INET6) {
         log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv6",
                  portname);
         goto err;
@@ -7101,12 +6016,12 @@ parse_port_config(smartlist_t *out,
                      portname, escaped(elt));
             goto err;
           }
-          if (sessiongroup >= 0) {
+          if (cfg->entry_cfg.session_group >= 0) {
             log_warn(LD_CONFIG, "Multiple SessionGroup options on %sPort",
                      portname);
             goto err;
           }
-          sessiongroup = group;
+          cfg->entry_cfg.session_group = group;
           continue;
         }
 
@@ -7116,15 +6031,15 @@ parse_port_config(smartlist_t *out,
         }
 
         if (!strcasecmp(elt, "GroupWritable")) {
-          group_writable = !no;
+          cfg->is_group_writable = !no;
           has_used_unix_socket_only_option = 1;
           continue;
         } else if (!strcasecmp(elt, "WorldWritable")) {
-          world_writable = !no;
+          cfg->is_world_writable = !no;
           has_used_unix_socket_only_option = 1;
           continue;
         } else if (!strcasecmp(elt, "RelaxDirModeCheck")) {
-          relax_dirmode_check = !no;
+          cfg->relax_dirmode_check = !no;
           has_used_unix_socket_only_option = 1;
           continue;
         }
@@ -7137,19 +6052,19 @@ parse_port_config(smartlist_t *out,
 
         if (takes_hostnames) {
           if (!strcasecmp(elt, "IPv4Traffic")) {
-            ipv4_traffic = ! no;
+            cfg->entry_cfg.ipv4_traffic = ! no;
             continue;
           } else if (!strcasecmp(elt, "IPv6Traffic")) {
-            ipv6_traffic = ! no;
+            cfg->entry_cfg.ipv6_traffic = ! no;
             continue;
           } else if (!strcasecmp(elt, "PreferIPv6")) {
-            prefer_ipv6 = ! no;
+            cfg->entry_cfg.prefer_ipv6 = ! no;
             continue;
           } else if (!strcasecmp(elt, "DNSRequest")) {
-            dns_request = ! no;
+            cfg->entry_cfg.dns_request = ! no;
             continue;
           } else if (!strcasecmp(elt, "OnionTraffic")) {
-            onion_traffic = ! no;
+            cfg->entry_cfg.onion_traffic = ! no;
             continue;
           } else if (!strcasecmp(elt, "OnionTrafficOnly")) {
             /* Only connect to .onion addresses.  Equivalent to
@@ -7160,43 +6075,50 @@ parse_port_config(smartlist_t *out,
                        "DNSRequest, IPv4Traffic, and/or IPv6Traffic instead.",
                        portname, escaped(elt));
             } else {
-              ipv4_traffic = ipv6_traffic = dns_request = 0;
+              cfg->entry_cfg.ipv4_traffic = 0;
+              cfg->entry_cfg.ipv6_traffic = 0;
+              cfg->entry_cfg.dns_request = 0;
             }
             continue;
           }
         }
         if (!strcasecmp(elt, "CacheIPv4DNS")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          cache_ipv4 = ! no;
+          cfg->entry_cfg.cache_ipv4_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "CacheIPv6DNS")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          cache_ipv6 = ! no;
+          cfg->entry_cfg.cache_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "CacheDNS")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          cache_ipv4 = cache_ipv6 = ! no;
+          cfg->entry_cfg.cache_ipv4_answers = ! no;
+          cfg->entry_cfg.cache_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "UseIPv4Cache")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          use_cached_ipv4 = ! no;
+          cfg->entry_cfg.use_cached_ipv4_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "UseIPv6Cache")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          use_cached_ipv6 = ! no;
+          cfg->entry_cfg.use_cached_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "UseDNSCache")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          use_cached_ipv4 = use_cached_ipv6 = ! no;
+          cfg->entry_cfg.use_cached_ipv4_answers = ! no;
+          cfg->entry_cfg.use_cached_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "PreferIPv6Automap")) {
-          prefer_ipv6_automap = ! no;
+          cfg->entry_cfg.prefer_ipv6_virtaddr = ! no;
           continue;
         } else if (!strcasecmp(elt, "PreferSOCKSNoAuth")) {
-          prefer_no_auth = ! no;
+          cfg->entry_cfg.socks_prefer_no_auth = ! no;
           continue;
         } else if (!strcasecmp(elt, "KeepAliveIsolateSOCKSAuth")) {
-          socks_iso_keep_alive = ! no;
+          cfg->entry_cfg.socks_iso_keep_alive = ! no;
+          continue;
+        } else if (!strcasecmp(elt, "ExtendedErrors")) {
+          cfg->entry_cfg.extended_socks5_codes = ! no;
           continue;
         }
 
@@ -7219,9 +6141,9 @@ parse_port_config(smartlist_t *out,
         }
 
         if (no) {
-          isolation &= ~isoflag;
+          cfg->entry_cfg.isolation_flags &= ~isoflag;
         } else {
-          isolation |= isoflag;
+          cfg->entry_cfg.isolation_flags |= isoflag;
         }
       } SMARTLIST_FOREACH_END(elt);
     }
@@ -7231,51 +6153,51 @@ parse_port_config(smartlist_t *out,
     else
       got_zero_port = 1;
 
-    if (dns_request == 0 && listener_type == CONN_TYPE_AP_DNS_LISTENER) {
+    if (cfg->entry_cfg.dns_request == 0 &&
+        listener_type == CONN_TYPE_AP_DNS_LISTENER) {
       log_warn(LD_CONFIG, "You have a %sPort entry with DNS disabled; that "
                "won't work.", portname);
       goto err;
     }
-
-    if (ipv4_traffic == 0 && ipv6_traffic == 0 && onion_traffic == 0
-        && listener_type != CONN_TYPE_AP_DNS_LISTENER) {
+    if (cfg->entry_cfg.ipv4_traffic == 0 &&
+        cfg->entry_cfg.ipv6_traffic == 0 &&
+        cfg->entry_cfg.onion_traffic == 0 &&
+        listener_type != CONN_TYPE_AP_DNS_LISTENER) {
       log_warn(LD_CONFIG, "You have a %sPort entry with all of IPv4 and "
                "IPv6 and .onion disabled; that won't work.", portname);
       goto err;
     }
-
-    if (dns_request == 1 && ipv4_traffic == 0 && ipv6_traffic == 0
-        && listener_type != CONN_TYPE_AP_DNS_LISTENER) {
+    if (cfg->entry_cfg.dns_request == 1 &&
+        cfg->entry_cfg.ipv4_traffic == 0 &&
+        cfg->entry_cfg.ipv6_traffic == 0 &&
+        listener_type != CONN_TYPE_AP_DNS_LISTENER) {
       log_warn(LD_CONFIG, "You have a %sPort entry with DNSRequest enabled, "
                "but IPv4 and IPv6 disabled; DNS-based sites won't work.",
                portname);
       goto err;
     }
-
-    if ( has_used_unix_socket_only_option && ! unix_socket_path) {
+    if (has_used_unix_socket_only_option && !unix_socket_path) {
       log_warn(LD_CONFIG, "You have a %sPort entry with GroupWritable, "
                "WorldWritable, or RelaxDirModeCheck, but it is not a "
                "unix socket.", portname);
       goto err;
     }
-
-    if (!(isolation & ISO_SOCKSAUTH) && socks_iso_keep_alive) {
+    if (!(cfg->entry_cfg.isolation_flags & ISO_SOCKSAUTH) &&
+        cfg->entry_cfg.socks_iso_keep_alive) {
       log_warn(LD_CONFIG, "You have a %sPort entry with both "
                "NoIsolateSOCKSAuth and KeepAliveIsolateSOCKSAuth set.",
                portname);
       goto err;
     }
-
-    if (unix_socket_path && (isolation & ISO_CLIENTADDR)) {
+    if (unix_socket_path &&
+        (cfg->entry_cfg.isolation_flags & ISO_CLIENTADDR)) {
       /* `IsolateClientAddr` is nonsensical in the context of AF_LOCAL.
        * just silently remove the isolation flag.
        */
-      isolation &= ~ISO_CLIENTADDR;
+      cfg->entry_cfg.isolation_flags &= ~ISO_CLIENTADDR;
     }
-
     if (out && port) {
       size_t namelen = unix_socket_path ? strlen(unix_socket_path) : 0;
-      port_cfg_t *cfg = port_cfg_new(namelen);
       if (unix_socket_path) {
         tor_addr_make_unspec(&cfg->addr);
         memcpy(cfg->unix_addr, unix_socket_path, namelen + 1);
@@ -7286,32 +6208,13 @@ parse_port_config(smartlist_t *out,
         cfg->port = port;
       }
       cfg->type = listener_type;
-      cfg->is_world_writable = world_writable;
-      cfg->is_group_writable = group_writable;
-      cfg->relax_dirmode_check = relax_dirmode_check;
-      cfg->entry_cfg.isolation_flags = isolation;
-      cfg->entry_cfg.session_group = sessiongroup;
-      cfg->server_cfg.no_advertise = no_advertise;
-      cfg->server_cfg.no_listen = no_listen;
-      cfg->server_cfg.all_addrs = all_addrs;
-      cfg->server_cfg.bind_ipv4_only = bind_ipv4_only;
-      cfg->server_cfg.bind_ipv6_only = bind_ipv6_only;
-      cfg->entry_cfg.ipv4_traffic = ipv4_traffic;
-      cfg->entry_cfg.ipv6_traffic = ipv6_traffic;
-      cfg->entry_cfg.prefer_ipv6 = prefer_ipv6;
-      cfg->entry_cfg.dns_request = dns_request;
-      cfg->entry_cfg.onion_traffic = onion_traffic;
-      cfg->entry_cfg.cache_ipv4_answers = cache_ipv4;
-      cfg->entry_cfg.cache_ipv6_answers = cache_ipv6;
-      cfg->entry_cfg.use_cached_ipv4_answers = use_cached_ipv4;
-      cfg->entry_cfg.use_cached_ipv6_answers = use_cached_ipv6;
-      cfg->entry_cfg.prefer_ipv6_virtaddr = prefer_ipv6_automap;
-      cfg->entry_cfg.socks_prefer_no_auth = prefer_no_auth;
-      if (! (isolation & ISO_SOCKSAUTH))
+      if (! (cfg->entry_cfg.isolation_flags & ISO_SOCKSAUTH))
         cfg->entry_cfg.socks_prefer_no_auth = 1;
-      cfg->entry_cfg.socks_iso_keep_alive = socks_iso_keep_alive;
-
       smartlist_add(out, cfg);
+      /* out owns cfg now, don't re-use or free it */
+      cfg = NULL;
+    } else {
+      tor_free(cfg);
     }
     SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
     smartlist_clear(elts);
@@ -7323,7 +6226,7 @@ parse_port_config(smartlist_t *out,
     if (is_control)
       warn_nonlocal_controller_ports(out, forbid_nonlocal);
     else if (is_ext_orport)
-      warn_nonlocal_ext_orports(out, portname);
+      port_warn_nonlocal_ext_orports(out, portname);
     else
       warn_nonlocal_client_ports(out, portname, listener_type);
   }
@@ -7337,18 +6240,30 @@ parse_port_config(smartlist_t *out,
 
   retval = 0;
  err:
+  /* There are two ways we can error out:
+   *  1. part way through the loop: cfg needs to be freed;
+   *  2. ending the loop normally: cfg is always NULL.
+   *     In this case, cfg has either been:
+   *     - added to out, then set to NULL, or
+   *     - freed and set to NULL (because out is NULL, or port is 0).
+   */
+  tor_free(cfg);
+
+  /* Free the other variables from the loop.
+   * elts is always non-NULL here, but it may or may not be empty. */
   SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
   smartlist_free(elts);
   tor_free(unix_socket_path);
   tor_free(addrport);
+
   return retval;
 }
 
 /** Return the number of ports which are actually going to listen with type
  * <b>listenertype</b>.  Do not count no_listen ports.  Only count unix
  * sockets if count_sockets is true. */
-static int
-count_real_listeners(const smartlist_t *ports, int listenertype,
+int
+port_count_real_listeners(const smartlist_t *ports, int listenertype,
                      int count_sockets)
 {
   int n = 0;
@@ -7386,7 +6301,7 @@ parse_ports(or_options_t *options, int validate_only,
 
   const unsigned gw_flag = options->UnixSocksGroupWritable ?
     CL_PORT_DFLT_GROUP_WRITABLE : 0;
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
              options->SocksPort_lines,
              "Socks", CONN_TYPE_AP_LISTENER,
              "127.0.0.1", 9050,
@@ -7395,7 +6310,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid SocksPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->DNSPort_lines,
                         "DNS", CONN_TYPE_AP_DNS_LISTENER,
                         "127.0.0.1", 0,
@@ -7403,7 +6318,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid DNSPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->TransPort_lines,
                         "Trans", CONN_TYPE_AP_TRANS_LISTENER,
                         "127.0.0.1", 0,
@@ -7411,7 +6326,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid TransPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->NATDPort_lines,
                         "NATD", CONN_TYPE_AP_NATD_LISTENER,
                         "127.0.0.1", 0,
@@ -7419,7 +6334,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid NatdPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->HTTPTunnelPort_lines,
                         "HTTP Tunnel", CONN_TYPE_AP_HTTP_CONNECT_LISTENER,
                         "127.0.0.1", 0,
@@ -7439,7 +6354,7 @@ parse_ports(or_options_t *options, int validate_only,
     if (options->ControlSocketsGroupWritable)
       control_port_flags |= CL_PORT_DFLT_GROUP_WRITABLE;
 
-    if (parse_port_config(ports,
+    if (port_parse_config(ports,
                           options->ControlPort_lines,
                           "Control", CONN_TYPE_CONTROL_LISTENER,
                           "127.0.0.1", 0,
@@ -7448,7 +6363,7 @@ parse_ports(or_options_t *options, int validate_only,
       goto err;
     }
 
-    if (parse_port_config(ports, options->ControlSocket,
+    if (port_parse_config(ports, options->ControlSocket,
                           "ControlSocket",
                           CONN_TYPE_CONTROL_LISTENER, NULL, 0,
                           control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) {
@@ -7456,40 +6371,9 @@ parse_ports(or_options_t *options, int validate_only,
       goto err;
     }
   }
-  if (! options->ClientOnly) {
-    if (parse_port_config(ports,
-                          options->ORPort_lines,
-                          "OR", CONN_TYPE_OR_LISTENER,
-                          "0.0.0.0", 0,
-                          CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid ORPort configuration");
-      goto err;
-    }
-    if (parse_port_config(ports,
-                          options->ExtORPort_lines,
-                          "ExtOR", CONN_TYPE_EXT_OR_LISTENER,
-                          "127.0.0.1", 0,
-                          CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) {
-      *msg = tor_strdup("Invalid ExtORPort configuration");
-      goto err;
-    }
-    if (parse_port_config(ports,
-                          options->DirPort_lines,
-                          "Dir", CONN_TYPE_DIR_LISTENER,
-                          "0.0.0.0", 0,
-                          CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid DirPort configuration");
-      goto err;
-    }
-  }
 
-  int n_low_ports = 0;
-  if (check_server_ports(ports, options, &n_low_ports) < 0) {
-    *msg = tor_strdup("Misconfigured server ports");
+  if (port_parse_ports_relay(options, msg, ports, &have_low_ports) < 0)
     goto err;
-  }
-  if (have_low_ports < 0)
-    have_low_ports = (n_low_ports > 0);
 
   *n_ports_out = smartlist_len(ports);
 
@@ -7497,25 +6381,20 @@ parse_ports(or_options_t *options, int validate_only,
 
   /* Update the *Port_set options.  The !! here is to force a boolean out of
      an integer. */
-  options->ORPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_OR_LISTENER, 0);
+  port_update_port_set_relay(options, ports);
   options->SocksPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_LISTENER, 1);
   options->TransPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_TRANS_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_TRANS_LISTENER, 1);
   options->NATDPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_NATD_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_NATD_LISTENER, 1);
   options->HTTPTunnelPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_HTTP_CONNECT_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_HTTP_CONNECT_LISTENER, 1);
   /* Use options->ControlSocket to test if a control socket is set */
   options->ControlPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_CONTROL_LISTENER, 0);
-  options->DirPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_DIR_LISTENER, 0);
+    !! port_count_real_listeners(ports, CONN_TYPE_CONTROL_LISTENER, 0);
   options->DNSPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_DNS_LISTENER, 1);
-  options->ExtORPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_EXT_OR_LISTENER, 0);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_DNS_LISTENER, 1);
 
   if (world_writable_control_socket) {
     SMARTLIST_FOREACH(ports, port_cfg_t *, p,
@@ -7546,7 +6425,7 @@ parse_ports(or_options_t *options, int validate_only,
 }
 
 /* Does port bind to IPv4? */
-static int
+int
 port_binds_ipv4(const port_cfg_t *port)
 {
   return tor_addr_family(&port->addr) == AF_INET ||
@@ -7555,7 +6434,7 @@ port_binds_ipv4(const port_cfg_t *port)
 }
 
 /* Does port bind to IPv6? */
-static int
+int
 port_binds_ipv6(const port_cfg_t *port)
 {
   return tor_addr_family(&port->addr) == AF_INET6 ||
@@ -7563,94 +6442,6 @@ port_binds_ipv6(const port_cfg_t *port)
           && !port->server_cfg.bind_ipv4_only);
 }
 
-/** Given a list of <b>port_cfg_t</b> in <b>ports</b>, check them for internal
- * consistency and warn as appropriate.  Set *<b>n_low_ports_out</b> to the
- * number of sub-1024 ports we will be binding. */
-static int
-check_server_ports(const smartlist_t *ports,
-                   const or_options_t *options,
-                   int *n_low_ports_out)
-{
-  int n_orport_advertised = 0;
-  int n_orport_advertised_ipv4 = 0;
-  int n_orport_listeners = 0;
-  int n_dirport_advertised = 0;
-  int n_dirport_listeners = 0;
-  int n_low_port = 0;
-  int r = 0;
-
-  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
-    if (port->type == CONN_TYPE_DIR_LISTENER) {
-      if (! port->server_cfg.no_advertise)
-        ++n_dirport_advertised;
-      if (! port->server_cfg.no_listen)
-        ++n_dirport_listeners;
-    } else if (port->type == CONN_TYPE_OR_LISTENER) {
-      if (! port->server_cfg.no_advertise) {
-        ++n_orport_advertised;
-        if (port_binds_ipv4(port))
-          ++n_orport_advertised_ipv4;
-      }
-      if (! port->server_cfg.no_listen)
-        ++n_orport_listeners;
-    } else {
-      continue;
-    }
-#ifndef _WIN32
-    if (!port->server_cfg.no_listen && port->port < 1024)
-      ++n_low_port;
-#endif
-  } SMARTLIST_FOREACH_END(port);
-
-  if (n_orport_advertised && !n_orport_listeners) {
-    log_warn(LD_CONFIG, "We are advertising an ORPort, but not actually "
-             "listening on one.");
-    r = -1;
-  }
-  if (n_orport_listeners && !n_orport_advertised) {
-    log_warn(LD_CONFIG, "We are listening on an ORPort, but not advertising "
-             "any ORPorts. This will keep us from building a %s "
-             "descriptor, and make us impossible to use.",
-             options->BridgeRelay ? "bridge" : "router");
-    r = -1;
-  }
-  if (n_dirport_advertised && !n_dirport_listeners) {
-    log_warn(LD_CONFIG, "We are advertising a DirPort, but not actually "
-             "listening on one.");
-    r = -1;
-  }
-  if (n_dirport_advertised > 1) {
-    log_warn(LD_CONFIG, "Can't advertise more than one DirPort.");
-    r = -1;
-  }
-  if (n_orport_advertised && !n_orport_advertised_ipv4 &&
-      !options->BridgeRelay) {
-    log_warn(LD_CONFIG, "Configured public relay to listen only on an IPv6 "
-             "address. Tor needs to listen on an IPv4 address too.");
-    r = -1;
-  }
-
-  if (n_low_port && options->AccountingMax &&
-      (!have_capability_support() || options->KeepBindCapabilities == 0)) {
-    const char *extra = "";
-    if (options->KeepBindCapabilities == 0 && have_capability_support())
-      extra = ", and you have disabled KeepBindCapabilities.";
-    log_warn(LD_CONFIG,
-          "You have set AccountingMax to use hibernation. You have also "
-          "chosen a low DirPort or OrPort%s."
-          "This combination can make Tor stop "
-          "working when it tries to re-attach the port after a period of "
-          "hibernation. Please choose a different port or turn off "
-          "hibernation unless you know this combination will work on your "
-          "platform.", extra);
-  }
-
-  if (n_low_ports_out)
-    *n_low_ports_out = n_low_port;
-
-  return r;
-}
-
 /** Return a list of port_cfg_t for client ports parsed from the
  * options. */
 MOCK_IMPL(const smartlist_t *,
@@ -7835,7 +6626,7 @@ get_data_directory(const char *val)
   } else {
     return tor_strdup(get_windows_conf_root());
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *d = val;
   if (!d)
     d = "~/.tor";
@@ -8020,7 +6811,7 @@ get_num_cpus(const or_options_t *options)
 static void
 init_libevent(const or_options_t *options)
 {
-  tor_libevent_cfg cfg;
+  tor_libevent_cfg_t cfg;
 
   tor_assert(options);
 
@@ -8129,43 +6920,6 @@ write_to_data_subdir(const char* subdir, const char* fname,
   return return_val;
 }
 
-/** Return a smartlist of ports that must be forwarded by
- *  tor-fw-helper. The smartlist contains the ports in a string format
- *  that is understandable by tor-fw-helper. */
-smartlist_t *
-get_list_of_ports_to_forward(void)
-{
-  smartlist_t *ports_to_forward = smartlist_new();
-  int port = 0;
-
-  /** XXX TODO tor-fw-helper does not support forwarding ports to
-      other hosts than the local one. If the user is binding to a
-      different IP address, tor-fw-helper won't work.  */
-  port = router_get_advertised_or_port(get_options());  /* Get ORPort */
-  if (port)
-    smartlist_add_asprintf(ports_to_forward, "%d:%d", port, port);
-
-  port = router_get_advertised_dir_port(get_options(), 0); /* Get DirPort */
-  if (port)
-    smartlist_add_asprintf(ports_to_forward, "%d:%d", port, port);
-
-  /* Get ports of transport proxies */
-  {
-    smartlist_t *transport_ports = get_transport_proxy_ports();
-    if (transport_ports) {
-      smartlist_add_all(ports_to_forward, transport_ports);
-      smartlist_free(transport_ports);
-    }
-  }
-
-  if (!smartlist_len(ports_to_forward)) {
-    smartlist_free(ports_to_forward);
-    ports_to_forward = NULL;
-  }
-
-  return ports_to_forward;
-}
-
 /** Helper to implement GETINFO functions about configuration variables (not
  * their values).  Given a "config/names" question, set *<b>answer</b> to a
  * new string describing the supported configuration variables and their
@@ -8179,72 +6933,48 @@ getinfo_helper_config(control_connection_t *conn,
   (void) errmsg;
   if (!strcmp(question, "config/names")) {
     smartlist_t *sl = smartlist_new();
-    int i;
-    for (i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
-      const char *type;
-      /* don't tell controller about triple-underscore options */
-      if (!strncmp(option_vars_[i].name, "___", 3))
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+      /* don't tell controller about invisible options */
+      if (! config_var_is_listable(var))
         continue;
-      switch (var->type) {
-        case CONFIG_TYPE_STRING: type = "String"; break;
-        case CONFIG_TYPE_FILENAME: type = "Filename"; break;
-        case CONFIG_TYPE_UINT: type = "Integer"; break;
-        case CONFIG_TYPE_UINT64: type = "Integer"; break;
-        case CONFIG_TYPE_INT: type = "SignedInteger"; break;
-        case CONFIG_TYPE_PORT: type = "Port"; break;
-        case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break;
-        case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break;
-        case CONFIG_TYPE_DOUBLE: type = "Float"; break;
-        case CONFIG_TYPE_BOOL: type = "Boolean"; break;
-        case CONFIG_TYPE_AUTOBOOL: type = "Boolean+Auto"; break;
-        case CONFIG_TYPE_ISOTIME: type = "Time"; break;
-        case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break;
-        case CONFIG_TYPE_CSV: type = "CommaList"; break;
-        /* This type accepts more inputs than TimeInterval, but it ignores
-         * everything after the first entry, so we may as well pretend
-         * it's a TimeInterval. */
-        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_LINELIST: type = "LineList"; break;
-        case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break;
-        case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break;
-        default:
-        case CONFIG_TYPE_OBSOLETE:
-          type = NULL; break;
-      }
+      const char *type = struct_var_get_typename(&var->member);
       if (!type)
         continue;
-      smartlist_add_asprintf(sl, "%s %s\n",var->name,type);
-    }
+      smartlist_add_asprintf(sl, "%s %s\n",var->member.name,type);
+    } SMARTLIST_FOREACH_END(var);
     *answer = smartlist_join_strings(sl, "", 0, NULL);
     SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
     smartlist_free(sl);
+    smartlist_free(vars);
   } else if (!strcmp(question, "config/defaults")) {
     smartlist_t *sl = smartlist_new();
     int dirauth_lines_seen = 0, fallback_lines_seen = 0;
-    for (int i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
+    /* Possibly this should check whether the variables are listable,
+     * but currently it does not.  See ticket 31654. */
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
       if (var->initvalue != NULL) {
-        if (strcmp(option_vars_[i].name, "DirAuthority") == 0) {
+        if (strcmp(var->member.name, "DirAuthority") == 0) {
           /*
            * Count dirauth lines we have a default for; we'll use the
            * count later to decide whether to add the defaults manually
            */
           ++dirauth_lines_seen;
         }
-        if (strcmp(option_vars_[i].name, "FallbackDir") == 0) {
+        if (strcmp(var->member.name, "FallbackDir") == 0) {
           /*
-           * Similarly count fallback lines, so that we can decided later
+           * Similarly count fallback lines, so that we can decide later
            * to add the defaults manually.
            */
           ++fallback_lines_seen;
         }
         char *val = esc_for_log(var->initvalue);
-        smartlist_add_asprintf(sl, "%s %s\n",var->name,val);
+        smartlist_add_asprintf(sl, "%s %s\n",var->member.name,val);
         tor_free(val);
       }
-    }
+    } SMARTLIST_FOREACH_END(var);
+    smartlist_free(vars);
 
     if (dirauth_lines_seen == 0) {
       /*
@@ -8405,7 +7135,7 @@ config_load_geoip_file_(sa_family_t family,
   }
   r = geoip_load_file(family, fname, severity);
   tor_free(free_fname);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)default_fname;
   r = geoip_load_file(family, fname, severity);
 #endif /* defined(_WIN32) */
@@ -8493,7 +7223,7 @@ init_cookie_authentication(const char *fname, const char *header,
       log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname));
     }
   }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void) group_readable;
 #endif /* !defined(_WIN32) */
 
diff --git a/src/app/config/config.h b/src/app/config/config.h
index 6852d352dc..1ba10d1d37 100644
--- a/src/app/config/config.h
+++ b/src/app/config/config.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@
 
 #include "app/config/or_options_st.h"
 #include "lib/testsupport/testsupport.h"
+#include "app/config/quiet_level.h"
 
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(DARWIN)
 #define KERNEL_MAY_SUPPORT_IPFW
@@ -30,7 +31,6 @@
 #define MAX_DEFAULT_MEMORY_QUEUE_SIZE (UINT64_C(2) << 30)
 #endif
 
-MOCK_DECL(const char*, get_dirportfrontpage, (void));
 MOCK_DECL(const or_options_t *, get_options, (void));
 MOCK_DECL(or_options_t *, get_options_mutable, (void));
 int set_options(or_options_t *new_val, char **msg);
@@ -41,8 +41,8 @@ const char *escaped_safe_str_client(const char *address);
 const char *escaped_safe_str(const char *address);
 void init_protocol_warning_severity_level(void);
 int get_protocol_warning_severity_level(void);
-const char *get_version(void);
-const char *get_short_version(void);
+
+#define LOG_PROTOCOL_WARN (get_protocol_warning_severity_level())
 
 /** An error from options_trial_assign() or options_init_from_string(). */
 typedef enum setopt_err_t {
@@ -55,12 +55,6 @@ typedef enum setopt_err_t {
 setopt_err_t options_trial_assign(struct config_line_t *list, unsigned flags,
                                   char **msg);
 
-uint32_t get_last_resolved_addr(void);
-void reset_last_resolved_addr(void);
-int resolve_my_address(int warn_severity, const or_options_t *options,
-                       uint32_t *addr_out,
-                       const char **method_out, char **hostname_out);
-MOCK_DECL(int, is_local_addr, (const tor_addr_t *addr));
 void options_init(or_options_t *options);
 
 #define OPTIONS_DUMP_MINIMAL 1
@@ -164,6 +158,8 @@ int write_to_data_subdir(const char* subdir, const char* fname,
 int get_num_cpus(const or_options_t *options);
 
 MOCK_DECL(const smartlist_t *,get_configured_ports,(void));
+int port_binds_ipv4(const port_cfg_t *port);
+int port_binds_ipv6(const port_cfg_t *port);
 int get_first_advertised_port_by_type_af(int listener_type,
                                          int address_family);
 #define get_primary_or_port() \
@@ -182,26 +178,36 @@ char *get_first_listener_addrport_string(int listener_type);
 int options_need_geoip_info(const or_options_t *options,
                             const char **reason_out);
 
-smartlist_t *get_list_of_ports_to_forward(void);
-
 int getinfo_helper_config(control_connection_t *conn,
                           const char *question, char **answer,
                           const char **errmsg);
 
-uint32_t get_effective_bwrate(const or_options_t *options);
-uint32_t get_effective_bwburst(const or_options_t *options);
-
-char *get_transport_bindaddr_from_config(const char *transport);
-
 int init_cookie_authentication(const char *fname, const char *header,
                                int cookie_len, int group_readable,
                                uint8_t **cookie_out, int *cookie_is_set_out);
 
 or_options_t *options_new(void);
 
-int config_parse_commandline(int argc, char **argv, int ignore_errors,
-                             struct config_line_t **result,
-                             struct config_line_t **cmdline_result);
+/** Options settings parsed from the command-line. */
+typedef struct {
+  /** List of options that can only be set from the command-line */
+  struct config_line_t *cmdline_opts;
+  /** List of other options, to be handled by the general Tor configuration
+      system. */
+  struct config_line_t *other_opts;
+  /** Subcommand that Tor has been told to run */
+  tor_cmdline_mode_t command;
+  /** Argument for the command mode, if any. */
+  const char *command_arg;
+  /** How quiet have we been told to be? */
+  quiet_level_t quiet_level;
+} parsed_cmdline_t;
+
+parsed_cmdline_t *config_parse_commandline(int argc, char **argv,
+                                           int ignore_errors);
+void parsed_cmdline_free_(parsed_cmdline_t *cmdline);
+#define parsed_cmdline_free(c) \
+  FREE_AND_NULL(parsed_cmdline_t, parsed_cmdline_free_, (c))
 
 void config_register_addressmaps(const or_options_t *options);
 /* XXXX move to connection_edge.h */
@@ -230,14 +236,16 @@ void bridge_line_free_(bridge_line_t *bridge_line);
 #define bridge_line_free(line) \
   FREE_AND_NULL(bridge_line_t, bridge_line_free_, (line))
 bridge_line_t *parse_bridge_line(const char *line);
-smartlist_t *get_options_from_transport_options_line(const char *line,
-                                                     const char *transport);
-smartlist_t *get_options_for_server_transport(const char *transport);
 
 /* Port helper functions. */
 int options_any_client_port_set(const or_options_t *options);
-
-#ifdef CONFIG_PRIVATE
+int port_parse_config(smartlist_t *out,
+                      const struct config_line_t *ports,
+                      const char *portname,
+                      int listener_type,
+                      const char *defaultaddr,
+                      int defaultport,
+                      const unsigned flags);
 
 #define CL_PORT_NO_STREAM_OPTIONS (1u<<0)
 #define CL_PORT_WARN_NONLOCAL (1u<<1)
@@ -248,27 +256,34 @@ int options_any_client_port_set(const or_options_t *options);
 #define CL_PORT_IS_UNIXSOCKET (1u<<6)
 #define CL_PORT_DFLT_GROUP_WRITABLE (1u<<7)
 
-STATIC int options_act(const or_options_t *old_options);
-#ifdef TOR_UNIT_TESTS
-extern struct config_format_t options_format;
-#endif
-
-STATIC port_cfg_t *port_cfg_new(size_t namelen);
+port_cfg_t *port_cfg_new(size_t namelen);
 #define port_cfg_free(port) \
   FREE_AND_NULL(port_cfg_t, port_cfg_free_, (port))
-STATIC void port_cfg_free_(port_cfg_t *port);
+void port_cfg_free_(port_cfg_t *port);
+
+int port_count_real_listeners(const smartlist_t *ports,
+                         int listenertype,
+                         int count_sockets);
+int pt_parse_transport_line(const or_options_t *options,
+                         const char *line, int validate_only,
+                         int server);
+int config_ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg);
+
+#ifdef CONFIG_PRIVATE
+
+MOCK_DECL(STATIC int, options_act,(const or_options_t *old_options));
+MOCK_DECL(STATIC int, options_act_reversible,(const or_options_t *old_options,
+                                             char **msg));
+struct config_mgr_t;
+STATIC const struct config_mgr_t *get_options_mgr(void);
+
 #define or_options_free(opt) \
   FREE_AND_NULL(or_options_t, or_options_free_, (opt))
 STATIC void or_options_free_(or_options_t *options);
 STATIC int options_validate_single_onion(or_options_t *options,
                                          char **msg);
-STATIC int options_validate(or_options_t *old_options,
-                            or_options_t *options,
-                            or_options_t *default_options,
-                            int from_setconf, char **msg);
-STATIC int parse_transport_line(const or_options_t *options,
-                                const char *line, int validate_only,
-                                int server);
+STATIC int parse_tcp_proxy_line(const char *line, or_options_t *options,
+                                char **msg);
 STATIC int consider_adding_dir_servers(const or_options_t *options,
                                        const or_options_t *old_options);
 STATIC void add_default_trusted_dir_authorities(dirinfo_type_t type);
@@ -277,23 +292,28 @@ STATIC int parse_dir_authority_line(const char *line,
                                     dirinfo_type_t required_type,
                                     int validate_only);
 STATIC int parse_dir_fallback_line(const char *line, int validate_only);
-STATIC int have_enough_mem_for_dircache(const or_options_t *options,
-                                        size_t total_mem, char **msg);
-STATIC int parse_port_config(smartlist_t *out,
-                  const struct config_line_t *ports,
-                  const char *portname,
-                  int listener_type,
-                  const char *defaultaddr,
-                  int defaultport,
-                  const unsigned flags);
-
-STATIC int check_bridge_distribution_setting(const char *bd);
 
 STATIC uint64_t compute_real_max_mem_in_queues(const uint64_t val,
-                                               int log_guess);
+                                               bool is_server);
 STATIC int open_and_add_file_log(const log_severity_list_t *severity,
                                  const char *fname,
                                  int truncate_log);
+STATIC int options_init_logs(const or_options_t *old_options,
+                             const or_options_t *options, int validate_only);
+
+STATIC int options_create_directories(char **msg_out);
+struct log_transaction_t;
+STATIC struct log_transaction_t *options_start_log_transaction(
+                              const or_options_t *old_options,
+                              char **msg_out);
+STATIC void options_commit_log_transaction(struct log_transaction_t *xn);
+STATIC void options_rollback_log_transaction(struct log_transaction_t *xn);
+
+#ifdef TOR_UNIT_TESTS
+int options_validate(const or_options_t *old_options,
+                     or_options_t *options,
+                     char **msg);
+#endif
 
 STATIC int parse_ports(or_options_t *options, int validate_only,
                        char **msg, int *n_ports_out,
diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c
deleted file mode 100644
index efa0c19fa6..0000000000
--- a/src/app/config/confparse.c
+++ /dev/null
@@ -1,1207 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.c
- *
- * \brief Back-end for parsing and generating key-value files, used to
- *   implement the torrc file format and the state file.
- *
- * This module is used by config.c to parse and encode torrc
- * configuration files, and by statefile.c to parse and encode the
- * $DATADIR/state file.
- *
- * To use this module, its callers provide an instance of
- * config_format_t to describe the mappings from a set of configuration
- * options to a number of fields in a C structure.  With this mapping,
- * the functions here can convert back and forth between the C structure
- * specified, and a linked list of key-value pairs.
- */
-
-#include "core/or/or.h"
-#include "app/config/confparse.h"
-#include "feature/nodelist/routerset.h"
-
-#include "lib/container/bitarray.h"
-#include "lib/encoding/confline.h"
-
-static uint64_t config_parse_memunit(const char *s, int *ok);
-static int config_parse_msec_interval(const char *s, int *ok);
-static int config_parse_interval(const char *s, int *ok);
-static void config_reset(const config_format_t *fmt, void *options,
-                         const config_var_t *var, int use_defaults);
-
-/** Allocate an empty configuration object of a given format type. */
-void *
-config_new(const config_format_t *fmt)
-{
-  void *opts = tor_malloc_zero(fmt->size);
-  *(uint32_t*)STRUCT_VAR_P(opts, fmt->magic_offset) = fmt->magic;
-  CONFIG_CHECK(fmt, opts);
-  return opts;
-}
-
-/*
- * Functions to parse config options
- */
-
-/** If <b>option</b> is an official abbreviation for a longer option,
- * return the longer option.  Otherwise return <b>option</b>.
- * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
- * apply abbreviations that work for the config file and the command line.
- * If <b>warn_obsolete</b> is set, warn about deprecated names. */
-const char *
-config_expand_abbrev(const config_format_t *fmt, const char *option,
-                     int command_line, int warn_obsolete)
-{
-  int i;
-  if (! fmt->abbrevs)
-    return option;
-  for (i=0; fmt->abbrevs[i].abbreviated; ++i) {
-    /* Abbreviations are case insensitive. */
-    if (!strcasecmp(option,fmt->abbrevs[i].abbreviated) &&
-        (command_line || !fmt->abbrevs[i].commandline_only)) {
-      if (warn_obsolete && fmt->abbrevs[i].warn) {
-        log_warn(LD_CONFIG,
-                 "The configuration option '%s' is deprecated; "
-                 "use '%s' instead.",
-                 fmt->abbrevs[i].abbreviated,
-                 fmt->abbrevs[i].full);
-      }
-      /* Keep going through the list in case we want to rewrite it more.
-       * (We could imagine recursing here, but I don't want to get the
-       * user into an infinite loop if we craft our list wrong.) */
-      option = fmt->abbrevs[i].full;
-    }
-  }
-  return option;
-}
-
-/** If <b>key</b> is a deprecated configuration option, return the message
- * explaining why it is deprecated (which may be an empty string). Return NULL
- * if it is not deprecated. The <b>key</b> field must be fully expanded. */
-const char *
-config_find_deprecation(const config_format_t *fmt, const char *key)
-{
-  if (BUG(fmt == NULL) || BUG(key == NULL))
-    return NULL;
-  if (fmt->deprecations == NULL)
-    return NULL;
-
-  const config_deprecation_t *d;
-  for (d = fmt->deprecations; d->name; ++d) {
-    if (!strcasecmp(d->name, key)) {
-      return d->why_deprecated ? d->why_deprecated : "";
-    }
-  }
-  return NULL;
-}
-
-/** As config_find_option, but return a non-const pointer. */
-config_var_t *
-config_find_option_mutable(config_format_t *fmt, const char *key)
-{
-  int i;
-  size_t keylen = strlen(key);
-  if (!keylen)
-    return NULL; /* if they say "--" on the command line, it's not an option */
-  /* First, check for an exact (case-insensitive) match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strcasecmp(key, fmt->vars[i].name)) {
-      return &fmt->vars[i];
-    }
-  }
-  /* If none, check for an abbreviated match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strncasecmp(key, fmt->vars[i].name, keylen)) {
-      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
-               "Please use '%s' instead",
-               key, fmt->vars[i].name);
-      return &fmt->vars[i];
-    }
-  }
-  /* Okay, unrecognized option */
-  return NULL;
-}
-
-/** If <b>key</b> is a configuration option, return the corresponding const
- * config_var_t.  Otherwise, if <b>key</b> is a non-standard abbreviation,
- * warn, and return the corresponding const config_var_t.  Otherwise return
- * NULL.
- */
-const config_var_t *
-config_find_option(const config_format_t *fmt, const char *key)
-{
-  return config_find_option_mutable((config_format_t*)fmt, key);
-}
-
-/** Return the number of option entries in <b>fmt</b>. */
-static int
-config_count_options(const config_format_t *fmt)
-{
-  int i;
-  for (i=0; fmt->vars[i].name; ++i)
-    ;
-  return i;
-}
-
-/*
- * Functions to assign config options.
- */
-
-/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
- * with <b>c</b>-\>value and return 0, or return -1 if bad value.
- *
- * Called from config_assign_line() and option_reset().
- */
-static int
-config_assign_value(const config_format_t *fmt, void *options,
-                    config_line_t *c, char **msg)
-{
-  int i, ok;
-  const config_var_t *var;
-  void *lvalue;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  tor_assert(var);
-
-  lvalue = STRUCT_VAR_P(options, var->var_offset);
-
-  switch (var->type) {
-
-  case CONFIG_TYPE_PORT:
-    if (!strcasecmp(c->value, "auto")) {
-      *(int *)lvalue = CFG_AUTO_PORT;
-      break;
-    }
-    FALLTHROUGH;
-  case CONFIG_TYPE_INT:
-  case CONFIG_TYPE_UINT:
-    i = (int)tor_parse_long(c->value, 10,
-                            var->type==CONFIG_TYPE_INT ? INT_MIN : 0,
-                            var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX,
-                            &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Int keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_UINT64: {
-    uint64_t u64 = tor_parse_uint64(c->value, 10,
-                                    0, UINT64_MAX, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "uint64 keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_CSV_INTERVAL: {
-    /* We used to have entire smartlists here.  But now that all of our
-     * download schedules use exponential backoff, only the first part
-     * matters. */
-    const char *comma = strchr(c->value, ',');
-    const char *val = c->value;
-    char *tmp = NULL;
-    if (comma) {
-      tmp = tor_strndup(c->value, comma - c->value);
-      val = tmp;
-    }
-
-    i = config_parse_interval(val, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      tor_free(tmp);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    tor_free(tmp);
-    break;
-  }
-
-  case CONFIG_TYPE_INTERVAL: {
-    i = config_parse_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MSEC_INTERVAL: {
-    i = config_parse_msec_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Msec interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MEMUNIT: {
-    uint64_t u64 = config_parse_memunit(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Value '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_BOOL:
-    i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Boolean '%s %s' expects 0 or 1.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_AUTOBOOL:
-    if (!strcasecmp(c->value, "auto"))
-      *(int *)lvalue = -1;
-    else if (!strcmp(c->value, "0"))
-      *(int *)lvalue = 0;
-    else if (!strcmp(c->value, "1"))
-      *(int *)lvalue = 1;
-    else {
-      tor_asprintf(msg, "Boolean '%s %s' expects 0, 1, or 'auto'.",
-                   c->key, c->value);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_STRING:
-  case CONFIG_TYPE_FILENAME:
-    tor_free(*(char **)lvalue);
-    *(char **)lvalue = tor_strdup(c->value);
-    break;
-
-  case CONFIG_TYPE_DOUBLE:
-    *(double *)lvalue = atof(c->value);
-    break;
-
-  case CONFIG_TYPE_ISOTIME:
-    if (parse_iso_time(c->value, (time_t *)lvalue)) {
-      tor_asprintf(msg,
-          "Invalid time '%s' for keyword '%s'", c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_ROUTERSET:
-    if (*(routerset_t**)lvalue) {
-      routerset_free(*(routerset_t**)lvalue);
-    }
-    *(routerset_t**)lvalue = routerset_new();
-    if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) {
-      tor_asprintf(msg, "Invalid exit list '%s' for option '%s'",
-                   c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_CSV:
-    if (*(smartlist_t**)lvalue) {
-      SMARTLIST_FOREACH(*(smartlist_t**)lvalue, char *, cp, tor_free(cp));
-      smartlist_clear(*(smartlist_t**)lvalue);
-    } else {
-      *(smartlist_t**)lvalue = smartlist_new();
-    }
-
-    smartlist_split_string(*(smartlist_t**)lvalue, c->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    break;
-
-  case CONFIG_TYPE_LINELIST:
-  case CONFIG_TYPE_LINELIST_S:
-    {
-      config_line_t *lastval = *(config_line_t**)lvalue;
-      if (lastval && lastval->fragile) {
-        if (c->command != CONFIG_LINE_APPEND) {
-          config_free_lines(lastval);
-          *(config_line_t**)lvalue = NULL;
-        } else {
-          lastval->fragile = 0;
-        }
-      }
-
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-    }
-    break;
-  case CONFIG_TYPE_OBSOLETE:
-    log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key);
-    break;
-  case CONFIG_TYPE_LINELIST_V:
-    tor_asprintf(msg,
-        "You may not provide a value for virtual option '%s'", c->key);
-    return -1;
-  default:
-    tor_assert(0);
-    break;
-  }
-  return 0;
-}
-
-/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
- * to it will replace old ones. */
-static void
-config_mark_lists_fragile(const config_format_t *fmt, void *options)
-{
-  int i;
-  tor_assert(fmt);
-  tor_assert(options);
-
-  for (i = 0; fmt->vars[i].name; ++i) {
-    const config_var_t *var = &fmt->vars[i];
-    config_line_t *list;
-    if (var->type != CONFIG_TYPE_LINELIST &&
-        var->type != CONFIG_TYPE_LINELIST_V)
-      continue;
-
-    list = *(config_line_t **)STRUCT_VAR_P(options, var->var_offset);
-    if (list)
-      list->fragile = 1;
-  }
-}
-
-void
-warn_deprecated_option(const char *what, const char *why)
-{
-  const char *space = (why && strlen(why)) ? " " : "";
-  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
-           "be removed in a future version of Tor.%s%s (If you think this is "
-           "a mistake, please let us know!)",
-           what, space, why);
-}
-
-/** If <b>c</b> is a syntactically valid configuration line, update
- * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
- * key, -2 for bad value.
- *
- * If <b>clear_first</b> is set, clear the value first. Then if
- * <b>use_defaults</b> is set, set the value to the default.
- *
- * Called from config_assign().
- */
-static int
-config_assign_line(const config_format_t *fmt, void *options,
-                   config_line_t *c, unsigned flags,
-                   bitarray_t *options_seen, char **msg)
-{
-  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
-  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
-  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  if (!var) {
-    if (fmt->extra) {
-      void *lvalue = STRUCT_VAR_P(options, fmt->extra->var_offset);
-      log_info(LD_CONFIG,
-               "Found unrecognized option '%s'; saving it.", c->key);
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-      return 0;
-    } else {
-      tor_asprintf(msg,
-                "Unknown option '%s'.  Failing.", c->key);
-      return -1;
-    }
-  }
-
-  /* Put keyword into canonical case. */
-  if (strcmp(var->name, c->key)) {
-    tor_free(c->key);
-    c->key = tor_strdup(var->name);
-  }
-
-  const char *deprecation_msg;
-  if (warn_deprecations &&
-      (deprecation_msg = config_find_deprecation(fmt, var->name))) {
-    warn_deprecated_option(var->name, deprecation_msg);
-  }
-
-  if (!strlen(c->value)) {
-    /* reset or clear it, then return */
-    if (!clear_first) {
-      if ((var->type == CONFIG_TYPE_LINELIST ||
-           var->type == CONFIG_TYPE_LINELIST_S) &&
-          c->command != CONFIG_LINE_CLEAR) {
-        /* We got an empty linelist from the torrc or command line.
-           As a special case, call this an error. Warn and ignore. */
-        log_warn(LD_CONFIG,
-                 "Linelist option '%s' has no value. Skipping.", c->key);
-      } else { /* not already cleared */
-        config_reset(fmt, options, var, use_defaults);
-      }
-    }
-    return 0;
-  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
-    config_reset(fmt, options, var, use_defaults);
-  }
-
-  if (options_seen && (var->type != CONFIG_TYPE_LINELIST &&
-                       var->type != CONFIG_TYPE_LINELIST_S)) {
-    /* We're tracking which options we've seen, and this option is not
-     * supposed to occur more than once. */
-    int var_index = (int)(var - fmt->vars);
-    if (bitarray_is_set(options_seen, var_index)) {
-      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
-               "value will be ignored.", var->name);
-    }
-    bitarray_set(options_seen, var_index);
-  }
-
-  if (config_assign_value(fmt, options, c, msg) < 0)
-    return -2;
-  return 0;
-}
-
-/** Restore the option named <b>key</b> in options to its default value.
- * Called from config_assign(). */
-static void
-config_reset_line(const config_format_t *fmt, void *options,
-                  const char *key, int use_defaults)
-{
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var)
-    return; /* give error on next pass. */
-
-  config_reset(fmt, options, var, use_defaults);
-}
-
-/** Return true iff value needs to be quoted and escaped to be used in
- * a configuration file. */
-static int
-config_value_needs_escape(const char *value)
-{
-  if (*value == '\"')
-    return 1;
-  while (*value) {
-    switch (*value)
-    {
-    case '\r':
-    case '\n':
-    case '#':
-      /* Note: quotes and backspaces need special handling when we are using
-       * quotes, not otherwise, so they don't trigger escaping on their
-       * own. */
-      return 1;
-    default:
-      if (!TOR_ISPRINT(*value))
-        return 1;
-    }
-    ++value;
-  }
-  return 0;
-}
-
-/** Return newly allocated line or lines corresponding to <b>key</b> in the
- * configuration <b>options</b>.  If <b>escape_val</b> is true and a
- * value needs to be quoted before it's put in a config file, quote and
- * escape that value. Return NULL if no such key exists. */
-config_line_t *
-config_get_assigned_option(const config_format_t *fmt, const void *options,
-                           const char *key, int escape_val)
-{
-  const config_var_t *var;
-  const void *value;
-  config_line_t *result;
-  tor_assert(options && key);
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var) {
-    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
-    return NULL;
-  }
-  value = STRUCT_VAR_P(options, var->var_offset);
-
-  result = tor_malloc_zero(sizeof(config_line_t));
-  result->key = tor_strdup(var->name);
-  switch (var->type)
-    {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      if (*(char**)value) {
-        result->value = tor_strdup(*(char**)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-        return NULL;
-      }
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      if (*(time_t*)value) {
-        result->value = tor_malloc(ISO_TIME_LEN+1);
-        format_iso_time(result->value, *(time_t*)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-      }
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_PORT:
-      if (*(int*)value == CFG_AUTO_PORT) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-      /* This means every or_options_t uint or bool element
-       * needs to be an int. Not, say, a uint16_t or char. */
-      tor_asprintf(&result->value, "%d", *(int*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_UINT64: FALLTHROUGH;
-    case CONFIG_TYPE_MEMUNIT:
-      tor_asprintf(&result->value, "%"PRIu64,
-                   (*(uint64_t*)value));
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      tor_asprintf(&result->value, "%f", *(double*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-
-    case CONFIG_TYPE_AUTOBOOL:
-      if (*(int*)value == -1) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_BOOL:
-      result->value = tor_strdup(*(int*)value ? "1" : "0");
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      result->value = routerset_to_string(*(routerset_t**)value);
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)value)
-        result->value =
-          smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
-      else
-        result->value = tor_strdup("");
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      log_fn(LOG_INFO, LD_CONFIG,
-             "You asked me for the value of an obsolete config option '%s'.",
-             key);
-      tor_free(result->key);
-      tor_free(result);
-      return NULL;
-    case CONFIG_TYPE_LINELIST_S:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup_and_filter(*(const config_line_t **)value,
-                                           key);
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_V:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup(*(const config_line_t**)value);
-      break;
-    default:
-      tor_free(result->key);
-      tor_free(result);
-      log_warn(LD_BUG,"Unknown type %d for known key '%s'",
-               var->type, key);
-      return NULL;
-    }
-
-  if (escape_val) {
-    config_line_t *line;
-    for (line = result; line; line = line->next) {
-      if (line->value && config_value_needs_escape(line->value)) {
-        char *newval = esc_for_log(line->value);
-        tor_free(line->value);
-        line->value = newval;
-      }
-    }
-  }
-
-  return result;
-}
-/** Iterate through the linked list of requested options <b>list</b>.
- * For each item, convert as appropriate and assign to <b>options</b>.
- * If an item is unrecognized, set *msg and return -1 immediately,
- * else return 0 for success.
- *
- * If <b>clear_first</b>, interpret config options as replacing (not
- * extending) their previous values. If <b>clear_first</b> is set,
- * then <b>use_defaults</b> to decide if you set to defaults after
- * clearing, or make the value 0 or NULL.
- *
- * Here are the use cases:
- * 1. A non-empty AllowInvalid line in your torrc. Appends to current
- *    if linelist, replaces current if csv.
- * 2. An empty AllowInvalid line in your torrc. Should clear it.
- * 3. "RESETCONF AllowInvalid" sets it to default.
- * 4. "SETCONF AllowInvalid" makes it NULL.
- * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
- *
- * Use_defaults   Clear_first
- *    0                0       "append"
- *    1                0       undefined, don't use
- *    0                1       "set to null first"
- *    1                1       "set to defaults first"
- * Return 0 on success, -1 on bad key, -2 on bad value.
- *
- * As an additional special case, if a LINELIST config option has
- * no value and clear_first is 0, then warn and ignore it.
- */
-
-/*
-There are three call cases for config_assign() currently.
-
-Case one: Torrc entry
-options_init_from_torrc() calls config_assign(0, 0)
-  calls config_assign_line(0, 0).
-    if value is empty, calls config_reset(0) and returns.
-    calls config_assign_value(), appends.
-
-Case two: setconf
-options_trial_assign() calls config_assign(0, 1)
-  calls config_reset_line(0)
-    calls config_reset(0)
-      calls option_clear().
-  calls config_assign_line(0, 1).
-    if value is empty, returns.
-    calls config_assign_value(), appends.
-
-Case three: resetconf
-options_trial_assign() calls config_assign(1, 1)
-  calls config_reset_line(1)
-    calls config_reset(1)
-      calls option_clear().
-      calls config_assign_value(default)
-  calls config_assign_line(1, 1).
-    returns.
-*/
-int
-config_assign(const config_format_t *fmt, void *options, config_line_t *list,
-              unsigned config_assign_flags, char **msg)
-{
-  config_line_t *p;
-  bitarray_t *options_seen;
-  const int n_options = config_count_options(fmt);
-  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
-  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
-
-  CONFIG_CHECK(fmt, options);
-
-  /* pass 1: normalize keys */
-  for (p = list; p; p = p->next) {
-    const char *full = config_expand_abbrev(fmt, p->key, 0, 1);
-    if (strcmp(full,p->key)) {
-      tor_free(p->key);
-      p->key = tor_strdup(full);
-    }
-  }
-
-  /* pass 2: if we're reading from a resetting source, clear all
-   * mentioned config options, and maybe set to their defaults. */
-  if (clear_first) {
-    for (p = list; p; p = p->next)
-      config_reset_line(fmt, options, p->key, use_defaults);
-  }
-
-  options_seen = bitarray_init_zero(n_options);
-  /* pass 3: assign. */
-  while (list) {
-    int r;
-    if ((r=config_assign_line(fmt, options, list, config_assign_flags,
-                              options_seen, msg))) {
-      bitarray_free(options_seen);
-      return r;
-    }
-    list = list->next;
-  }
-  bitarray_free(options_seen);
-
-  /** Now we're done assigning a group of options to the configuration.
-   * Subsequent group assignments should _replace_ linelists, not extend
-   * them. */
-  config_mark_lists_fragile(fmt, options);
-
-  return 0;
-}
-
-/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
- * Called from config_reset() and config_free(). */
-static void
-config_clear(const config_format_t *fmt, void *options,
-             const config_var_t *var)
-{
-  void *lvalue = STRUCT_VAR_P(options, var->var_offset);
-  (void)fmt; /* unused */
-  switch (var->type) {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      tor_free(*(char**)lvalue);
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      *(double*)lvalue = 0.0;
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      *(time_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-    case CONFIG_TYPE_PORT:
-    case CONFIG_TYPE_BOOL:
-      *(int*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_AUTOBOOL:
-      *(int*)lvalue = -1;
-      break;
-    case CONFIG_TYPE_UINT64:
-    case CONFIG_TYPE_MEMUNIT:
-      *(uint64_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      if (*(routerset_t**)lvalue) {
-        routerset_free(*(routerset_t**)lvalue);
-        *(routerset_t**)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)lvalue) {
-        SMARTLIST_FOREACH(*(smartlist_t **)lvalue, char *, cp, tor_free(cp));
-        smartlist_free(*(smartlist_t **)lvalue);
-        *(smartlist_t **)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_S:
-      config_free_lines(*(config_line_t **)lvalue);
-      *(config_line_t **)lvalue = NULL;
-      break;
-    case CONFIG_TYPE_LINELIST_V:
-      /* handled by linelist_s. */
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      break;
-  }
-}
-
-/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
- * <b>use_defaults</b>, set it to its default value.
- * Called by config_init() and option_reset_line() and option_assign_line(). */
-static void
-config_reset(const config_format_t *fmt, void *options,
-             const config_var_t *var, int use_defaults)
-{
-  config_line_t *c;
-  char *msg = NULL;
-  CONFIG_CHECK(fmt, options);
-  config_clear(fmt, options, var); /* clear it first */
-  if (!use_defaults)
-    return; /* all done */
-  if (var->initvalue) {
-    c = tor_malloc_zero(sizeof(config_line_t));
-    c->key = tor_strdup(var->name);
-    c->value = tor_strdup(var->initvalue);
-    if (config_assign_value(fmt, options, c, &msg) < 0) {
-      log_warn(LD_BUG, "Failed to assign default: %s", msg);
-      tor_free(msg); /* if this happens it's a bug */
-    }
-    config_free_lines(c);
-  }
-}
-
-/** Release storage held by <b>options</b>. */
-void
-config_free_(const config_format_t *fmt, void *options)
-{
-  int i;
-
-  if (!options)
-    return;
-
-  tor_assert(fmt);
-
-  for (i=0; fmt->vars[i].name; ++i)
-    config_clear(fmt, options, &(fmt->vars[i]));
-  if (fmt->extra) {
-    config_line_t **linep = STRUCT_VAR_P(options, fmt->extra->var_offset);
-    config_free_lines(*linep);
-    *linep = NULL;
-  }
-  tor_free(options);
-}
-
-/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
- * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
- */
-int
-config_is_same(const config_format_t *fmt,
-               const void *o1, const void *o2,
-               const char *name)
-{
-  config_line_t *c1, *c2;
-  int r = 1;
-  CONFIG_CHECK(fmt, o1);
-  CONFIG_CHECK(fmt, o2);
-
-  c1 = config_get_assigned_option(fmt, o1, name, 0);
-  c2 = config_get_assigned_option(fmt, o2, name, 0);
-  r = config_lines_eq(c1, c2);
-  config_free_lines(c1);
-  config_free_lines(c2);
-  return r;
-}
-
-/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
-void *
-config_dup(const config_format_t *fmt, const void *old)
-{
-  void *newopts;
-  int i;
-  config_line_t *line;
-
-  newopts = config_new(fmt);
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE)
-      continue;
-    line = config_get_assigned_option(fmt, old, fmt->vars[i].name, 0);
-    if (line) {
-      char *msg = NULL;
-      if (config_assign(fmt, newopts, line, 0, &msg) < 0) {
-        log_err(LD_BUG, "config_get_assigned_option() generated "
-                "something we couldn't config_assign(): %s", msg);
-        tor_free(msg);
-        tor_assert(0);
-      }
-    }
-    config_free_lines(line);
-  }
-  return newopts;
-}
-/** Set all vars in the configuration object <b>options</b> to their default
- * values. */
-void
-config_init(const config_format_t *fmt, void *options)
-{
-  int i;
-  const config_var_t *var;
-  CONFIG_CHECK(fmt, options);
-
-  for (i=0; fmt->vars[i].name; ++i) {
-    var = &fmt->vars[i];
-    if (!var->initvalue)
-      continue; /* defaults to NULL or 0 */
-    config_reset(fmt, options, var, 1);
-  }
-}
-
-/** Allocate and return a new string holding the written-out values of the vars
- * in 'options'.  If 'minimal', do not write out any default-valued vars.
- * Else, if comment_defaults, write default values as comments.
- */
-char *
-config_dump(const config_format_t *fmt, const void *default_options,
-            const void *options, int minimal,
-            int comment_defaults)
-{
-  smartlist_t *elements;
-  const void *defaults = default_options;
-  void *defaults_tmp = NULL;
-  config_line_t *line, *assigned;
-  char *result;
-  int i;
-  char *msg = NULL;
-
-  if (defaults == NULL) {
-    defaults = defaults_tmp = config_new(fmt);
-    config_init(fmt, defaults_tmp);
-  }
-
-  /* XXX use a 1 here so we don't add a new log line while dumping */
-  if (default_options == NULL) {
-    if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) {
-      log_err(LD_BUG, "Failed to validate default config: %s", msg);
-      tor_free(msg);
-      tor_assert(0);
-    }
-  }
-
-  elements = smartlist_new();
-  for (i=0; fmt->vars[i].name; ++i) {
-    int comment_option = 0;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE ||
-        fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    /* Don't save 'hidden' control variables. */
-    if (!strcmpstart(fmt->vars[i].name, "__"))
-      continue;
-    if (minimal && config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      continue;
-    else if (comment_defaults &&
-             config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      comment_option = 1;
-
-    line = assigned =
-      config_get_assigned_option(fmt, options, fmt->vars[i].name, 1);
-
-    for (; line; line = line->next) {
-      if (!strcmpstart(line->key, "__")) {
-        /* This check detects "hidden" variables inside LINELIST_V structures.
-         */
-        continue;
-      }
-      smartlist_add_asprintf(elements, "%s%s %s\n",
-                   comment_option ? "# " : "",
-                   line->key, line->value);
-    }
-    config_free_lines(assigned);
-  }
-
-  if (fmt->extra) {
-    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset);
-    for (; line; line = line->next) {
-      smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value);
-    }
-  }
-
-  result = smartlist_join_strings(elements, "", 0, NULL);
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-  if (defaults_tmp) {
-    fmt->free_fn(defaults_tmp);
-  }
-  return result;
-}
-
-/** Mapping from a unit name to a multiplier for converting that unit into a
- * base unit.  Used by config_parse_unit. */
-struct unit_table_t {
-  const char *unit; /**< The name of the unit */
-  uint64_t multiplier; /**< How many of the base unit appear in this unit */
-};
-
-/** Table to map the names of memory units to the number of bytes they
- * contain. */
-static struct unit_table_t memory_units[] = {
-  { "",          1 },
-  { "b",         1<< 0 },
-  { "byte",      1<< 0 },
-  { "bytes",     1<< 0 },
-  { "kb",        1<<10 },
-  { "kbyte",     1<<10 },
-  { "kbytes",    1<<10 },
-  { "kilobyte",  1<<10 },
-  { "kilobytes", 1<<10 },
-  { "kilobits",  1<<7  },
-  { "kilobit",   1<<7  },
-  { "kbits",     1<<7  },
-  { "kbit",      1<<7  },
-  { "m",         1<<20 },
-  { "mb",        1<<20 },
-  { "mbyte",     1<<20 },
-  { "mbytes",    1<<20 },
-  { "megabyte",  1<<20 },
-  { "megabytes", 1<<20 },
-  { "megabits",  1<<17 },
-  { "megabit",   1<<17 },
-  { "mbits",     1<<17 },
-  { "mbit",      1<<17 },
-  { "gb",        1<<30 },
-  { "gbyte",     1<<30 },
-  { "gbytes",    1<<30 },
-  { "gigabyte",  1<<30 },
-  { "gigabytes", 1<<30 },
-  { "gigabits",  1<<27 },
-  { "gigabit",   1<<27 },
-  { "gbits",     1<<27 },
-  { "gbit",      1<<27 },
-  { "tb",        UINT64_C(1)<<40 },
-  { "tbyte",     UINT64_C(1)<<40 },
-  { "tbytes",    UINT64_C(1)<<40 },
-  { "terabyte",  UINT64_C(1)<<40 },
-  { "terabytes", UINT64_C(1)<<40 },
-  { "terabits",  UINT64_C(1)<<37 },
-  { "terabit",   UINT64_C(1)<<37 },
-  { "tbits",     UINT64_C(1)<<37 },
-  { "tbit",      UINT64_C(1)<<37 },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of seconds they
- * contain. */
-static struct unit_table_t time_units[] = {
-  { "",         1 },
-  { "second",   1 },
-  { "seconds",  1 },
-  { "minute",   60 },
-  { "minutes",  60 },
-  { "hour",     60*60 },
-  { "hours",    60*60 },
-  { "day",      24*60*60 },
-  { "days",     24*60*60 },
-  { "week",     7*24*60*60 },
-  { "weeks",    7*24*60*60 },
-  { "month",    2629728, }, /* about 30.437 days */
-  { "months",   2629728, },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of milliseconds
- * they contain. */
-static struct unit_table_t time_msec_units[] = {
-  { "",         1 },
-  { "msec",     1 },
-  { "millisecond", 1 },
-  { "milliseconds", 1 },
-  { "second",   1000 },
-  { "seconds",  1000 },
-  { "minute",   60*1000 },
-  { "minutes",  60*1000 },
-  { "hour",     60*60*1000 },
-  { "hours",    60*60*1000 },
-  { "day",      24*60*60*1000 },
-  { "days",     24*60*60*1000 },
-  { "week",     7*24*60*60*1000 },
-  { "weeks",    7*24*60*60*1000 },
-  { NULL, 0 },
-};
-
-/** Parse a string <b>val</b> containing a number, zero or more
- * spaces, and an optional unit string.  If the unit appears in the
- * table <b>u</b>, then multiply the number by the unit multiplier.
- * On success, set *<b>ok</b> to 1 and return this product.
- * Otherwise, set *<b>ok</b> to 0.
- */
-static uint64_t
-config_parse_units(const char *val, struct unit_table_t *u, int *ok)
-{
-  uint64_t v = 0;
-  double d = 0;
-  int use_float = 0;
-  char *cp;
-
-  tor_assert(ok);
-
-  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
-  if (!*ok || (cp && *cp == '.')) {
-    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
-    if (!*ok)
-      goto done;
-    use_float = 1;
-  }
-
-  if (!cp) {
-    *ok = 1;
-    v = use_float ? ((uint64_t)d) :  v;
-    goto done;
-  }
-
-  cp = (char*) eat_whitespace(cp);
-
-  for ( ;u->unit;++u) {
-    if (!strcasecmp(u->unit, cp)) {
-      if (use_float)
-        v = (uint64_t)(u->multiplier * d);
-      else
-        v *= u->multiplier;
-      *ok = 1;
-      goto done;
-    }
-  }
-  log_warn(LD_CONFIG, "Unknown unit '%s'.", cp);
-  *ok = 0;
- done:
-
-  if (*ok)
-    return v;
-  else
-    return 0;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
- * and return the number of bytes specified.  Otherwise, set
- * *<b>ok</b> to false and return 0. */
-static uint64_t
-config_parse_memunit(const char *s, int *ok)
-{
-  uint64_t u = config_parse_units(s, memory_units, ok);
-  return u;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * time in milliseconds.  On success, set *<b>ok</b> to true and return
- * the number of milliseconds in the provided interval.  Otherwise, set
- * *<b>ok</b> to 0 and return -1. */
-static int
-config_parse_msec_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_msec_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of time.
- * On success, set *<b>ok</b> to true and return the number of seconds in
- * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
- */
-static int
-config_parse_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h
deleted file mode 100644
index 57f1ec1762..0000000000
--- a/src/app/config/confparse.h
+++ /dev/null
@@ -1,233 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.h
- *
- * \brief Header for confparse.c.
- */
-
-#ifndef TOR_CONFPARSE_H
-#define TOR_CONFPARSE_H
-
-/** Enumeration of types which option values can take */
-typedef enum config_type_t {
-  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
-  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
-  CONFIG_TYPE_UINT,         /**< A non-negative integer less than MAX_INT */
-  CONFIG_TYPE_INT,          /**< Any integer. */
-  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
-  CONFIG_TYPE_PORT,         /**< A port from 1...65535, 0 for "not set", or
-                             * "auto".  */
-  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
-  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
-                              * units */
-  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
-  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
-  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
-  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
-                             * 1 for true, and -1 for auto  */
-  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
-  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
-                              * optional whitespace. */
-  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
-                              * optional whitespace, representing intervals in
-                              * seconds, with optional units.  We allow
-                              * multiple values here for legacy reasons, but
-                              * ignore every value after the first. */
-  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
-  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
-                             * mixed with other keywords. */
-  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
-                             * context-sensitive config lines when fetching.
-                             */
-  CONFIG_TYPE_ROUTERSET,    /**< A list of router names, addrs, and fps,
-                             * parsed into a routerset_t. */
-  CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
-} config_type_t;
-
-#ifdef TOR_UNIT_TESTS
-/**
- * Union used when building in test mode typechecking the members of a type
- * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
- * it is used. */
-typedef union {
-  char **STRING;
-  char **FILENAME;
-  int *UINT; /* yes, really: Even though the confparse type is called
-              * "UINT", it still uses the C int type -- it just enforces that
-              * the values are in range [0,INT_MAX].
-              */
-  uint64_t *UINT64;
-  int *INT;
-  int *PORT;
-  int *INTERVAL;
-  int *MSEC_INTERVAL;
-  uint64_t *MEMUNIT;
-  double *DOUBLE;
-  int *BOOL;
-  int *AUTOBOOL;
-  time_t *ISOTIME;
-  smartlist_t **CSV;
-  int *CSV_INTERVAL;
-  struct config_line_t **LINELIST;
-  struct config_line_t **LINELIST_S;
-  struct config_line_t **LINELIST_V;
-  routerset_t **ROUTERSET;
-} confparse_dummy_values_t;
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** An abbreviation for a configuration option allowed on the command line. */
-typedef struct config_abbrev_t {
-  const char *abbreviated;
-  const char *full;
-  int commandline_only;
-  int warn;
-} config_abbrev_t;
-
-typedef struct config_deprecation_t {
-  const char *name;
-  const char *why_deprecated;
-} config_deprecation_t;
-
-/* Handy macro for declaring "In the config file or on the command line,
- * you can abbreviate <b>tok</b>s as <b>tok</b>". */
-#define PLURAL(tok) { #tok, #tok "s", 0, 0 }
-
-/** A variable allowed in the configuration file or on the command line. */
-typedef struct config_var_t {
-  const char *name; /**< The full keyword (case insensitive). */
-  config_type_t type; /**< How to interpret the type and turn it into a
-                       * value. */
-  off_t var_offset; /**< Offset of the corresponding member of or_options_t. */
-  const char *initvalue; /**< String (or null) describing initial value. */
-
-#ifdef TOR_UNIT_TESTS
-  /** Used for compiler-magic to typecheck the corresponding field in the
-   * corresponding struct. Only used in unit test mode, at compile-time. */
-  confparse_dummy_values_t var_ptr_dummy;
-#endif
-} config_var_t;
-
-/* Macros to define extra members inside config_var_t fields, and at the
- * end of a list of them.
- */
-#ifdef TOR_UNIT_TESTS
-/* This is a somewhat magic type-checking macro for users of confparse.c.
- * It initializes a union member "confparse_dummy_values_t.conftype" with
- * the address of a static member "tp_dummy.member".   This
- * will give a compiler warning unless the member field is of the correct
- * type.
- *
- * (This warning is mandatory, because a type mismatch here violates the type
- * compatibility constraint for simple assignment, and requires a diagnostic,
- * according to the C spec.)
- *
- * For example, suppose you say:
- *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
- * Then this macro will evaluate to:
- *     { .STRING = &or_options_t_dummy.Address }
- * And since confparse_dummy_values_t.STRING has type "char **", that
- * expression will create a warning unless or_options_t.Address also
- * has type "char *".
- */
-#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
-  { . conftype = &tp ## _dummy . member }
-#define CONF_TEST_MEMBERS(tp, conftype, member) \
-  , CONF_CHECK_VAR_TYPE(tp, conftype, member)
-#define END_OF_CONFIG_VARS                                      \
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } }
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  static tp tp ## _dummy
-#else /* !(defined(TOR_UNIT_TESTS)) */
-#define CONF_TEST_MEMBERS(tp, conftype, member)
-#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
-/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  struct tor_semicolon_eater
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Type of a callback to validate whether a given configuration is
- * well-formed and consistent. See options_trial_assign() for documentation
- * of arguments. */
-typedef int (*validate_fn_t)(void*,void*,void*,int,char**);
-
-/** Callback to free a configuration object. */
-typedef void (*free_cfg_fn_t)(void*);
-
-/** Information on the keys, value types, key-to-struct-member mappings,
- * variable descriptions, validation functions, and abbreviations for a
- * configuration or storage format. */
-typedef struct config_format_t {
-  size_t size; /**< Size of the struct that everything gets parsed into. */
-  uint32_t magic; /**< Required 'magic value' to make sure we have a struct
-                   * of the right type. */
-  off_t magic_offset; /**< Offset of the magic value within the struct. */
-  config_abbrev_t *abbrevs; /**< List of abbreviations that we expand when
-                             * parsing this format. */
-  const config_deprecation_t *deprecations; /** List of deprecated options */
-  config_var_t *vars; /**< List of variables we recognize, their default
-                       * values, and where we stick them in the structure. */
-  validate_fn_t validate_fn; /**< Function to validate config. */
-  free_cfg_fn_t free_fn; /**< Function to free the configuration. */
-  /** If present, extra is a LINELIST variable for unrecognized
-   * lines.  Otherwise, unrecognized lines are an error. */
-  config_var_t *extra;
-} config_format_t;
-
-/** Macro: assert that <b>cfg</b> has the right magic field for format
- * <b>fmt</b>. */
-#define CONFIG_CHECK(fmt, cfg) STMT_BEGIN                               \
-    tor_assert(fmt && cfg);                                             \
-    tor_assert((fmt)->magic ==                                          \
-               *(uint32_t*)STRUCT_VAR_P(cfg,fmt->magic_offset));        \
-  STMT_END
-
-#define CAL_USE_DEFAULTS      (1u<<0)
-#define CAL_CLEAR_FIRST       (1u<<1)
-#define CAL_WARN_DEPRECATIONS (1u<<2)
-
-void *config_new(const config_format_t *fmt);
-void config_free_(const config_format_t *fmt, void *options);
-#define config_free(fmt, options) do {                \
-    config_free_((fmt), (options));                   \
-    (options) = NULL;                                 \
-  } while (0)
-
-struct config_line_t *config_get_assigned_option(const config_format_t *fmt,
-                                          const void *options, const char *key,
-                                          int escape_val);
-int config_is_same(const config_format_t *fmt,
-                   const void *o1, const void *o2,
-                   const char *name);
-void config_init(const config_format_t *fmt, void *options);
-void *config_dup(const config_format_t *fmt, const void *old);
-char *config_dump(const config_format_t *fmt, const void *default_options,
-                  const void *options, int minimal,
-                  int comment_defaults);
-int config_assign(const config_format_t *fmt, void *options,
-                  struct config_line_t *list,
-                  unsigned flags, char **msg);
-config_var_t *config_find_option_mutable(config_format_t *fmt,
-                                         const char *key);
-const char *config_find_deprecation(const config_format_t *fmt,
-                                     const char *key);
-const config_var_t *config_find_option(const config_format_t *fmt,
-                                       const char *key);
-const char *config_expand_abbrev(const config_format_t *fmt,
-                                 const char *option,
-                                 int command_line, int warn_obsolete);
-void warn_deprecated_option(const char *what, const char *why);
-
-/* Helper macros to compare an option across two configuration objects */
-#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
-#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
-#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
-#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
-
-#endif /* !defined(TOR_CONFPARSE_H) */
diff --git a/src/app/config/include.am b/src/app/config/include.am
new file mode 100644
index 0000000000..14320a6b11
--- /dev/null
+++ b/src/app/config/include.am
@@ -0,0 +1,23 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/app/config/config.c			\
+	src/app/config/quiet_level.c		\
+	src/app/config/resolve_addr.c		\
+	src/app/config/statefile.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/app/config/config.h				\
+	src/app/config/or_options_st.h			\
+	src/app/config/or_state_st.h			\
+	src/app/config/quiet_level.h			\
+	src/app/config/resolve_addr.h			\
+	src/app/config/statefile.h			\
+	src/app/config/tor_cmdline_mode.h
+
+
+noinst_HEADERS +=				\
+	src/app/config/auth_dirs.inc		\
+	src/app/config/fallback_dirs.inc	\
+	src/app/config/testnet.inc
diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h
index 74d2fefa16..bf58205f89 100644
--- a/src/app/config/or_options_st.h
+++ b/src/app/config/or_options_st.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,11 +13,15 @@
 #ifndef TOR_OR_OPTIONS_ST_H
 #define TOR_OR_OPTIONS_ST_H
 
+#include "core/or/or.h"
 #include "lib/cc/torint.h"
 #include "lib/net/address.h"
+#include "app/config/tor_cmdline_mode.h"
 
 struct smartlist_t;
 struct config_line_t;
+struct config_suite_t;
+struct routerset_t;
 
 /** Enumeration of outbound address configuration types:
  * Exit-only, OR-only, or both */
@@ -25,17 +29,18 @@ typedef enum {OUTBOUND_ADDR_EXIT, OUTBOUND_ADDR_OR,
               OUTBOUND_ADDR_EXIT_AND_OR,
               OUTBOUND_ADDR_MAX} outbound_addr_t;
 
+/** Which protocol to use for TCPProxy. */
+typedef enum {
+  /** Use the HAProxy proxy protocol. */
+  TCP_PROXY_PROTOCOL_HAPROXY
+} tcp_proxy_protocol_t;
+
 /** Configuration options for a Tor process. */
 struct or_options_t {
   uint32_t magic_;
 
   /** What should the tor process actually do? */
-  enum {
-    CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD,
-    CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG,
-    CMD_KEYGEN,
-    CMD_KEY_EXPIRATION,
-  } command;
+  tor_cmdline_mode_t command;
   char *command_arg; /**< Argument for command-line option. */
 
   struct config_line_t *Logs; /**< New-style list of configuration lines
@@ -69,25 +74,29 @@ struct or_options_t {
   char *Address; /**< OR only: configured address for this onion router. */
   char *PidFile; /**< Where to store PID of Tor process. */
 
-  routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
+  struct routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as exits. */
-  routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
+  struct routerset_t *MiddleNodes; /**< Structure containing nicknames,
+                             * digests, country codes and IP address patterns
+                             * of ORs to consider as middles. */
+  struct routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as entry points. */
   int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes
                     * are up, or we need to access a node in ExcludeNodes,
                     * do we just fail instead? */
-  routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests,
-                             * country codes and IP address patterns of ORs
-                             * not to use in circuits. But see StrictNodes
-                             * above. */
-  routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests,
-                                 * country codes and IP address patterns of
-                                 * ORs not to consider as exits. */
+  struct routerset_t *ExcludeNodes;/**< Structure containing nicknames,
+                             * digests, country codes and IP address patterns
+                             * of ORs not to use in circuits. But see
+                             * StrictNodes above. */
+  struct routerset_t *ExcludeExitNodes;/**< Structure containing nicknames,
+                                 * digests, country codes and IP address
+                                 * patterns of ORs not to consider as
+                                 * exits. */
 
   /** Union of ExcludeNodes and ExcludeExitNodes */
-  routerset_t *ExcludeExitNodesUnion_;
+  struct routerset_t *ExcludeExitNodesUnion_;
 
   int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our
                        * process for all current and future memory. */
@@ -113,12 +122,6 @@ struct or_options_t {
    * [][0] is IPv4, [][1] is IPv6
    */
   tor_addr_t OutboundBindAddresses[OUTBOUND_ADDR_MAX][2];
-  /** Directory server only: which versions of
-   * Tor should we tell users to run? */
-  struct config_line_t *RecommendedVersions;
-  struct config_line_t *RecommendedClientVersions;
-  struct config_line_t *RecommendedServerVersions;
-  struct config_line_t *RecommendedPackages;
   /** Whether dirservers allow router descriptors with private IPs. */
   int DirAllowPrivateAddresses;
   /** Whether routers accept EXTEND cells to routers with private IPs. */
@@ -193,9 +196,6 @@ struct or_options_t {
   int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
   int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory
                            * for version 3 directories? */
-  int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative
-                                   * directory that's willing to recommend
-                                   * versions? */
   int BridgeAuthoritativeDir; /**< Boolean: is this an authoritative directory
                                * that aggregates bridge descriptors? */
 
@@ -245,6 +245,17 @@ struct or_options_t {
    * pad to the server regardless of server support. */
   int ConnectionPadding;
 
+  /** Boolean: if true, then circuit padding will be negotiated by client
+   * and server, subject to consenus limits (default). If 0, it will be fully
+   * disabled. */
+  int CircuitPadding;
+
+  /** Boolean: if true, then this client will only use circuit padding
+   * algorithms that are known to use a low amount of overhead. If false,
+   * we will use all available circuit padding algorithms.
+   */
+  int ReducedCircuitPadding;
+
   /** To what authority types do we publish our descriptor? Choices are
    * "v1", "v2", "v3", "bridge", or "". */
   struct smartlist_t *PublishServerDescriptor;
@@ -255,20 +266,17 @@ struct or_options_t {
   int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */
   int FetchHidServDescriptors; /**< and hidden service descriptors? */
 
-  int MinUptimeHidServDirectoryV2; /**< As directory authority, accept hidden
-                                    * service directories after what time? */
-
   int FetchUselessDescriptors; /**< Do we fetch non-running descriptors too? */
   int AllDirActionsPrivate; /**< Should every directory action be sent
                              * through a Tor circuit? */
 
   /** A routerset that should be used when picking middle nodes for HS
    *  circuits. */
-  routerset_t *HSLayer2Nodes;
+  struct routerset_t *HSLayer2Nodes;
 
   /** A routerset that should be used when picking third-hop nodes for HS
    *  circuits. */
-  routerset_t *HSLayer3Nodes;
+  struct routerset_t *HSLayer3Nodes;
 
   /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct)
    * circuits between the onion service server, and the introduction and
@@ -409,6 +417,11 @@ struct or_options_t {
   char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */
   char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */
 
+  char *TCPProxy; /**< protocol and hostname:port to use as a proxy, if any. */
+  tcp_proxy_protocol_t TCPProxyProtocol; /**< Derived from TCPProxy. */
+  tor_addr_t TCPProxyAddr; /**< Derived from TCPProxy. */
+  uint16_t TCPProxyPort; /**< Derived from TCPProxy. */
+
   /** List of configuration lines for replacement directory authorities.
    * If you just want to replace one class of authority at a time,
    * use the "Alternate*Authority" options below instead. */
@@ -455,21 +468,6 @@ struct or_options_t {
   struct smartlist_t *AuthDirRejectCCs;
   /**@}*/
 
-  int AuthDirListBadExits; /**< True iff we should list bad exits,
-                            * and vote for all other exits as good. */
-  int AuthDirMaxServersPerAddr; /**< Do not permit more than this
-                                 * number of servers per IP address. */
-  int AuthDirHasIPv6Connectivity; /**< Boolean: are we on IPv6?  */
-  int AuthDirPinKeys; /**< Boolean: Do we enforce key-pinning? */
-
-  /** If non-zero, always vote the Fast flag for any relay advertising
-   * this amount of capacity or more. */
-  uint64_t AuthDirFastGuarantee;
-
-  /** If non-zero, this advertised capacity or more is always sufficient
-   * to satisfy the bandwidth requirement for the Guard flag. */
-  uint64_t AuthDirGuardBWGuarantee;
-
   char *AccountingStart; /**< How long is the accounting interval, and when
                           * does it start? */
   uint64_t AccountingMax; /**< How many bytes do we allow per accounting
@@ -526,12 +524,8 @@ struct or_options_t {
                          * protocol, is it a warn or an info in our logs? */
   int TestSocks; /**< Boolean: when we get a socks connection, do we loudly
                   * log whether it was DNS-leaking or not? */
-  int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware
-                      * acceleration where available? */
   /** Token Bucket Refill resolution in milliseconds. */
   int TokenBucketRefillInterval;
-  char *AccelName; /**< Optional hardware acceleration engine name. */
-  char *AccelDir; /**< Optional hardware acceleration engine search dir. */
 
   /** Boolean: Do we try to enter from a smallish number
    * of fixed nodes? */
@@ -563,7 +557,9 @@ struct or_options_t {
 
   int DirCache; /**< Cache all directory documents and accept requests via
                  * tunnelled dir conns from clients. If 1, enabled (default);
-                 * If 0, disabled. */
+                 * If 0, disabled. Use dir_server_mode() rather than
+                 * referencing this option directly. (Except for routermode
+                 * and relay_config, which do direct checks.) */
 
   char *VirtualAddrNetworkIPv4; /**< Address and mask to hand out for virtual
                                  * MAPADDRESS requests for IPv4 addresses */
@@ -685,14 +681,6 @@ struct or_options_t {
   /** Location of guardfraction file */
   char *GuardfractionFile;
 
-  /** Authority only: key=value pairs that we add to our networkstatus
-   * consensus vote on the 'params' line. */
-  char *ConsensusParams;
-
-  /** Authority only: minimum number of measured bandwidths we must see
-   * before we only believe measured bandwidths to assign flags. */
-  int MinMeasuredBWsForAuthToIgnoreAdvertised;
-
   /** The length of time that we think an initial consensus should be fresh.
    * Only altered on testing networks. */
   int TestingV3AuthInitialVotingInterval;
@@ -709,16 +697,6 @@ struct or_options_t {
       voting. Only altered on testing networks. */
   int TestingV3AuthVotingStartOffset;
 
-  /** If an authority has been around for less than this amount of time, it
-   * does not believe its reachability information is accurate.  Only
-   * altered on testing networks. */
-  int TestingAuthDirTimeToLearnReachability;
-
-  /** Clients don't download any descriptor this recent, since it will
-   * probably not have propagated to enough caches.  Only altered on testing
-   * networks. */
-  int TestingEstimatedDescriptorPropagationTime;
-
   /** Schedule for when servers should download things in general.  Only
    * altered on testing networks. */
   int TestingServerDownloadInitialDelay;
@@ -792,27 +770,6 @@ struct or_options_t {
    * of certain configuration options. */
   int TestingTorNetwork;
 
-  /** Minimum value for the Exit flag threshold on testing networks. */
-  uint64_t TestingMinExitFlagThreshold;
-
-  /** Minimum value for the Fast flag threshold on testing networks. */
-  uint64_t TestingMinFastFlagThreshold;
-
-  /** Relays in a testing network which should be voted Exit
-   * regardless of exit policy. */
-  routerset_t *TestingDirAuthVoteExit;
-  int TestingDirAuthVoteExitIsStrict;
-
-  /** Relays in a testing network which should be voted Guard
-   * regardless of uptime and bandwidth. */
-  routerset_t *TestingDirAuthVoteGuard;
-  int TestingDirAuthVoteGuardIsStrict;
-
-  /** Relays in a testing network which should be voted HSDir
-   * regardless of uptime and DirPort. */
-  routerset_t *TestingDirAuthVoteHSDir;
-  int TestingDirAuthVoteHSDirIsStrict;
-
   /** Enable CONN_BW events.  Only altered on testing networks. */
   int TestingEnableConnBwEvent;
 
@@ -837,7 +794,7 @@ struct or_options_t {
    * to make this false. */
   int ReloadTorrcOnSIGHUP;
 
-  /* The main parameter for picking circuits within a connection.
+  /** The main parameter for picking circuits within a connection.
    *
    * If this value is positive, when picking a cell to relay on a connection,
    * we always relay from the circuit whose weighted cell count is lowest.
@@ -991,12 +948,6 @@ struct or_options_t {
    */
   uint64_t MaxUnparseableDescSizeToLog;
 
-  /** Bool (default: 1): Switch for the shared random protocol. Only
-   * relevant to a directory authority. If off, the authority won't
-   * participate in the protocol. If on (default), a flag is added to the
-   * vote indicating participation. */
-  int AuthDirSharedRandomness;
-
   /** If 1, we skip all OOS checks. */
   int DisableOOSCheck;
 
@@ -1004,11 +955,6 @@ struct or_options_t {
    * If -1, we should do whatever the consensus parameter says. */
   int ExtendByEd25519ID;
 
-  /** Bool (default: 1): When testing routerinfos as a directory authority,
-   * do we enforce Ed25519 identity match? */
-  /* NOTE: remove this option someday. */
-  int AuthDirTestEd25519LinkKeys;
-
   /** Bool (default: 0): Tells if a %include was used on torrc */
   int IncludeUsed;
 
@@ -1033,7 +979,7 @@ struct or_options_t {
   /** The list of scheduler type string ordered by priority that is first one
    * has to be tried first. Default: KIST,KISTLite,Vanilla */
   struct smartlist_t *Schedulers;
-  /* An ordered list of scheduler_types mapped from Schedulers. */
+  /** An ordered list of scheduler_types mapped from Schedulers. */
   struct smartlist_t *SchedulerTypes_;
 
   /** List of files that were opened by %include in torrc and torrc-defaults */
@@ -1072,6 +1018,33 @@ struct or_options_t {
 
   /** Autobool: Do we refuse single hop client rendezvous? */
   int DoSRefuseSingleHopClientRendezvous;
+
+  /** Interval: how long without activity does it take for a client
+   * to become dormant?
+   **/
+  int DormantClientTimeout;
+
+  /** Boolean: true if having an idle stream is sufficient to prevent a client
+   * from becoming dormant.
+   **/
+  int DormantTimeoutDisabledByIdleStreams;
+
+  /** Boolean: true if Tor should be dormant the first time it starts with
+   * a datadirectory; false otherwise. */
+  int DormantOnFirstStartup;
+  /**
+   * Boolean: true if Tor should treat every startup event as cancelling
+   * a possible previous dormant state.
+   **/
+  int DormantCanceledByStartup;
+
+  /**
+   * Configuration objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant configuration object.
+   */
+  struct config_suite_t *subconfigs_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_OPTIONS_ST_H) */
diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h
index 5f8214d146..8c4e9d5e61 100644
--- a/src/app/config/or_state_st.h
+++ b/src/app/config/or_state_st.h
@@ -1,11 +1,11 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file or_state_t
+ * \file or_state_st.h
  *
  * \brief The or_state_t structure, which represents Tor's state file.
  */
@@ -15,6 +15,7 @@
 
 #include "lib/cc/torint.h"
 struct smartlist_t;
+struct config_suite_t;
 
 /** Persistent state for an onion router, as saved to disk. */
 struct or_state_t {
@@ -87,6 +88,14 @@ struct or_state_t {
 
   /** When did we last rotate our onion key?  "0" for 'no idea'. */
   time_t LastRotatedOnionKey;
+
+  /**
+   * State objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant state object if you must.
+   */
+  struct config_suite_t *substates_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_STATE_ST_H) */
diff --git a/src/app/config/quiet_level.c b/src/app/config/quiet_level.c
new file mode 100644
index 0000000000..e04faaef3a
--- /dev/null
+++ b/src/app/config/quiet_level.c
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file quiet_level.c
+ * @brief Code to handle default logging level (quiet/hush/normal).
+ **/
+
+#include "orconfig.h"
+#include "lib/log/log.h"
+#include "app/config/quiet_level.h"
+
+/** Decides our behavior when no logs are configured/before any logs have been
+ * configured.  For QUIET_NONE, we log notice to stdout as normal.  For
+ * QUIET_HUSH, we log warnings only.  For QUIET_SILENT, we log nothing.
+ */
+quiet_level_t quiet_level = 0;
+
+/** Add a default log (or not), depending on the value of <b>quiet</b>. */
+void
+add_default_log_for_quiet_level(quiet_level_t quiet)
+{
+  switch (quiet) {
+    case QUIET_SILENT:
+      /* --quiet: no initial logging */
+      return;
+    case QUIET_HUSH:
+    /* --hush: log at warning or higher. */
+      add_default_log(LOG_WARN);
+      break;
+    case QUIET_NONE: FALLTHROUGH;
+    default:
+      add_default_log(LOG_NOTICE);
+  }
+}
diff --git a/src/app/config/quiet_level.h b/src/app/config/quiet_level.h
new file mode 100644
index 0000000000..3a630b90e7
--- /dev/null
+++ b/src/app/config/quiet_level.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file quiet_level.h
+ * \brief Declare the quiet_level enumeration and global.
+ **/
+
+#ifndef QUIET_LEVEL_H
+#define QUIET_LEVEL_H
+
+/** Enumeration to define how quietly Tor should log at startup. */
+typedef enum {
+   /** Default quiet level: we log everything of level NOTICE or higher. */
+   QUIET_NONE = 0,
+   /** "--hush" quiet level: we log everything of level WARNING or higher. */
+   QUIET_HUSH = 1 ,
+   /** "--quiet" quiet level: we log nothing at all. */
+   QUIET_SILENT = 2
+} quiet_level_t;
+
+/** How quietly should Tor log at startup? */
+extern quiet_level_t quiet_level;
+
+void add_default_log_for_quiet_level(quiet_level_t quiet);
+
+#endif /* !defined(QUIET_LEVEL_H) */
diff --git a/src/app/config/resolve_addr.c b/src/app/config/resolve_addr.c
new file mode 100644
index 0000000000..9d1a8e0260
--- /dev/null
+++ b/src/app/config/resolve_addr.c
@@ -0,0 +1,314 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file resolve_addr.c
+ * \brief Implement resolving address functions
+ **/
+
+#define RESOLVE_ADDR_PRIVATE
+
+#include "app/config/config.h"
+#include "app/config/resolve_addr.h"
+
+#include "core/mainloop/mainloop.h"
+
+#include "feature/control/control_events.h"
+
+#include "lib/net/gethostname.h"
+#include "lib/net/resolve.h"
+
+/** Last value actually set by resolve_my_address. */
+static uint32_t last_resolved_addr = 0;
+
+/** Accessor for last_resolved_addr from outside this file. */
+uint32_t
+get_last_resolved_addr(void)
+{
+  return last_resolved_addr;
+}
+
+/** Reset last_resolved_addr from outside this file. */
+void
+reset_last_resolved_addr(void)
+{
+  last_resolved_addr = 0;
+}
+
+/**
+ * Attempt getting our non-local (as judged by tor_addr_is_internal()
+ * function) IP address using following techniques, listed in
+ * order from best (most desirable, try first) to worst (least
+ * desirable, try if everything else fails).
+ *
+ * First, attempt using <b>options-\>Address</b> to get our
+ * non-local IP address.
+ *
+ * If <b>options-\>Address</b> represents a non-local IP address,
+ * consider it ours.
+ *
+ * If <b>options-\>Address</b> is a DNS name that resolves to
+ * a non-local IP address, consider this IP address ours.
+ *
+ * If <b>options-\>Address</b> is NULL, fall back to getting local
+ * hostname and using it in above-described ways to try and
+ * get our IP address.
+ *
+ * In case local hostname cannot be resolved to a non-local IP
+ * address, try getting an IP address of network interface
+ * in hopes it will be non-local one.
+ *
+ * Fail if one or more of the following is true:
+ *   - DNS name in <b>options-\>Address</b> cannot be resolved.
+ *   - <b>options-\>Address</b> is a local host address.
+ *   - Attempt at getting local hostname fails.
+ *   - Attempt at getting network interface address fails.
+ *
+ * Return 0 if all is well, or -1 if we can't find a suitable
+ * public IP address.
+ *
+ * If we are returning 0:
+ *   - Put our public IP address (in host order) into *<b>addr_out</b>.
+ *   - If <b>method_out</b> is non-NULL, set *<b>method_out</b> to a static
+ *     string describing how we arrived at our answer.
+ *      - "CONFIGURED" - parsed from IP address string in
+ *        <b>options-\>Address</b>
+ *      - "RESOLVED" - resolved from DNS name in <b>options-\>Address</b>
+ *      - "GETHOSTNAME" - resolved from a local hostname.
+ *      - "INTERFACE" - retrieved from a network interface.
+ *   - If <b>hostname_out</b> is non-NULL, and we resolved a hostname to
+ *     get our address, set *<b>hostname_out</b> to a newly allocated string
+ *     holding that hostname. (If we didn't get our address by resolving a
+ *     hostname, set *<b>hostname_out</b> to NULL.)
+ *
+ * XXXX ipv6
+ */
+int
+resolve_my_address(int warn_severity, const or_options_t *options,
+                   uint32_t *addr_out,
+                   const char **method_out, char **hostname_out)
+{
+  struct in_addr in;
+  uint32_t addr; /* host order */
+  char hostname[256];
+  const char *method_used;
+  const char *hostname_used;
+  int explicit_ip=1;
+  int explicit_hostname=1;
+  int from_interface=0;
+  char *addr_string = NULL;
+  const char *address = options->Address;
+  int notice_severity = warn_severity <= LOG_NOTICE ?
+    LOG_NOTICE : warn_severity;
+
+  tor_addr_t myaddr;
+  tor_assert(addr_out);
+
+  /*
+   * Step one: Fill in 'hostname' to be our best guess.
+   */
+
+  if (address && *address) {
+    strlcpy(hostname, address, sizeof(hostname));
+    log_debug(LD_CONFIG, "Trying configured Address '%s' as local hostname",
+              hostname);
+  } else { /* then we need to guess our address */
+    explicit_ip = 0; /* it's implicit */
+    explicit_hostname = 0; /* it's implicit */
+
+    if (tor_gethostname(hostname, sizeof(hostname)) < 0) {
+      log_fn(warn_severity, LD_NET,"Error obtaining local hostname");
+      return -1;
+    }
+    log_debug(LD_CONFIG, "Guessed local host name as '%s'", hostname);
+  }
+
+  /*
+   * Step two: Now that we know 'hostname', parse it or resolve it. If
+   * it doesn't parse or resolve, look at the interface address. Set 'addr'
+   * to be our (host-order) 32-bit answer.
+   */
+
+  if (tor_inet_aton(hostname, &in) == 0) {
+    /* then we have to resolve it */
+    log_debug(LD_CONFIG, "Local hostname '%s' is DNS address. "
+              "Trying to resolve to IP address.", hostname);
+    explicit_ip = 0;
+    if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */
+      uint32_t interface_ip; /* host order */
+
+      if (explicit_hostname) {
+        log_fn(warn_severity, LD_CONFIG,
+               "Could not resolve local Address '%s'. Failing.", hostname);
+        return -1;
+      }
+      log_fn(notice_severity, LD_CONFIG,
+             "Could not resolve guessed local hostname '%s'. "
+             "Trying something else.", hostname);
+      if (get_interface_address(warn_severity, &interface_ip)) {
+        log_fn(warn_severity, LD_CONFIG,
+               "Could not get local interface IP address. Failing.");
+        return -1;
+      }
+      from_interface = 1;
+      addr = interface_ip;
+      log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for "
+             "local interface. Using that.", fmt_addr32(addr));
+      strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
+    } else { /* resolved hostname into addr */
+      tor_addr_from_ipv4h(&myaddr, addr);
+
+      if (!explicit_hostname &&
+          tor_addr_is_internal(&myaddr, 0)) {
+        tor_addr_t interface_ip;
+
+        log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' "
+               "resolves to a private IP address (%s). Trying something "
+               "else.", hostname, fmt_addr32(addr));
+
+        if (get_interface_address6(warn_severity, AF_INET, &interface_ip)<0) {
+          log_fn(warn_severity, LD_CONFIG,
+                 "Could not get local interface IP address. Too bad.");
+        } else if (tor_addr_is_internal(&interface_ip, 0)) {
+          log_fn(notice_severity, LD_CONFIG,
+                 "Interface IP address '%s' is a private address too. "
+                 "Ignoring.", fmt_addr(&interface_ip));
+        } else {
+          from_interface = 1;
+          addr = tor_addr_to_ipv4h(&interface_ip);
+          log_fn(notice_severity, LD_CONFIG,
+                 "Learned IP address '%s' for local interface."
+                 " Using that.", fmt_addr32(addr));
+          strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
+        }
+      }
+    }
+  } else {
+    log_debug(LD_CONFIG, "Local hostname '%s' is already IP address, "
+              "skipping DNS resolution", hostname);
+    addr = ntohl(in.s_addr); /* set addr so that addr_string is not
+                              * illformed */
+  }
+
+  /*
+   * Step three: Check whether 'addr' is an internal IP address, and error
+   * out if it is and we don't want that.
+   */
+
+  tor_addr_from_ipv4h(&myaddr,addr);
+
+  addr_string = tor_dup_ip(addr);
+  if (addr_string && tor_addr_is_internal(&myaddr, 0)) {
+    /* make sure we're ok with publishing an internal IP */
+    if (using_default_dir_authorities(options)) {
+      /* if they are using the default authorities, disallow internal IPs
+       * always. For IPv6 ORPorts, this check is done in
+       * router_get_advertised_ipv6_or_ap(). See #33681. */
+      log_fn(warn_severity, LD_CONFIG,
+             "Address '%s' resolves to private IP address '%s'. "
+             "Tor servers that use the default DirAuthorities must have "
+             "public IP addresses.", hostname, addr_string);
+      tor_free(addr_string);
+      return -1;
+    }
+    if (!explicit_ip) {
+      /* even if they've set their own authorities, require an explicit IP if
+       * they're using an internal address. */
+      log_fn(warn_severity, LD_CONFIG, "Address '%s' resolves to private "
+             "IP address '%s'. Please set the Address config option to be "
+             "the IP address you want to use.", hostname, addr_string);
+      tor_free(addr_string);
+      return -1;
+    }
+  }
+
+  /*
+   * Step four: We have a winner! 'addr' is our answer for sure, and
+   * 'addr_string' is its string form. Fill out the various fields to
+   * say how we decided it.
+   */
+
+  log_debug(LD_CONFIG, "Resolved Address to '%s'.", addr_string);
+
+  if (explicit_ip) {
+    method_used = "CONFIGURED";
+    hostname_used = NULL;
+  } else if (explicit_hostname) {
+    method_used = "RESOLVED";
+    hostname_used = hostname;
+  } else if (from_interface) {
+    method_used = "INTERFACE";
+    hostname_used = NULL;
+  } else {
+    method_used = "GETHOSTNAME";
+    hostname_used = hostname;
+  }
+
+  *addr_out = addr;
+  if (method_out)
+    *method_out = method_used;
+  if (hostname_out)
+    *hostname_out = hostname_used ? tor_strdup(hostname_used) : NULL;
+
+  /*
+   * Step five: Check if the answer has changed since last time (or if
+   * there was no last time), and if so call various functions to keep
+   * us up-to-date.
+   */
+
+  if (last_resolved_addr && last_resolved_addr != *addr_out) {
+    /* Leave this as a notice, regardless of the requested severity,
+     * at least until dynamic IP address support becomes bulletproof. */
+    log_notice(LD_NET,
+               "Your IP address seems to have changed to %s "
+               "(METHOD=%s%s%s). Updating.",
+               addr_string, method_used,
+               hostname_used ? " HOSTNAME=" : "",
+               hostname_used ? hostname_used : "");
+    ip_address_changed(0);
+  }
+
+  if (last_resolved_addr != *addr_out) {
+    control_event_server_status(LOG_NOTICE,
+                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=%s%s%s",
+                                addr_string, method_used,
+                                hostname_used ? " HOSTNAME=" : "",
+                                hostname_used ? hostname_used : "");
+  }
+  last_resolved_addr = *addr_out;
+
+  /*
+   * And finally, clean up and return success.
+   */
+
+  tor_free(addr_string);
+  return 0;
+}
+
+/** Return true iff <b>addr</b> is judged to be on the same network as us, or
+ * on a private network.
+ */
+MOCK_IMPL(int,
+is_local_addr, (const tor_addr_t *addr))
+{
+  if (tor_addr_is_internal(addr, 0))
+    return 1;
+  /* Check whether ip is on the same /24 as we are. */
+  if (get_options()->EnforceDistinctSubnets == 0)
+    return 0;
+  if (tor_addr_family(addr) == AF_INET) {
+    uint32_t ip = tor_addr_to_ipv4h(addr);
+
+    /* It's possible that this next check will hit before the first time
+     * resolve_my_address actually succeeds.  (For clients, it is likely that
+     * resolve_my_address will never be called at all).  In those cases,
+     * last_resolved_addr will be 0, and so checking to see whether ip is on
+     * the same /24 as last_resolved_addr will be the same as checking whether
+     * it was on net 0, which is already done by tor_addr_is_internal.
+     */
+    if ((last_resolved_addr & (uint32_t)0xffffff00ul)
+        == (ip & (uint32_t)0xffffff00ul))
+      return 1;
+  }
+  return 0;
+}
diff --git a/src/app/config/resolve_addr.h b/src/app/config/resolve_addr.h
new file mode 100644
index 0000000000..3747546402
--- /dev/null
+++ b/src/app/config/resolve_addr.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file resolve_addr.h
+ * \brief Header file for resolve_addr.c.
+ **/
+
+#ifndef TOR_CONFIG_RESOLVE_ADDR_H
+#define TOR_CONFIG_RESOLVE_ADDR_H
+
+#include "app/config/or_options_st.h"
+
+int resolve_my_address(int warn_severity, const or_options_t *options,
+                       uint32_t *addr_out,
+                       const char **method_out, char **hostname_out);
+
+uint32_t get_last_resolved_addr(void);
+void reset_last_resolved_addr(void);
+
+MOCK_DECL(int, is_local_addr, (const tor_addr_t *addr));
+
+#ifdef RESOLVE_ADDR_PRIVATE
+
+#endif /* RESOLVE_ADDR_PRIVATE */
+
+#endif /* TOR_CONFIG_RESOLVE_ADDR_H */
+
diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c
index 89039a05b5..dcc55f1898 100644
--- a/src/app/config/statefile.c
+++ b/src/app/config/statefile.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
  *
  * This 'state' file is a typed key-value store that allows multiple
  * entries for the same key.  It follows the same metaformat as described
- * in confparse.c, and uses the same code to read and write itself.
+ * in confmgt.c, and uses the same code to read and write itself.
  *
  * The state file is most suitable for small values that don't change too
  * frequently.  For values that become very large, we typically use a separate
@@ -32,10 +32,12 @@
 #include "core/or/or.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "feature/relay/transport_config.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/stats/rephist.h"
@@ -43,8 +45,10 @@
 #include "feature/relay/routermode.h"
 #include "lib/sandbox/sandbox.h"
 #include "app/config/statefile.h"
+#include "app/main/subsysmgr.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
@@ -68,16 +72,14 @@ static config_abbrev_t state_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_state_t);
 
-/*XXXX these next two are duplicates or near-duplicates from config.c */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member),       \
-      initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
 
 /** Array of "state" variables saved to the ~/.tor/state file. */
-static config_var_t state_vars_[] = {
+// clang-format off
+static const config_var_t state_vars_[] = {
   /* Remember to document these in state-contents.txt ! */
 
   V(AccountingBytesReadInInterval,    MEMUNIT,  NULL),
@@ -103,19 +105,19 @@ static config_var_t state_vars_[] = {
   V(HidServRevCounter,            LINELIST, NULL),
 
   V(BWHistoryReadEnds,                ISOTIME,  NULL),
-  V(BWHistoryReadInterval,            UINT,     "900"),
+  V(BWHistoryReadInterval,            POSINT,     "900"),
   V(BWHistoryReadValues,              CSV,      ""),
   V(BWHistoryReadMaxima,              CSV,      ""),
   V(BWHistoryWriteEnds,               ISOTIME,  NULL),
-  V(BWHistoryWriteInterval,           UINT,     "900"),
+  V(BWHistoryWriteInterval,           POSINT,     "900"),
   V(BWHistoryWriteValues,             CSV,      ""),
   V(BWHistoryWriteMaxima,             CSV,      ""),
   V(BWHistoryDirReadEnds,             ISOTIME,  NULL),
-  V(BWHistoryDirReadInterval,         UINT,     "900"),
+  V(BWHistoryDirReadInterval,         POSINT,     "900"),
   V(BWHistoryDirReadValues,           CSV,      ""),
   V(BWHistoryDirReadMaxima,           CSV,      ""),
   V(BWHistoryDirWriteEnds,            ISOTIME,  NULL),
-  V(BWHistoryDirWriteInterval,        UINT,     "900"),
+  V(BWHistoryDirWriteInterval,        POSINT,     "900"),
   V(BWHistoryDirWriteValues,          CSV,      ""),
   V(BWHistoryDirWriteMaxima,          CSV,      ""),
 
@@ -126,48 +128,70 @@ static config_var_t state_vars_[] = {
   V(LastRotatedOnionKey,              ISOTIME,  NULL),
   V(LastWritten,                      ISOTIME,  NULL),
 
-  V(TotalBuildTimes,                  UINT,     NULL),
-  V(CircuitBuildAbandonedCount,       UINT,     "0"),
+  V(TotalBuildTimes,                  POSINT,     NULL),
+  V(CircuitBuildAbandonedCount,       POSINT,     "0"),
   VAR("CircuitBuildTimeBin",          LINELIST_S, BuildtimeHistogram, NULL),
   VAR("BuildtimeHistogram",           LINELIST_V, BuildtimeHistogram, NULL),
 
   END_OF_CONFIG_VARS
 };
+// clang-format on
 
 #undef VAR
 #undef V
 
 static int or_state_validate(or_state_t *state, char **msg);
 
-static int or_state_validate_cb(void *old_options, void *options,
-                                void *default_options,
-                                int from_setconf, char **msg);
-
-static void or_state_free_cb(void *state);
+static int or_state_validate_cb(const void *old_options,
+                                void *options, char **msg);
 
 /** Magic value for or_state_t. */
 #define OR_STATE_MAGIC 0x57A73f57
 
 /** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines)
+static struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(or_state_t, ExtraLines),
 };
 
 /** Configuration format for or_state_t. */
 static const config_format_t state_format = {
-  sizeof(or_state_t),
-  OR_STATE_MAGIC,
-  offsetof(or_state_t, magic_),
-  state_abbrevs_,
-  NULL,
-  state_vars_,
-  or_state_validate_cb,
-  or_state_free_cb,
-  &state_extra_var,
+  .size = sizeof(or_state_t),
+  .magic = {
+   "or_state_t",
+   OR_STATE_MAGIC,
+   offsetof(or_state_t, magic_),
+  },
+  .abbrevs = state_abbrevs_,
+  .vars = state_vars_,
+  .legacy_validate_fn = or_state_validate_cb,
+  .extra = &state_extra_var,
+  .has_config_suite = true,
+  .config_suite_offset = offsetof(or_state_t, substates_),
 };
 
+/* A global configuration manager for state-file objects */
+static config_mgr_t *state_mgr = NULL;
+
+/** Return the configuration manager for state-file objects. */
+STATIC const config_mgr_t *
+get_state_mgr(void)
+{
+  if (PREDICT_UNLIKELY(state_mgr == NULL)) {
+    state_mgr = config_mgr_new(&state_format);
+    int rv = subsystems_register_state_formats(state_mgr);
+    tor_assert(rv == 0);
+    config_mgr_freeze(state_mgr);
+  }
+  return state_mgr;
+}
+
+#define CHECK_STATE_MAGIC(s) STMT_BEGIN                        \
+    config_check_toplevel_magic(get_state_mgr(), (s));         \
+  STMT_END
+
 /** Persistent serialized state. */
 static or_state_t *global_state = NULL;
 
@@ -249,25 +273,6 @@ validate_transports_in_state(or_state_t *state)
   return 0;
 }
 
-static int
-or_state_validate_cb(void *old_state, void *state, void *default_state,
-                     int from_setconf, char **msg)
-{
-  /* We don't use these; only options do. Still, we need to match that
-   * signature. */
-  (void) from_setconf;
-  (void) default_state;
-  (void) old_state;
-
-  return or_state_validate(state, msg);
-}
-
-static void
-or_state_free_cb(void *state)
-{
-  or_state_free_(state);
-}
-
 /** Return 0 if every setting in <b>state</b> is reasonable, and a
  * permissible transition from <b>old_state</b>.  Else warn and return -1.
  * Should have no side effects, except for normalizing the contents of
@@ -276,6 +281,23 @@ or_state_free_cb(void *state)
 static int
 or_state_validate(or_state_t *state, char **msg)
 {
+  return config_validate(get_state_mgr(), NULL, state, msg);
+}
+
+/**
+ * Legacy validation/normalization callback for or_state_t.  See
+ * legacy_validate_fn_t for more information.
+ */
+static int
+or_state_validate_cb(const void *old_state, void *state_, char **msg)
+{
+  /* There is not a meaningful concept of a state-to-state transition,
+   * since we do not reload the state after we start. */
+  (void) old_state;
+  CHECK_STATE_MAGIC(state_);
+
+  or_state_t *state = state_;
+
   if (entry_guards_parse_state(state, 0, msg)<0)
     return -1;
 
@@ -292,8 +314,11 @@ or_state_set(or_state_t *new_state)
   char *err = NULL;
   int ret = 0;
   tor_assert(new_state);
-  config_free(&state_format, global_state);
+  config_free(get_state_mgr(), global_state);
   global_state = new_state;
+  if (subsystems_set_state(get_state_mgr(), global_state) < 0) {
+    ret = -1;
+  }
   if (entry_guards_parse_state(global_state, 1, &err)<0) {
     log_warn(LD_GENERAL,"%s",err);
     tor_free(err);
@@ -308,6 +333,7 @@ or_state_set(or_state_t *new_state)
       get_circuit_build_times_mutable(),global_state) < 0) {
     ret = -1;
   }
+
   return ret;
 }
 
@@ -353,9 +379,8 @@ or_state_save_broken(char *fname)
 STATIC or_state_t *
 or_state_new(void)
 {
-  or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t));
-  new_state->magic_ = OR_STATE_MAGIC;
-  config_init(&state_format, new_state);
+  or_state_t *new_state = config_new(get_state_mgr());
+  config_init(get_state_mgr(), new_state);
 
   return new_state;
 }
@@ -396,7 +421,7 @@ or_state_load(void)
     int assign_retval;
     if (config_get_lines(contents, &lines, 0)<0)
       goto done;
-    assign_retval = config_assign(&state_format, new_state,
+    assign_retval = config_assign(get_state_mgr(), new_state,
                                   lines, 0, &errmsg);
     config_free_lines(lines);
     if (assign_retval<0)
@@ -423,7 +448,7 @@ or_state_load(void)
     or_state_save_broken(fname);
 
     tor_free(contents);
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
     new_state = or_state_new();
   } else if (contents) {
@@ -456,7 +481,7 @@ or_state_load(void)
   tor_free(fname);
   tor_free(contents);
   if (new_state)
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
   return r;
 }
@@ -496,9 +521,11 @@ or_state_save(time_t now)
 
   /* Call everything else that might dirty the state even more, in order
    * to avoid redundant writes. */
+  (void) subsystems_flush_state(get_state_mgr(), global_state);
   entry_guards_update_state(global_state);
   rep_hist_update_state(global_state);
   circuit_build_times_update_state(get_circuit_build_times(), global_state);
+
   if (accounting_is_enabled(get_options()))
     accounting_run_housekeeping(now);
 
@@ -507,7 +534,7 @@ or_state_save(time_t now)
   tor_free(global_state->TorVersion);
   tor_asprintf(&global_state->TorVersion, "Tor %s", get_version());
 
-  state = config_dump(&state_format, NULL, global_state, 1, 0);
+  state = config_dump(get_state_mgr(), NULL, global_state, 1, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&contents,
                "# Tor state file last generated on %s local time\n"
@@ -617,7 +644,7 @@ get_stored_bindaddr_for_server_transport(const char *transport)
   {
     /* See if the user explicitly asked for a specific listening
        address for this transport. */
-    char *conf_bindaddr = get_transport_bindaddr_from_config(transport);
+    char *conf_bindaddr = pt_get_bindaddr_from_config(transport);
     if (conf_bindaddr)
       return conf_bindaddr;
   }
@@ -717,7 +744,7 @@ or_state_free_(or_state_t *state)
   if (!state)
     return;
 
-  config_free(&state_format, state);
+  config_free(get_state_mgr(), state);
 }
 
 void
@@ -725,4 +752,5 @@ or_state_free_all(void)
 {
   or_state_free(global_state);
   global_state = NULL;
+  config_mgr_free(state_mgr);
 }
diff --git a/src/app/config/statefile.h b/src/app/config/statefile.h
index 1950078450..98d9d2dda1 100644
--- a/src/app/config/statefile.h
+++ b/src/app/config/statefile.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,6 +31,8 @@ STATIC struct config_line_t *get_transport_in_state_by_name(
 STATIC void or_state_free_(or_state_t *state);
 #define or_state_free(st) FREE_AND_NULL(or_state_t, or_state_free_, (st))
 STATIC or_state_t *or_state_new(void);
-#endif
+struct config_mgr_t;
+STATIC const struct config_mgr_t *get_state_mgr(void);
+#endif /* defined(STATEFILE_PRIVATE) */
 
 #endif /* !defined(TOR_STATEFILE_H) */
diff --git a/src/app/config/testnet.inc b/src/app/config/testnet.inc
new file mode 100644
index 0000000000..907c35f97c
--- /dev/null
+++ b/src/app/config/testnet.inc
@@ -0,0 +1,34 @@
+// When modifying, don't forget to update the defaults
+// for 'TestingTorNetwork' in 'doc/tor.1.txt'
+{ "DirAllowPrivateAddresses", "1" },
+{ "EnforceDistinctSubnets", "0" },
+{ "AssumeReachable", "1" },
+{ "AuthDirMaxServersPerAddr", "0" },
+{ "ClientBootstrapConsensusAuthorityDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusFallbackDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay", "0" },
+{ "ClientDNSRejectInternalAddresses", "0" },
+{ "ClientRejectInternalAddresses", "0" },
+{ "CountPrivateBandwidth", "1" },
+{ "ExitPolicyRejectPrivate", "0" },
+{ "ExtendAllowPrivateAddresses", "1" },
+{ "V3AuthVotingInterval", "5 minutes" },
+{ "V3AuthVoteDelay", "20 seconds" },
+{ "V3AuthDistDelay", "20 seconds" },
+{ "TestingV3AuthInitialVotingInterval", "150 seconds" },
+{ "TestingV3AuthInitialVoteDelay", "20 seconds" },
+{ "TestingV3AuthInitialDistDelay", "20 seconds" },
+{ "TestingAuthDirTimeToLearnReachability", "0 minutes" },
+{ "MinUptimeHidServDirectoryV2", "0 minutes" },
+{ "TestingServerDownloadInitialDelay", "0" },
+{ "TestingClientDownloadInitialDelay", "0" },
+{ "TestingServerConsensusDownloadInitialDelay", "0" },
+{ "TestingClientConsensusDownloadInitialDelay", "0" },
+{ "TestingBridgeDownloadInitialDelay", "10" },
+{ "TestingBridgeBootstrapDownloadInitialDelay", "0" },
+{ "TestingClientMaxIntervalWithoutRequest", "5 seconds" },
+{ "TestingDirConnectionMaxStall", "30 seconds" },
+{ "TestingEnableConnBwEvent", "1" },
+{ "TestingEnableCellStatsEvent", "1" },
+{ "RendPostPeriod", "2 minutes" },
+{ "___UsingTestNetworkDefaults", "1" },
diff --git a/src/app/config/tor_cmdline_mode.h b/src/app/config/tor_cmdline_mode.h
new file mode 100644
index 0000000000..30a339a438
--- /dev/null
+++ b/src/app/config/tor_cmdline_mode.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_cmdline_mode.h
+ * \brief Declare the tor_cmdline_mode_t enumeration
+ **/
+
+#ifndef TOR_CMDLINE_MODE_H
+#define TOR_CMDLINE_MODE_H
+
+/**
+ * Enumeration to describe which command Tor is running.  These commands
+ * are controlled by command-line options.
+ **/
+typedef enum {
+  CMD_RUN_TOR=0, /**< The default: run Tor as a daemon. */
+  CMD_LIST_FINGERPRINT, /**< Running --list-fingerprint. */
+  CMD_HASH_PASSWORD, /**< Running --hash-password. */
+  CMD_VERIFY_CONFIG, /**< Running --verify-config. */
+  CMD_DUMP_CONFIG, /**< Running --dump-config. */
+  CMD_KEYGEN, /**< Running --keygen */
+  CMD_KEY_EXPIRATION, /**< Running --key-expiration */
+  CMD_IMMEDIATE, /**< Special value: indicates a command that is handled
+                  * immediately during configuration processing. */
+  CMD_RUN_UNITTESTS, /**< Special value: indicates that we have entered
+                      * the Tor code from the unit tests, not from the
+                      * regular Tor binary at all. */
+} tor_cmdline_mode_t;
+
+#endif /* !defined(TOR_CMDLINE_MODE_H) */
diff --git a/src/app/main/.may_include b/src/app/main/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/app/main/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/app/main/app_main.md b/src/app/main/app_main.md
new file mode 100644
index 0000000000..b8c789716c
--- /dev/null
+++ b/src/app/main/app_main.md
@@ -0,0 +1,2 @@
+@dir /app/main
+@brief app/main: Entry point for tor.
diff --git a/src/app/main/include.am b/src/app/main/include.am
new file mode 100644
index 0000000000..ea392a8581
--- /dev/null
+++ b/src/app/main/include.am
@@ -0,0 +1,18 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/app/main/main.c			\
+	src/app/main/shutdown.c			\
+	src/app/main/subsystem_list.c		\
+	src/app/main/subsysmgr.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/app/main/main.h				\
+	src/app/main/ntmain.h				\
+	src/app/main/shutdown.h 			\
+	src/app/main/subsysmgr.h
+
+if BUILD_NT_SERVICES
+LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
+endif
diff --git a/src/app/main/main.c b/src/app/main/main.c
index 8a5d4cfd15..fd166638db 100644
--- a/src/app/main/main.c
+++ b/src/app/main/main.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,80 +13,69 @@
 
 #include "app/config/config.h"
 #include "app/config/statefile.h"
+#include "app/config/quiet_level.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
-#include "core/or/channeltls.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitlist.h"
-#include "core/or/circuitmux_ewma.h"
 #include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/dos.h"
-#include "core/or/policies.h"
-#include "core/or/protover.h"
 #include "core/or/relay.h"
-#include "core/or/scheduler.h"
 #include "core/or/status.h"
 #include "feature/api/tor_api.h"
 #include "feature/api/tor_api_internal.h"
 #include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/entrynodes.h"
-#include "feature/client/transports.h"
 #include "feature/control/control.h"
-#include "feature/dirauth/bwauth.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
 #include "feature/dirparse/routerparse.h"
 #include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
-#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/relay/onion_queue.h"
 #include "feature/relay/routerkeys.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
-#include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/err/backtrace.h"
-#include "lib/geoip/geoip.h"
+#include "lib/net/resolve.h"
 
 #include "lib/process/waitpid.h"
+#include "lib/pubsub/pubsub_build.h"
 
 #include "lib/meminfo/meminfo.h"
 #include "lib/osinfo/uname.h"
 #include "lib/sandbox/sandbox.h"
 #include "lib/fs/lockfile.h"
-#include "lib/net/resolve.h"
 #include "lib/tls/tortls.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/encoding/confline.h"
 #include "lib/evloop/timers.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
 
@@ -107,8 +96,6 @@
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
-void evdns_shutdown(int);
-
 #ifdef HAVE_RUST
 // helper function defined in Rust to output a log message indicating if tor is
 // running with Rust enabled. See src/rust/tor_util
@@ -121,16 +108,6 @@ static void dumpmemusage(int severity);
 static void dumpstats(int severity); /* log stats */
 static void process_signal(int sig);
 
-/********* START VARIABLES **********/
-
-/** Decides our behavior when no logs are configured/before any
- * logs have been configured.  For 0, we log notice to stdout as normal.
- * For 1, we log warnings only.  For 2, we log nothing.
- */
-int quiet_level = 0;
-
-/********* END VARIABLES ************/
-
 /** Called when we get a SIGHUP: reload configuration files and keys,
  * retry all connections, and so on. */
 static int
@@ -301,9 +278,35 @@ process_signal(int sig)
       log_heartbeat(time(NULL));
       control_event_signal(sig);
       break;
+    case SIGACTIVE:
+      /* "SIGACTIVE" counts as ersatz user activity. */
+      note_user_activity(approx_time());
+      control_event_signal(sig);
+      break;
+    case SIGDORMANT:
+      /* "SIGDORMANT" means to ignore past user activity */
+      log_notice(LD_GENERAL, "Going dormant because of controller request.");
+      reset_user_activity(0);
+      set_network_participation(false);
+      schedule_rescan_periodic_events();
+      control_event_signal(sig);
+      break;
   }
 }
 
+#ifdef _WIN32
+/** Activate SIGINT on reciving a control signal in console */
+static BOOL WINAPI
+process_win32_console_ctrl(DWORD ctrl_type)
+{
+  /* Ignore type of the ctrl signal */
+  (void) ctrl_type;
+
+  activate_signal(SIGINT);
+  return TRUE;
+}
+#endif
+
 /**
  * Write current memory usage information to the log.
  */
@@ -423,18 +426,7 @@ dumpstats(int severity)
 
   rep_hist_dump_stats(now,severity);
   rend_service_dump_stats(severity);
-}
-
-/** Called by exit() as we shut down the process.
- */
-static void
-exit_function(void)
-{
-  /* NOTE: If we ever daemonize, this gets called immediately.  That's
-   * okay for now, because we only use this on Windows.  */
-#ifdef _WIN32
-  WSACleanup();
-#endif
+  hs_service_dump_stats(severity);
 }
 
 #ifdef _WIN32
@@ -481,6 +473,8 @@ static struct {
   { SIGNEWNYM, 0, NULL },
   { SIGCLEARDNSCACHE, 0, NULL },
   { SIGHEARTBEAT, 0, NULL },
+  { SIGACTIVE, 0, NULL },
+  { SIGDORMANT, 0, NULL },
   { -1, -1, NULL }
 };
 
@@ -515,6 +509,13 @@ handle_signals(void)
                       &signal_handlers[i].signal_value);
     }
   }
+
+#ifdef _WIN32
+    /* Windows lacks traditional POSIX signals but WinAPI provides a function
+     * to handle control signals like Ctrl+C in the console, we can use this to
+     * simulate the SIGINT signal */
+    if (enabled) SetConsoleCtrlHandler(process_win32_console_ctrl, TRUE);
+#endif
 }
 
 /* Cause the signal handler for signal_num to be called in the event loop. */
@@ -537,7 +538,7 @@ int
 tor_init(int argc, char *argv[])
 {
   char progname[256];
-  int quiet = 0;
+  quiet_level_t quiet = QUIET_NONE;
 
   time_of_process_start = time(NULL);
   tor_init_connection_lists();
@@ -545,59 +546,28 @@ tor_init(int argc, char *argv[])
   tor_snprintf(progname, sizeof(progname), "Tor %s", get_version());
   log_set_application_name(progname);
 
-  /* Set up the crypto nice and early */
-  if (crypto_early_init() < 0) {
-    log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!");
-    return -1;
-  }
-
   /* Initialize the history structures. */
   rep_hist_init();
   /* Initialize the service cache. */
   rend_cache_init();
   addressmap_init(); /* Init the client dns cache. Do it always, since it's
                       * cheap. */
+
   /* Initialize the HS subsystem. */
   hs_init();
 
   {
-  /* We search for the "quiet" option first, since it decides whether we
-   * will log anything at all to the command line. */
-    config_line_t *opts = NULL, *cmdline_opts = NULL;
-    const config_line_t *cl;
-    (void) config_parse_commandline(argc, argv, 1, &opts, &cmdline_opts);
-    for (cl = cmdline_opts; cl; cl = cl->next) {
-      if (!strcmp(cl->key, "--hush"))
-        quiet = 1;
-      if (!strcmp(cl->key, "--quiet") ||
-          !strcmp(cl->key, "--dump-config"))
-        quiet = 2;
-      /* The following options imply --hush */
-      if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") ||
-          !strcmp(cl->key, "--list-torrc-options") ||
-          !strcmp(cl->key, "--library-versions") ||
-          !strcmp(cl->key, "--list-modules") ||
-          !strcmp(cl->key, "--hash-password") ||
-          !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) {
-        if (quiet < 1)
-          quiet = 1;
-      }
-    }
-    config_free_lines(opts);
-    config_free_lines(cmdline_opts);
+    /* We check for the "quiet"/"hush" settings first, since they decide
+       whether we log anything at all to stdout. */
+    parsed_cmdline_t *cmdline;
+    cmdline = config_parse_commandline(argc, argv, 1);
+    if (cmdline)
+      quiet = cmdline->quiet_level;
+    parsed_cmdline_free(cmdline);
   }
 
  /* give it somewhere to log to initially */
-  switch (quiet) {
-    case 2:
-      /* no initial logging */
-      break;
-    case 1:
-      add_temp_log(LOG_WARN);
-      break;
-    default:
-      add_temp_log(LOG_NOTICE);
-  }
+  add_default_log_for_quiet_level(quiet);
   quiet_level = quiet;
 
   {
@@ -631,12 +601,6 @@ tor_init(int argc, char *argv[])
   rust_log_welcome_string();
 #endif /* defined(HAVE_RUST) */
 
-  if (network_init()<0) {
-    log_err(LD_BUG,"Error initializing network; exiting.");
-    return -1;
-  }
-  atexit(exit_function);
-
   int init_rv = options_init_from_torrc(argc,argv);
   if (init_rv < 0) {
     log_err(LD_CONFIG,"Reading config failed--see warnings above.");
@@ -647,12 +611,17 @@ tor_init(int argc, char *argv[])
     return 1;
   }
 
-  /* The options are now initialised */
-  const or_options_t *options = get_options();
-
-  /* Initialize channelpadding parameters to defaults until we get
-   * a consensus */
+  /* Initialize channelpadding and circpad parameters to defaults
+   * until we get a consensus */
   channelpadding_new_consensus_params(NULL);
+  circpad_new_consensus_params(NULL);
+
+  /* Initialize circuit padding to defaults+torrc until we get a consensus */
+  circpad_machines_init();
+
+  /* Initialize hidden service DoS subsystem. We need to do this once the
+   * configuration object has been set because it can be accessed. */
+  hs_dos_init();
 
   /* Initialize predicted ports list after loading options */
   predicted_ports_init();
@@ -663,17 +632,6 @@ tor_init(int argc, char *argv[])
              "and you probably shouldn't.");
 #endif
 
-  if (crypto_global_init(options->HardwareAccel,
-                         options->AccelName,
-                         options->AccelDir)) {
-    log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
-    return -1;
-  }
-  stream_choice_seed_weak_rng();
-  if (tor_init_libevent_rng() < 0) {
-    log_warn(LD_NET, "Problem initializing libevent RNG.");
-  }
-
   /* Scan/clean unparseable descriptors; after reading config */
   routerparse_init();
 
@@ -742,86 +700,6 @@ release_lockfile(void)
   }
 }
 
-/** Free all memory that we might have allocated somewhere.
- * If <b>postfork</b>, we are a worker process and we want to free
- * only the parts of memory that we won't touch. If !<b>postfork</b>,
- * Tor is shutting down and we should free everything.
- *
- * Helps us find the real leaks with sanitizers and the like. Also valgrind
- * should then report 0 reachable in its leak report (in an ideal world --
- * in practice libevent, SSL, libc etc never quite free everything). */
-void
-tor_free_all(int postfork)
-{
-  if (!postfork) {
-    evdns_shutdown(1);
-  }
-  geoip_free_all();
-  geoip_stats_free_all();
-  dirvote_free_all();
-  routerlist_free_all();
-  networkstatus_free_all();
-  addressmap_free_all();
-  dirserv_free_fingerprint_list();
-  dirserv_free_all();
-  dirserv_clear_measured_bw_cache();
-  rend_cache_free_all();
-  rend_service_authorization_free_all();
-  rep_hist_free_all();
-  dns_free_all();
-  clear_pending_onions();
-  circuit_free_all();
-  entry_guards_free_all();
-  pt_free_all();
-  channel_tls_free_all();
-  channel_free_all();
-  connection_free_all();
-  connection_edge_free_all();
-  scheduler_free_all();
-  nodelist_free_all();
-  microdesc_free_all();
-  routerparse_free_all();
-  ext_orport_free_all();
-  control_free_all();
-  tor_free_getaddrinfo_cache();
-  protover_free_all();
-  bridges_free_all();
-  consdiffmgr_free_all();
-  hs_free_all();
-  dos_free_all();
-  circuitmux_ewma_free_all();
-  accounting_free_all();
-
-  if (!postfork) {
-    config_free_all();
-    or_state_free_all();
-    router_free_all();
-    routerkeys_free_all();
-    policies_free_all();
-  }
-  if (!postfork) {
-    tor_tls_free_all();
-#ifndef _WIN32
-    tor_getpwnam(NULL);
-#endif
-  }
-  /* stuff in main.c */
-
-  tor_mainloop_free_all();
-
-  if (!postfork) {
-    release_lockfile();
-  }
-  tor_libevent_free_all();
-  /* Stuff in util.c and address.c*/
-  if (!postfork) {
-    escaped(NULL);
-    esc_router_info(NULL);
-    clean_up_backtrace_handler();
-    logs_free_all(); /* free log strings. do this last so logs keep working. */
-  }
-}
-
 /**
  * Remove the specified file, and log a warning if the operation fails for
  * any reason other than the file not existing. Ignores NULL filenames.
@@ -835,51 +713,6 @@ tor_remove_file(const char *filename)
   }
 }
 
-/** Do whatever cleanup is necessary before shutting Tor down. */
-void
-tor_cleanup(void)
-{
-  const or_options_t *options = get_options();
-  if (options->command == CMD_RUN_TOR) {
-    time_t now = time(NULL);
-    /* Remove our pid file. We don't care if there was an error when we
-     * unlink, nothing we could do about it anyways. */
-    tor_remove_file(options->PidFile);
-    /* Remove control port file */
-    tor_remove_file(options->ControlPortWriteToFile);
-    /* Remove cookie authentication file */
-    {
-      char *cookie_fname = get_controller_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    /* Remove Extended ORPort cookie authentication file */
-    {
-      char *cookie_fname = get_ext_or_auth_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    if (accounting_is_enabled(options))
-      accounting_record_bandwidth_usage(now, get_or_state());
-    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
-    or_state_save(now);
-    if (authdir_mode(options)) {
-      sr_save_and_cleanup();
-    }
-    if (authdir_mode_tests_reachability(options))
-      rep_hist_record_mtbf_data(now, 0);
-    keypin_close_journal();
-  }
-
-  timers_shutdown();
-
-  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
-                      later, if it makes shutdown unacceptably slow.  But for
-                      now, leave it here: it's helped us catch bugs in the
-                      past. */
-  crypto_global_cleanup();
-}
-
 /** Read/create keys as needed, and echo our fingerprint to stdout. */
 static int
 do_list_fingerprint(void)
@@ -988,6 +821,9 @@ sandbox_init_filter(void)
 #define OPEN(name)                              \
   sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name))
 
+#define OPENDIR(dir)                            \
+  sandbox_cfg_allow_opendir_dirname(&cfg, tor_strdup(dir))
+
 #define OPEN_DATADIR(name)                      \
   sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname(name))
 
@@ -1004,8 +840,10 @@ sandbox_init_filter(void)
     OPEN_DATADIR2(name, name2 suffix);                  \
   } while (0)
 
+// KeyDirectory is a directory, but it is only opened in check_private_dir
+// which calls open instead of opendir
 #define OPEN_KEY_DIRECTORY() \
-  sandbox_cfg_allow_open_filename(&cfg, tor_strdup(options->KeyDirectory))
+  OPEN(options->KeyDirectory)
 #define OPEN_CACHEDIR(name)                      \
   sandbox_cfg_allow_open_filename(&cfg, get_cachedir_fname(name))
 #define OPEN_CACHEDIR_SUFFIX(name, suffix) do {  \
@@ -1019,6 +857,8 @@ sandbox_init_filter(void)
     OPEN_KEYDIR(name suffix);                    \
   } while (0)
 
+  // DataDirectory is a directory, but it is only opened in check_private_dir
+  // which calls open instead of opendir
   OPEN(options->DataDirectory);
   OPEN_KEY_DIRECTORY();
 
@@ -1066,7 +906,11 @@ sandbox_init_filter(void)
   }
 
   SMARTLIST_FOREACH(options->FilesOpenedByIncludes, char *, f, {
-    OPEN(f);
+    if (file_status(f) == FN_DIR) {
+      OPENDIR(f);
+    } else {
+      OPEN(f);
+    }
   });
 
 #define RENAME_SUFFIX(name, suffix)        \
@@ -1179,7 +1023,7 @@ sandbox_init_filter(void)
      * directory that holds it. */
     char *dirname = tor_strdup(port->unix_addr);
     if (get_parent_directory(dirname) == 0) {
-      OPEN(dirname);
+      OPENDIR(dirname);
     }
     tor_free(dirname);
     sandbox_cfg_allow_chmod_filename(&cfg, tor_strdup(port->unix_addr));
@@ -1273,7 +1117,6 @@ int
 run_tor_main_loop(void)
 {
   handle_signals();
-  monotime_init();
   timers_initialize();
   initialize_mainloop_events();
 
@@ -1378,6 +1221,32 @@ run_tor_main_loop(void)
   return do_main_loop();
 }
 
+/** Install the publish/subscribe relationships for all the subsystems. */
+void
+pubsub_install(void)
+{
+    pubsub_builder_t *builder = pubsub_builder_new();
+    int r = subsystems_add_pubsub(builder);
+    tor_assert(r == 0);
+    r = tor_mainloop_connect_pubsub(builder); // consumes builder
+    tor_assert(r == 0);
+}
+
+/** Connect the mainloop to its publish/subscribe message delivery events if
+ * appropriate, and configure the global channels appropriately. */
+void
+pubsub_connect(void)
+{
+  if (get_options()->command == CMD_RUN_TOR) {
+    tor_mainloop_connect_pubsub_events();
+    /* XXXX For each pubsub channel, its delivery strategy should be set at
+     * this XXXX point, using tor_mainloop_set_delivery_strategy().
+     */
+    tor_mainloop_set_delivery_strategy("orconn", DELIV_IMMEDIATE);
+    tor_mainloop_set_delivery_strategy("ocirc", DELIV_IMMEDIATE);
+  }
+}
+
 /* Main entry point for the Tor process.  Called from tor_main(), and by
  * anybody embedding Tor. */
 int
@@ -1385,54 +1254,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
 {
   int result = 0;
 
-#ifdef _WIN32
-#ifndef HeapEnableTerminationOnCorruption
-#define HeapEnableTerminationOnCorruption 1
-#endif
-  /* On heap corruption, just give up; don't try to play along. */
-  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
-
-  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
-   * (On 64-bit Windows it always fails, and some compilers don't like the
-   * PSETDEP cast.)
-   * 32-bit Windows defines _WIN32.
-   * 64-bit Windows defines _WIN32 and _WIN64. */
-#ifndef _WIN64
-  /* Call SetProcessDEPPolicy to permanently enable DEP.
-     The function will not resolve on earlier versions of Windows,
-     and failure is not dangerous. */
-  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
-  if (hMod) {
-    typedef BOOL (WINAPI *PSETDEP)(DWORD);
-    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
-                           "SetProcessDEPPolicy");
-    if (setdeppolicy) {
-      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
-      setdeppolicy(3);
-    }
-  }
-#endif /* !defined(_WIN64) */
-#endif /* defined(_WIN32) */
-
-  {
-    int bt_err = configure_backtrace_handler(get_version());
-    if (bt_err < 0) {
-      log_warn(LD_BUG, "Unable to install backtrace handler: %s",
-               strerror(-bt_err));
-    }
-  }
-
 #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED
   event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_);
 #endif
 
-  init_protocol_warning_severity_level();
+  subsystems_init();
 
-  update_approx_time(time(NULL));
-  tor_threads_init();
-  tor_compress_init();
-  init_logging(0);
-  monotime_init();
+  init_protocol_warning_severity_level();
 
   int argc = tor_cfg->argc + tor_cfg->argc_owned;
   char **argv = tor_calloc(argc, sizeof(char*));
@@ -1441,15 +1269,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     memcpy(argv + tor_cfg->argc, tor_cfg->argv_owned,
            tor_cfg->argc_owned*sizeof(char*));
 
-#ifdef NT_SERVICE
-  {
-     int done = 0;
-     result = nt_service_parse_options(argc, argv, &done);
-     if (done) {
-       goto done;
-     }
-  }
-#endif /* defined(NT_SERVICE) */
+  int done = 0;
+  result = nt_service_parse_options(argc, argv, &done);
+  if (POSSIBLE(done))
+    goto done;
+
+  pubsub_install();
+
   {
     int init_rv = tor_init(argc, argv);
     if (init_rv) {
@@ -1459,6 +1285,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     }
   }
 
+  pubsub_connect();
+
   if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
     sandbox_cfg_t* cfg = sandbox_init_filter();
 
@@ -1468,6 +1296,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
       tor_free_all(0);
       return -1;
     }
+    tor_make_getaddrinfo_cache_active();
 
     // registering libevent rng
 #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE
@@ -1478,9 +1307,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
 
   switch (get_options()->command) {
   case CMD_RUN_TOR:
-#ifdef NT_SERVICE
     nt_service_set_state(SERVICE_RUNNING);
-#endif
     result = run_tor_main_loop();
     break;
   case CMD_KEYGEN:
@@ -1498,7 +1325,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     result = 0;
     break;
   case CMD_VERIFY_CONFIG:
-    if (quiet_level == 0)
+    if (quiet_level == QUIET_NONE)
       printf("Configuration was valid\n");
     result = 0;
     break;
@@ -1506,6 +1333,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     result = do_dump_config();
     break;
   case CMD_RUN_UNITTESTS: /* only set by test.c */
+  case CMD_IMMEDIATE: /* Handled in config.c */
   default:
     log_warn(LD_BUG,"Illegal command number %d: internal error.",
              get_options()->command);
diff --git a/src/app/main/main.h b/src/app/main/main.h
index bbbbf984fb..e6ed978c61 100644
--- a/src/app/main/main.h
+++ b/src/app/main/main.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,11 +21,11 @@ void release_lockfile(void);
 
 void tor_remove_file(const char *filename);
 
-void tor_cleanup(void);
-void tor_free_all(int postfork);
-
 int tor_init(int argc, char **argv);
 
 int run_tor_main_loop(void);
 
+void pubsub_install(void);
+void pubsub_connect(void);
+
 #endif /* !defined(TOR_MAIN_H) */
diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c
index 05d203b0be..5dc0edd591 100644
--- a/src/app/main/ntmain.c
+++ b/src/app/main/ntmain.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,6 +24,7 @@
 #include "app/config/config.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
 #include "core/mainloop/mainloop.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/fs/winlib.h"
@@ -65,7 +66,7 @@ static int nt_service_cmd_stop(void);
 
 /** Struct to hold dynamically loaded NT-service related function pointers.
  */
-struct service_fns {
+struct {
   int loaded;
 
   /** @{ */
@@ -282,7 +283,9 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_install();
   r = tor_init(backup_argc, backup_argv);
+
   if (r) {
     /* Failed to start the Tor service */
     r = NT_SERVICE_ERROR_TORINIT_FAILED;
@@ -293,6 +296,8 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_connect();
+
   /* Set the service's status to SERVICE_RUNNING and start the main
    * event loop */
   service_status.dwCurrentState = SERVICE_RUNNING;
@@ -321,9 +326,12 @@ nt_service_main(void)
     errmsg = format_win32_error(result);
     printf("Service error %d : %s\n", (int) result, errmsg);
     tor_free(errmsg);
+
+    pubsub_install();
     if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
       if (tor_init(backup_argc, backup_argv))
         return;
+      pubsub_connect();
       switch (get_options()->command) {
       case CMD_RUN_TOR:
         run_tor_main_loop();
@@ -339,6 +347,7 @@ nt_service_main(void)
                "or --key-expiration) in NT service.");
         break;
       case CMD_RUN_UNITTESTS:
+      case CMD_IMMEDIATE:
       default:
         log_err(LD_CONFIG, "Illegal command number %d: internal error.",
                 get_options()->command);
@@ -594,7 +603,7 @@ nt_service_install(int argc, char **argv)
       /* Genericity is apparently _so_ last year in Redmond, where some
        * accounts are accounts that you can look up, and some accounts
        * are magic and undetectable via the security subsystem. See
-       * http://msdn2.microsoft.com/en-us/library/ms684188.aspx
+       * https://msdn2.microsoft.com/en-us/library/ms684188.aspx
        */
       printf("Running on a Post-Win2K OS, so we'll assume that the "
              "LocalService account exists.\n");
@@ -607,6 +616,7 @@ nt_service_install(int argc, char **argv)
                             &sidUse) == 0) {
     /* XXXX For some reason, the above test segfaults. Fix that. */
     printf("User \"%s\" doesn't seem to exist.\n", user_acct);
+    tor_free(command);
     return -1;
   } else {
     printf("Will try to install service as user \"%s\".\n", user_acct);
diff --git a/src/app/main/ntmain.h b/src/app/main/ntmain.h
index c39386c054..c2d6e23da7 100644
--- a/src/app/main/ntmain.h
+++ b/src/app/main/ntmain.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,8 @@ int nt_service_is_stopping(void);
 void nt_service_set_state(DWORD state);
 #else
 #define nt_service_is_stopping() 0
+#define nt_service_parse_options(a, b, c) (0)
+#define nt_service_set_state(s) STMT_NIL
 #endif /* defined(NT_SERVICE) */
 
 #endif /* !defined(TOR_NTMAIN_H) */
-
diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c
new file mode 100644
index 0000000000..aac15246b9
--- /dev/null
+++ b/src/app/main/shutdown.c
@@ -0,0 +1,169 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file shutdown.c
+ * @brief Code to free global resources used by Tor.
+ *
+ * In the future, this should all be handled by the subsystem manager. */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop_pubsub.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/connection_edge.h"
+#include "core/or/dos.h"
+#include "core/or/scheduler.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/relay_config.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendclient.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/geoip/geoip.h"
+
+void evdns_shutdown(int);
+
+/** Do whatever cleanup is necessary before shutting Tor down. */
+void
+tor_cleanup(void)
+{
+  const or_options_t *options = get_options();
+  if (options->command == CMD_RUN_TOR) {
+    time_t now = time(NULL);
+    /* Remove our pid file. We don't care if there was an error when we
+     * unlink, nothing we could do about it anyways. */
+    tor_remove_file(options->PidFile);
+    /* Remove control port file */
+    tor_remove_file(options->ControlPortWriteToFile);
+    /* Remove cookie authentication file */
+    {
+      char *cookie_fname = get_controller_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    /* Remove Extended ORPort cookie authentication file */
+    {
+      char *cookie_fname = get_ext_or_auth_cookie_file_name();
+      if (cookie_fname)
+        tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    if (accounting_is_enabled(options))
+      accounting_record_bandwidth_usage(now, get_or_state());
+    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
+    or_state_save(now);
+    if (authdir_mode(options)) {
+      sr_save_and_cleanup();
+    }
+    if (authdir_mode_tests_reachability(options))
+      rep_hist_record_mtbf_data(now, 0);
+  }
+
+  timers_shutdown();
+
+  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
+                      later, if it makes shutdown unacceptably slow.  But for
+                      now, leave it here: it's helped us catch bugs in the
+                      past. */
+}
+
+/** Free all memory that we might have allocated somewhere.
+ * If <b>postfork</b>, we are a worker process and we want to free
+ * only the parts of memory that we won't touch. If !<b>postfork</b>,
+ * Tor is shutting down and we should free everything.
+ *
+ * Helps us find the real leaks with sanitizers and the like. Also valgrind
+ * should then report 0 reachable in its leak report (in an ideal world --
+ * in practice libevent, SSL, libc etc never quite free everything). */
+void
+tor_free_all(int postfork)
+{
+  if (!postfork) {
+    evdns_shutdown(1);
+  }
+  geoip_free_all();
+  geoip_stats_free_all();
+  routerlist_free_all();
+  networkstatus_free_all();
+  addressmap_free_all();
+  dirserv_free_all();
+  rend_cache_free_all();
+  rend_service_authorization_free_all();
+  rep_hist_free_all();
+  circuit_free_all();
+  circpad_machines_free();
+  entry_guards_free_all();
+  pt_free_all();
+  channel_tls_free_all();
+  channel_free_all();
+  connection_free_all();
+  connection_edge_free_all();
+  scheduler_free_all();
+  nodelist_free_all();
+  microdesc_free_all();
+  routerparse_free_all();
+  control_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
+  hs_free_all();
+  dos_free_all();
+  circuitmux_ewma_free_all();
+  accounting_free_all();
+  circpad_free_all();
+
+  if (!postfork) {
+    config_free_all();
+    relay_config_free_all();
+    or_state_free_all();
+  }
+  if (!postfork) {
+#ifndef _WIN32
+    tor_getpwnam(NULL);
+#endif
+  }
+  /* stuff in main.c */
+
+  tor_mainloop_disconnect_pubsub();
+
+  if (!postfork) {
+    release_lockfile();
+  }
+
+  subsystems_shutdown();
+
+  /* Stuff in util.c and address.c*/
+  if (!postfork) {
+    esc_router_info(NULL);
+  }
+}
diff --git a/src/app/main/shutdown.h b/src/app/main/shutdown.h
new file mode 100644
index 0000000000..623ae9525b
--- /dev/null
+++ b/src/app/main/shutdown.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shutdown.h
+ * \brief Header file for shutdown.c.
+ **/
+
+#ifndef TOR_SHUTDOWN_H
+#define TOR_SHUTDOWN_H
+
+void tor_cleanup(void);
+void tor_free_all(int postfork);
+
+#endif /* !defined(TOR_SHUTDOWN_H) */
diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c
new file mode 100644
index 0000000000..de601d28cd
--- /dev/null
+++ b/src/app/main/subsysmgr.c
@@ -0,0 +1,478 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsysmgr.c
+ * @brief Manager for Tor's subsystems.
+ *
+ * This code is responsible for initializing, configuring, and shutting
+ * down all of Tor's individual subsystems.
+ **/
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+
+#include "lib/confmgt/confmgt.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * True iff we have checked tor_subsystems for consistency.
+ **/
+static bool subsystem_array_validated = false;
+
+/** Index value indicating that a subsystem has no options/state object, and
+ * so that object does not have an index. */
+#define IDX_NONE (-1)
+
+/**
+ * Runtime status of a single subsystem.
+ **/
+typedef struct subsys_status_t {
+  /** True if the given subsystem is initialized. */
+  bool initialized;
+  /** Index for this subsystem's options object, or IDX_NONE for none. */
+  int options_idx;
+  /** Index for this subsystem's state object, or IDX_NONE for none. */
+  int state_idx;
+} subsys_status_t;
+
+/** An overestimate of the number of subsystems. */
+#define N_SYS_STATUS 128
+/**
+ * True if a given subsystem is initialized.  Expand this array if there
+ * are more than this number of subsystems.  (We'd rather not
+ * dynamically allocate in this module.)
+ **/
+static subsys_status_t sys_status[N_SYS_STATUS];
+
+/** Set <b>status</b> to a default (not set-up) state. */
+static void
+subsys_status_clear(subsys_status_t *status)
+{
+  if (!status)
+    return;
+  memset(status, 0, sizeof(*status));
+  status->initialized = false;
+  status->state_idx = IDX_NONE;
+  status->options_idx = IDX_NONE;
+}
+
+/**
+ * Exit with a raw assertion if the subsystems list is inconsistent;
+ * initialize the subsystem_initialized array.
+ **/
+static void
+check_and_setup(void)
+{
+  if (subsystem_array_validated)
+    return;
+
+  raw_assert(ARRAY_LENGTH(sys_status) >= n_tor_subsystems);
+  memset(sys_status, 0, sizeof(sys_status));
+
+  int last_level = MIN_SUBSYS_LEVEL;
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) {
+      fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. "
+              "It is supposed to be between %d and %d (inclusive).\n",
+              sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    if (sys->level < last_level) {
+      fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. "
+              "Its level is %d; but the previous subsystem's level was %d.\n",
+              sys->name, i, sys->level, last_level);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    subsys_status_clear(&sys_status[i]);
+
+    last_level = sys->level;
+  }
+
+  subsystem_array_validated = true;
+}
+
+/**
+ * Initialize all the subsystems; exit on failure.
+ **/
+int
+subsystems_init(void)
+{
+  return subsystems_init_upto(MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Initialize all the subsystems whose level is less than or equal to
+ * <b>target_level</b>; exit on failure.
+ **/
+int
+subsystems_init_upto(int target_level)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (sys_status[i].initialized)
+      continue;
+    int r = 0;
+    if (sys->initialize) {
+      // Note that the logging subsystem is designed so that it does no harm
+      // to log a message in an uninitialized state.  These messages will be
+      // discarded for now, however.
+      log_debug(LD_GENERAL, "Initializing %s", sys->name);
+      r = sys->initialize();
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n",
+              sys->name, i);
+      raw_assert_unreached_msg("A subsystem couldn't be initialized.");
+    }
+    sys_status[i].initialized = true;
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems of level no more than <b>target_level</b>.
+ **/
+int
+subsystems_add_pubsub_upto(pubsub_builder_t *builder,
+                           int target_level)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (! sys_status[i].initialized)
+      continue;
+    int r = 0;
+    if (sys->add_pubsub) {
+      subsys_id_t sysid = get_subsys_id(sys->name);
+      raw_assert(sysid != ERROR_ID);
+      pubsub_connector_t *connector;
+      connector = pubsub_connector_for_subsystem(builder, sysid);
+      r = sys->add_pubsub(connector);
+      pubsub_connector_free(connector);
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) could not connect to "
+              "publish/subscribe system.", sys->name, sys->level);
+      raw_assert_unreached_msg("A subsystem couldn't be connected.");
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems.
+ **/
+int
+subsystems_add_pubsub(pubsub_builder_t *builder)
+{
+  return subsystems_add_pubsub_upto(builder, MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Shut down all the subsystems.
+ **/
+void
+subsystems_shutdown(void)
+{
+  subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1);
+}
+
+/**
+ * Shut down all the subsystems whose level is above <b>target_level</b>.
+ **/
+void
+subsystems_shutdown_downto(int target_level)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level <= target_level)
+      break;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->shutdown) {
+      log_debug(LD_GENERAL, "Shutting down %s", sys->name);
+      sys->shutdown();
+    }
+    subsys_status_clear(&sys_status[i]);
+  }
+}
+
+/**
+ * Run pre-fork code on all subsystems that declare any
+ **/
+void
+subsystems_prefork(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->prefork) {
+      log_debug(LD_GENERAL, "Pre-fork: %s", sys->name);
+      sys->prefork();
+    }
+  }
+}
+
+/**
+ * Run post-fork code on all subsystems that declare any
+ **/
+void
+subsystems_postfork(void)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->postfork) {
+      log_debug(LD_GENERAL, "Post-fork: %s", sys->name);
+      sys->postfork();
+    }
+  }
+}
+
+/**
+ * Run thread-cleanup code on all subsystems that declare any
+ **/
+void
+subsystems_thread_cleanup(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->thread_cleanup) {
+      log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name);
+      sys->thread_cleanup();
+    }
+  }
+}
+
+/**
+ * Dump a human- and machine-readable list of all the subsystems to stdout,
+ * in their initialization order, prefixed with their level.
+ **/
+void
+subsystems_dump_list(void)
+{
+  for (unsigned i = 0; i < n_tor_subsystems - 1; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    printf("% 4d\t%16s\t%s\n", sys->level, sys->name,
+           sys->location?sys->location:"");
+  }
+}
+
+/**
+ * Register all subsystem-declared options formats in <b>mgr</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+subsystems_register_options_formats(config_mgr_t *mgr)
+{
+  tor_assert(mgr);
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->options_format) {
+      int options_idx = config_mgr_add_format(mgr, sys->options_format);
+      sys_status[i].options_idx = options_idx;
+      log_debug(LD_CONFIG, "Added options format for %s with index %d",
+                sys->name, options_idx);
+    }
+  }
+  return 0;
+}
+
+/**
+ * Register all subsystem-declared state formats in <b>mgr</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+subsystems_register_state_formats(config_mgr_t *mgr)
+{
+  tor_assert(mgr);
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->state_format) {
+      int state_idx = config_mgr_add_format(mgr, sys->state_format);
+      sys_status[i].state_idx = state_idx;
+      log_debug(LD_CONFIG, "Added state format for %s with index %d",
+                sys->name, state_idx);
+    }
+  }
+  return 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Helper: look up the index for <b>sys</b>.  Return -1 if the subsystem
+ * is not recognized.
+ **/
+static int
+subsys_get_idx(const subsys_fns_t *sys)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    if (sys == tor_subsystems[i])
+      return (int)i;
+  }
+  return -1;
+}
+
+/**
+ * Return the current state-manager's index for any state held by the
+ * subsystem <b>sys</b>.  If <b>sys</b> has no options, return -1.
+ *
+ * Using raw indices can be error-prone: only do this from the unit
+ * tests. If you need a way to access another subsystem's configuration,
+ * that subsystem should provide access functions.
+ **/
+int
+subsystems_get_options_idx(const subsys_fns_t *sys)
+{
+  int i = subsys_get_idx(sys);
+  tor_assert(i >= 0);
+  return sys_status[i].options_idx;
+}
+
+/**
+ * Return the current state-manager's index for any state held by the
+ * subsystem <b>sys</b>.  If <b>sys</b> has no state, return -1.
+ *
+ * Using raw indices can be error-prone: only do this from the unit
+ * tests.  If you need a way to access another subsystem's state
+ * that subsystem should provide access functions.
+ **/
+int
+subsystems_get_state_idx(const subsys_fns_t *sys)
+{
+  int i = subsys_get_idx(sys);
+  tor_assert(i >= 0);
+  return sys_status[i].state_idx;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Call all appropriate set_options() methods to tell the various subsystems
+ * about a new set of torrc options.  Return 0 on success, -1 on
+ * nonrecoverable failure.
+ **/
+int
+subsystems_set_options(const config_mgr_t *mgr, struct or_options_t *options)
+{
+  /* XXXX This does not yet handle reversible option assignment; I'll
+   * do that later in this branch. */
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys_status[i].options_idx >= 0 && sys->set_options) {
+      void *obj = config_mgr_get_obj_mutable(mgr, options,
+                                             sys_status[i].options_idx);
+      int rv = sys->set_options(obj);
+      if (rv < 0) {
+        log_err(LD_CONFIG, "Error when handling option for %s; "
+                "cannot proceed.", sys->name);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/**
+ * Call all appropriate set_state() methods to tell the various subsystems
+ * about an initial DataDir/state file.  Return 0 on success, -1 on
+ * nonrecoverable failure.
+ **/
+int
+subsystems_set_state(const config_mgr_t *mgr, struct or_state_t *state)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys_status[i].state_idx >= 0 && sys->set_state) {
+      void *obj = config_mgr_get_obj_mutable(mgr, state,
+                                             sys_status[i].state_idx);
+      int rv = sys->set_state(obj);
+      if (rv < 0) {
+        log_err(LD_CONFIG, "Error when handling state for %s; "
+                "cannot proceed.", sys->name);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/**
+ * Call all appropriate flush_state() methods to tell the various subsystems
+ * to update the state objects in <b>state</b>.  Return 0 on success,
+ * -1 on failure.
+ **/
+int
+subsystems_flush_state(const config_mgr_t *mgr, struct or_state_t *state)
+{
+  int result = 0;
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys_status[i].state_idx >= 0 && sys->flush_state) {
+      void *obj = config_mgr_get_obj_mutable(mgr, state,
+                                             sys_status[i].state_idx);
+      int rv = sys->flush_state(obj);
+      if (rv < 0) {
+        log_warn(LD_CONFIG, "Error when flushing state to state object for %s",
+                sys->name);
+        result = -1;
+      }
+    }
+  }
+  return result;
+}
diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h
new file mode 100644
index 0000000000..ae0b3df469
--- /dev/null
+++ b/src/app/main/subsysmgr.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsysmgr.h
+ * @brief Header for subsysmgr.c
+ **/
+
+#ifndef TOR_SUBSYSMGR_T
+#define TOR_SUBSYSMGR_T
+
+#include "lib/subsys/subsys.h"
+
+extern const struct subsys_fns_t *tor_subsystems[];
+extern const unsigned n_tor_subsystems;
+
+int subsystems_init(void);
+int subsystems_init_upto(int level);
+
+struct pubsub_builder_t;
+int subsystems_add_pubsub_upto(struct pubsub_builder_t *builder,
+                               int target_level);
+int subsystems_add_pubsub(struct pubsub_builder_t *builder);
+
+void subsystems_shutdown(void);
+void subsystems_shutdown_downto(int level);
+
+void subsystems_prefork(void);
+void subsystems_postfork(void);
+void subsystems_thread_cleanup(void);
+
+void subsystems_dump_list(void);
+
+struct config_mgr_t;
+int subsystems_register_options_formats(struct config_mgr_t *mgr);
+int subsystems_register_state_formats(struct config_mgr_t *mgr);
+struct or_options_t;
+struct or_state_t;
+int subsystems_set_options(const struct config_mgr_t *mgr,
+                           struct or_options_t *options);
+int subsystems_set_state(const struct config_mgr_t *mgr,
+                         struct or_state_t *state);
+int subsystems_flush_state(const struct config_mgr_t *mgr,
+                           struct or_state_t *state);
+
+#ifdef TOR_UNIT_TESTS
+int subsystems_get_options_idx(const subsys_fns_t *sys);
+int subsystems_get_state_idx(const subsys_fns_t *sys);
+#endif
+
+#endif /* !defined(TOR_SUBSYSMGR_T) */
diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c
new file mode 100644
index 0000000000..e32083537f
--- /dev/null
+++ b/src/app/main/subsystem_list.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsystem_list.c
+ * @brief List of Tor's subsystems.
+ **/
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+#include "core/mainloop/mainloop_sys.h"
+#include "core/or/or_sys.h"
+#include "feature/control/btrack_sys.h"
+#include "lib/compress/compress_sys.h"
+#include "lib/crypt_ops/crypto_sys.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/log/log_sys.h"
+#include "lib/net/network_sys.h"
+#include "lib/process/process_sys.h"
+#include "lib/llharden/winprocess_sys.h"
+#include "lib/thread/thread_sys.h"
+#include "lib/time/time_sys.h"
+#include "lib/tls/tortls_sys.h"
+#include "lib/wallclock/wallclock_sys.h"
+#include "lib/evloop/evloop_sys.h"
+
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/relay/relay_sys.h"
+
+#include <stddef.h>
+
+/**
+ * Global list of the subsystems in Tor, in the order of their initialization.
+ * Want to know the exact level numbers?
+ * We'll implement a level dump command in #31614.
+ **/
+const subsys_fns_t *tor_subsystems[] = {
+  &sys_winprocess,
+  &sys_torerr,
+
+  &sys_wallclock,
+  &sys_logging,
+  &sys_threads,
+
+  &sys_time,
+
+  &sys_crypto,
+  &sys_compress,
+  &sys_network,
+  &sys_tortls,
+
+  &sys_evloop,
+  &sys_process,
+
+  &sys_mainloop,
+  &sys_or,
+
+  &sys_relay,
+
+  &sys_btrack,
+
+  &sys_dirauth,
+};
+
+const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems);
diff --git a/src/app/main/tor_main.c b/src/app/main/tor_main.c
index 8a887ed269..0ee03fd5e9 100644
--- a/src/app/main/tor_main.c
+++ b/src/app/main/tor_main.c
@@ -1,6 +1,6 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/arch_goals.md b/src/arch_goals.md
new file mode 100644
index 0000000000..92c86d9df8
--- /dev/null
+++ b/src/arch_goals.md
@@ -0,0 +1,31 @@
+@page arch_goals High level code design practices
+
+This page describes the high level design practices for Tor's code.
+This design is a long-term goal of what we want our code to look like,
+rather than a description of how it currently is.
+
+Overall, we want various parts of tor's code to interact with each
+other through a small number of interfaces.
+
+We want to avoid having "god objects" or "god modules".  These are
+objects or modules that know far too much about other parts of the
+code.  God objects/modules are generally recognized to be an
+antipattern of software design.
+
+Historically, there have been modules in tor that have tended toward
+becoming god modules.  These include modules that help more
+specialized code communicate with the outside world: the configuration
+and control modules, for example.  Others are modules that deal with
+global state, initialization, or shutdown.
+
+If a centralized module needs to invoke code in almost every other
+module in the system, it is better if it exports a small, general
+interface that other modules call.  The centralized module should not
+explicitly call out to all the modules that interact with it.
+
+Instead, modules that interact with the centralized module should call
+registration interfaces.  These interfaces allow modules to register
+handlers for things like configuration parsing and control command
+execution.  (The config and control modules are examples of this.)
+Alternatively, registration can happen through statically initialized
+data structures.  (The subsystem mechanism is an example of this.)
diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py
index 3a454a3fc1..4acfea6c0e 100644
--- a/src/config/mmdb-convert.py
+++ b/src/config/mmdb-convert.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/python
 
 #   This software has been dedicated to the public domain under the CC0
 #   public domain dedication.
@@ -9,7 +9,7 @@
 #
 #   You should have received a copy of the CC0 legalcode along with this
 #   work in doc/cc0.txt.  If not, see
-#      <http://creativecommons.org/publicdomain/zero/1.0/>.
+#      <https://creativecommons.org/publicdomain/zero/1.0/>.
 
 #  Nick Mathewson is responsible for this kludge, but takes no
 #  responsibility for it.
@@ -28,6 +28,11 @@
    pieces.
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import struct
 import bisect
 import socket
@@ -77,7 +82,7 @@ def to_int32(s):
 
 def to_int28(s):
     "Parse a pair of big-endian 28-bit integers from bytestring s."
-    a, b = unpack("!LL", s + b'\x00')
+    a, b = struct.unpack("!LL", s + b'\x00')
     return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8)
 
 class Tree(object):
diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging
index cb3adca35c..90bad7f7cc 100644
--- a/src/config/torrc.minimal.in-staging
+++ b/src/config/torrc.minimal.in-staging
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in
index c2ae707e93..51e1c3af4b 100644
--- a/src/config/torrc.sample.in
+++ b/src/config/torrc.sample.in
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
@@ -174,13 +177,11 @@
 
 ## Uncomment this if you want your relay to be an exit, with the default
 ## exit policy (or whatever exit policy you set below).
-## (If ReducedExitPolicy or ExitPolicy are set, relays are exits.
-## If neither exit policy option is set, relays are non-exits.)
+## (If ReducedExitPolicy, ExitPolicy, or IPv6Exit are set, relays are exits.
+## If none of these options are set, relays are non-exits.)
 #ExitRelay 1
 
 ## Uncomment this if you want your relay to allow IPv6 exit traffic.
-## You must also set ExitRelay, ReducedExitPolicy, or ExitPolicy to make your
-## relay into an exit.
 ## (Relays do not allow any exit traffic by default.)
 #IPv6Exit 1
 
diff --git a/src/core/core.md b/src/core/core.md
new file mode 100644
index 0000000000..8ecc43eaae
--- /dev/null
+++ b/src/core/core.md
@@ -0,0 +1,18 @@
+@dir /core
+@brief core: main loop and onion routing functionality
+
+The "core" directory has the central protocols for Tor, which every
+client and relay must implement in order to perform onion routing.
+
+It is divided into three lower-level pieces:
+
+   - \refdir{core/crypto} -- Tor-specific cryptography.
+
+   - \refdir{core/proto} -- Protocol encoding/decoding.
+
+   - \refdir{core/mainloop} -- A connection-oriented asynchronous mainloop.
+
+and one high-level piece:
+
+   - \refdir{core/or} -- Implements onion routing itself.
+
diff --git a/src/core/crypto/.may_include b/src/core/crypto/.may_include
new file mode 100644
index 0000000000..5782a36797
--- /dev/null
+++ b/src/core/crypto/.may_include
@@ -0,0 +1,10 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/cc/*.h
+lib/log/*.h
+
+core/crypto/*.h
diff --git a/src/core/crypto/core_crypto.md b/src/core/crypto/core_crypto.md
new file mode 100644
index 0000000000..26ade1f8f8
--- /dev/null
+++ b/src/core/crypto/core_crypto.md
@@ -0,0 +1,6 @@
+@dir /core/crypto
+@brief core/crypto: Tor-specific cryptography
+
+This module implements Tor's circuit-construction crypto and Tor's
+relay crypto.
+
diff --git a/src/core/crypto/hs_ntor.c b/src/core/crypto/hs_ntor.c
index c34073690e..07bcdc566c 100644
--- a/src/core/crypto/hs_ntor.c
+++ b/src/core/crypto/hs_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /** \file hs_ntor.c
@@ -170,19 +170,18 @@ get_rendezvous1_key_material(const uint8_t *rend_secret_hs_input,
  * necessary key material, and return 0. */
 static void
 get_introduce1_key_material(const uint8_t *secret_input,
-                        const uint8_t *subcredential,
+                        const hs_subcredential_t *subcredential,
                         hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
 {
   uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN];
   uint8_t info_blob[INFO_BLOB_LEN];
   uint8_t kdf_input[KDF_INPUT_LEN];
-  crypto_xof_t *xof;
   uint8_t *ptr;
 
   /* Let's build info */
   ptr = info_blob;
   APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
-  APPEND(ptr, subcredential, DIGEST256_LEN);
+  APPEND(ptr, subcredential->subcred, SUBCRED_LEN);
   tor_assert(ptr == info_blob + sizeof(info_blob));
 
   /* Let's build the input to the KDF */
@@ -193,10 +192,8 @@ get_introduce1_key_material(const uint8_t *secret_input,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Now we need to run kdf_input over SHAKE-256 */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ;
-  crypto_xof_free(xof);
+  crypto_xof(keystream, sizeof(keystream),
+             kdf_input, sizeof(kdf_input));
 
   { /* Get the keys */
     memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN);
@@ -320,7 +317,7 @@ hs_ntor_client_get_introduce1_keys(
                       const ed25519_public_key_t *intro_auth_pubkey,
                       const curve25519_public_key_t *intro_enc_pubkey,
                       const curve25519_keypair_t *client_ephemeral_enc_keypair,
-                      const uint8_t *subcredential,
+                      const hs_subcredential_t *subcredential,
                       hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
 {
   int bad = 0;
@@ -453,9 +450,31 @@ hs_ntor_service_get_introduce1_keys(
                     const ed25519_public_key_t *intro_auth_pubkey,
                     const curve25519_keypair_t *intro_enc_keypair,
                     const curve25519_public_key_t *client_ephemeral_enc_pubkey,
-                    const uint8_t *subcredential,
+                    const hs_subcredential_t *subcredential,
                     hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
 {
+  return hs_ntor_service_get_introduce1_keys_multi(
+                             intro_auth_pubkey,
+                             intro_enc_keypair,
+                             client_ephemeral_enc_pubkey,
+                             1,
+                             subcredential,
+                             hs_ntor_intro_cell_keys_out);
+}
+
+/**
+ * As hs_ntor_service_get_introduce1_keys(), but take multiple subcredentials
+ * as input, and yield multiple sets of keys as output.
+ **/
+int
+hs_ntor_service_get_introduce1_keys_multi(
+            const struct ed25519_public_key_t *intro_auth_pubkey,
+            const struct curve25519_keypair_t *intro_enc_keypair,
+            const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
+            size_t n_subcredentials,
+            const hs_subcredential_t *subcredentials,
+            hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
   int bad = 0;
   uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
   uint8_t dh_result[CURVE25519_OUTPUT_LEN];
@@ -463,7 +482,8 @@ hs_ntor_service_get_introduce1_keys(
   tor_assert(intro_auth_pubkey);
   tor_assert(intro_enc_keypair);
   tor_assert(client_ephemeral_enc_pubkey);
-  tor_assert(subcredential);
+  tor_assert(n_subcredentials >= 1);
+  tor_assert(subcredentials);
   tor_assert(hs_ntor_intro_cell_keys_out);
 
   /* Compute EXP(X, b) */
@@ -479,13 +499,16 @@ hs_ntor_service_get_introduce1_keys(
                             secret_input);
   bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
 
-  /* Get ENC_KEY and MAC_KEY! */
-  get_introduce1_key_material(secret_input, subcredential,
-                              hs_ntor_intro_cell_keys_out);
+  for (unsigned i = 0; i < n_subcredentials; ++i) {
+    /* Get ENC_KEY and MAC_KEY! */
+    get_introduce1_key_material(secret_input, &subcredentials[i],
+                                &hs_ntor_intro_cell_keys_out[i]);
+  }
 
   memwipe(secret_input,  0, sizeof(secret_input));
   if (bad) {
-    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+    memwipe(hs_ntor_intro_cell_keys_out, 0,
+            sizeof(hs_ntor_intro_cell_keys_t) * n_subcredentials);
   }
 
   return bad ? -1 : 0;
@@ -594,7 +617,6 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
 {
   uint8_t *ptr;
   uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN];
-  crypto_xof_t *xof;
 
   /* Sanity checks on lengths to make sure we are good */
   if (BUG(seed_len != DIGEST256_LEN)) {
@@ -611,10 +633,8 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Generate the keys */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN);
-  crypto_xof_free(xof);
+  crypto_xof(keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN,
+             kdf_input, sizeof(kdf_input));
 
   return 0;
 }
diff --git a/src/core/crypto/hs_ntor.h b/src/core/crypto/hs_ntor.h
index e5a5171915..9a975dd83f 100644
--- a/src/core/crypto/hs_ntor.h
+++ b/src/core/crypto/hs_ntor.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file hs_ntor.h
+ * @brief Header for hs_ntor.c
+ **/
+
 #ifndef TOR_HS_NTOR_H
 #define TOR_HS_NTOR_H
 
@@ -14,7 +19,7 @@ struct curve25519_keypair_t;
   (DIGEST256_LEN*2 + CIPHER256_KEY_LEN*2)
 
 /* Key material needed to encode/decode INTRODUCE1 cells */
-typedef struct {
+typedef struct hs_ntor_intro_cell_keys_t {
   /* Key used for encryption of encrypted INTRODUCE1 blob */
   uint8_t enc_key[CIPHER256_KEY_LEN];
   /* MAC key used to protect encrypted INTRODUCE1 blob */
@@ -22,7 +27,7 @@ typedef struct {
 } hs_ntor_intro_cell_keys_t;
 
 /* Key material needed to encode/decode RENDEZVOUS1 cells */
-typedef struct {
+typedef struct hs_ntor_rend_cell_keys_t {
   /* This is the MAC of the HANDSHAKE_INFO field */
   uint8_t rend_cell_auth_mac[DIGEST256_LEN];
   /* This is the key seed used to derive further rendezvous crypto keys as
@@ -30,11 +35,20 @@ typedef struct {
   uint8_t ntor_key_seed[DIGEST256_LEN];
 } hs_ntor_rend_cell_keys_t;
 
+#define SUBCRED_LEN DIGEST256_LEN
+
+/**
+ * A 'subcredential' used to prove knowledge of a hidden service.
+ **/
+typedef struct hs_subcredential_t {
+  uint8_t subcred[SUBCRED_LEN];
+} hs_subcredential_t;
+
 int hs_ntor_client_get_introduce1_keys(
               const struct ed25519_public_key_t *intro_auth_pubkey,
               const struct curve25519_public_key_t *intro_enc_pubkey,
               const struct curve25519_keypair_t *client_ephemeral_enc_keypair,
-              const uint8_t *subcredential,
+              const hs_subcredential_t *subcredential,
               hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
 
 int hs_ntor_client_get_rendezvous1_keys(
@@ -44,11 +58,19 @@ int hs_ntor_client_get_rendezvous1_keys(
           const struct curve25519_public_key_t *service_ephemeral_rend_pubkey,
           hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
 
+int hs_ntor_service_get_introduce1_keys_multi(
+            const struct ed25519_public_key_t *intro_auth_pubkey,
+            const struct curve25519_keypair_t *intro_enc_keypair,
+            const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
+            size_t n_subcredentials,
+            const hs_subcredential_t *subcredentials,
+            hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
 int hs_ntor_service_get_introduce1_keys(
             const struct ed25519_public_key_t *intro_auth_pubkey,
             const struct curve25519_keypair_t *intro_enc_keypair,
             const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
-            const uint8_t *subcredential,
+            const hs_subcredential_t *subcredential,
             hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
 
 int hs_ntor_service_get_rendezvous1_keys(
diff --git a/src/core/crypto/include.am b/src/core/crypto/include.am
new file mode 100644
index 0000000000..28b7e22905
--- /dev/null
+++ b/src/core/crypto/include.am
@@ -0,0 +1,18 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/crypto/hs_ntor.c		\
+	src/core/crypto/onion_crypto.c		\
+	src/core/crypto/onion_fast.c		\
+	src/core/crypto/onion_ntor.c		\
+	src/core/crypto/onion_tap.c		\
+	src/core/crypto/relay_crypto.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/crypto/hs_ntor.h			\
+	src/core/crypto/onion_crypto.h	        	\
+	src/core/crypto/onion_fast.h			\
+	src/core/crypto/onion_ntor.h			\
+	src/core/crypto/onion_tap.h			\
+	src/core/crypto/relay_crypto.h
diff --git a/src/core/crypto/onion_crypto.c b/src/core/crypto/onion_crypto.c
index 56b02e2996..69b4dc40aa 100644
--- a/src/core/crypto/onion_crypto.c
+++ b/src/core/crypto/onion_crypto.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_crypto.h b/src/core/crypto/onion_crypto.h
index 1cddde3610..2665d326a3 100644
--- a/src/core/crypto/onion_crypto.h
+++ b/src/core/crypto/onion_crypto.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,4 +44,4 @@ void server_onion_keys_free_(server_onion_keys_t *keys);
 #define server_onion_keys_free(keys) \
   FREE_AND_NULL(server_onion_keys_t, server_onion_keys_free_, (keys))
 
-#endif
+#endif /* !defined(TOR_ONION_CRYPTO_H) */
diff --git a/src/core/crypto/onion_fast.c b/src/core/crypto/onion_fast.c
index 31bd20235f..d760549325 100644
--- a/src/core/crypto/onion_fast.c
+++ b/src/core/crypto/onion_fast.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_fast.h b/src/core/crypto/onion_fast.h
index 0ba8cbbc35..da983a56d9 100644
--- a/src/core/crypto/onion_fast.h
+++ b/src/core/crypto/onion_fast.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_ntor.c b/src/core/crypto/onion_ntor.c
index 7087fe1bd7..5a77230d02 100644
--- a/src/core/crypto/onion_ntor.c
+++ b/src/core/crypto/onion_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_ntor.h b/src/core/crypto/onion_ntor.h
index 51e72b4083..9473409e40 100644
--- a/src/core/crypto/onion_ntor.h
+++ b/src/core/crypto/onion_ntor.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file onion_ntor.h
+ * @brief Header for onion_ntor.c
+ **/
+
 #ifndef TOR_ONION_NTOR_H
 #define TOR_ONION_NTOR_H
 
diff --git a/src/core/crypto/onion_tap.c b/src/core/crypto/onion_tap.c
index 854889d88d..119f55f206 100644
--- a/src/core/crypto/onion_tap.c
+++ b/src/core/crypto/onion_tap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_tap.h b/src/core/crypto/onion_tap.h
index 0e43b9c8ba..78174b1fab 100644
--- a/src/core/crypto/onion_tap.h
+++ b/src/core/crypto/onion_tap.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c
index 0b83b2d0a5..3e6167e0e1 100644
--- a/src/core/crypto/relay_crypto.c
+++ b/src/core/crypto/relay_crypto.c
@@ -1,17 +1,24 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file relay_crypto.h
+ * @brief Header for relay_crypto.c
+ **/
+
 #include "core/or/or.h"
 #include "core/or/circuitlist.h"
+#include "core/or/crypt_path.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "core/crypto/hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
+#include "core/or/sendme.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
@@ -20,7 +27,7 @@
 /** Update digest from the payload of cell. Assign integrity part to
  * cell.
  */
-static void
+void
 relay_set_digest(crypto_digest_t *digest, cell_t *cell)
 {
   char integrity[4];
@@ -84,12 +91,39 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
  *
  * Note that we use the same operation for encrypting and for decrypting.
  */
-static void
+void
 relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in)
 {
   crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
 }
 
+/** Return the sendme_digest within the <b>crypto</b> object. */
+uint8_t *
+relay_crypto_get_sendme_digest(relay_crypto_t *crypto)
+{
+  tor_assert(crypto);
+  return crypto->sendme_digest;
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                  bool is_foward_digest)
+{
+  struct crypto_digest_t *digest;
+
+  tor_assert(crypto);
+
+  digest = crypto->b_digest;
+  if (is_foward_digest) {
+    digest = crypto->f_digest;
+  }
+
+  crypto_digest_get_digest(digest, (char *) crypto->sendme_digest,
+                           sizeof(crypto->sendme_digest));
+}
+
 /** Do the appropriate en/decryptions for <b>cell</b> arriving on
  * <b>circ</b> in direction <b>cell_direction</b>.
  *
@@ -134,12 +168,12 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell,
         tor_assert(thishop);
 
         /* decrypt one layer */
-        relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload);
+        cpath_crypt_cell(thishop, cell->payload, true);
 
         relay_header_unpack(&rh, cell->payload);
         if (rh.recognized == 0) {
           /* it's possibly recognized. have to check digest to be sure. */
-          if (relay_digest_matches(thishop->crypto.b_digest, cell)) {
+          if (relay_digest_matches(cpath_get_incoming_digest(thishop), cell)) {
             *recognized = 1;
             *layer_hint = thishop;
             return 0;
@@ -187,14 +221,17 @@ relay_encrypt_cell_outbound(cell_t *cell,
                             crypt_path_t *layer_hint)
 {
   crypt_path_t *thishop; /* counter for repeated crypts */
-  relay_set_digest(layer_hint->crypto.f_digest, cell);
+  cpath_set_cell_forward_digest(layer_hint, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(circ), layer_hint);
 
   thishop = layer_hint;
   /* moving from farthest to nearest hop */
   do {
     tor_assert(thishop);
     log_debug(LD_OR,"encrypting a layer of the relay cell.");
-    relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload);
+    cpath_crypt_cell(thishop, cell->payload, false);
 
     thishop = thishop->prev;
   } while (thishop != circ->cpath->prev);
@@ -212,6 +249,10 @@ relay_encrypt_cell_inbound(cell_t *cell,
                            or_circuit_t *or_circ)
 {
   relay_set_digest(or_circ->crypto.b_digest, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(or_circ), NULL);
+
   /* encrypt one layer */
   relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload);
 }
diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h
index 45a21d14ab..5e36c7678c 100644
--- a/src/core/crypto/relay_crypto.h
+++ b/src/core/crypto/relay_crypto.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,5 +27,16 @@ void relay_crypto_clear(relay_crypto_t *crypto);
 
 void relay_crypto_assert_ok(const relay_crypto_t *crypto);
 
+uint8_t *relay_crypto_get_sendme_digest(relay_crypto_t *crypto);
+
+void relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                       bool is_foward_digest);
+
+void
+relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in);
+
+void
+relay_set_digest(crypto_digest_t *digest, cell_t *cell);
+
 #endif /* !defined(TOR_RELAY_CRYPTO_H) */
 
diff --git a/src/core/include.am b/src/core/include.am
index 1b8ef2ac58..7752a7974b 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -1,174 +1,21 @@
 
-noinst_LIBRARIES += \
-	src/core/libtor-app.a
-if UNITTESTS_ENABLED
-noinst_LIBRARIES += \
-	src/core/libtor-app-testing.a
-endif
-
-LIBTOR_APP_A_SOURCES = 				\
-	src/app/config/config.c			\
-	src/app/config/confparse.c		\
-	src/app/config/statefile.c		\
-	src/app/main/main.c			\
-	src/core/crypto/hs_ntor.c		\
-	src/core/crypto/onion_crypto.c		\
-	src/core/crypto/onion_fast.c		\
-	src/core/crypto/onion_ntor.c		\
-	src/core/crypto/onion_tap.c		\
-	src/core/crypto/relay_crypto.c		\
-	src/core/mainloop/connection.c		\
-	src/core/mainloop/cpuworker.c		\
-	src/core/mainloop/mainloop.c		\
-	src/core/mainloop/netstatus.c		\
-	src/core/mainloop/periodic.c		\
-	src/core/or/address_set.c		\
-	src/core/or/channel.c			\
-	src/core/or/channelpadding.c		\
-	src/core/or/channeltls.c		\
-	src/core/or/circuitbuild.c		\
-	src/core/or/circuitlist.c		\
-	src/core/or/circuitmux.c		\
-	src/core/or/circuitmux_ewma.c		\
-	src/core/or/circuitstats.c		\
-	src/core/or/circuituse.c		\
-	src/core/or/command.c			\
-	src/core/or/connection_edge.c		\
-	src/core/or/connection_or.c		\
-	src/core/or/dos.c			\
-	src/core/or/onion.c			\
-	src/core/or/policies.c			\
-	src/core/or/protover.c			\
-	src/core/or/protover_rust.c		\
-	src/core/or/reasons.c			\
-	src/core/or/relay.c			\
-	src/core/or/scheduler.c			\
-	src/core/or/scheduler_kist.c		\
-	src/core/or/scheduler_vanilla.c		\
-	src/core/or/status.c			\
-	src/core/or/versions.c			\
-	src/core/proto/proto_cell.c		\
-	src/core/proto/proto_control0.c		\
-	src/core/proto/proto_ext_or.c		\
-	src/core/proto/proto_http.c		\
-	src/core/proto/proto_socks.c		\
-	src/feature/api/tor_api.c		\
-	src/feature/client/addressmap.c		\
-	src/feature/client/bridges.c		\
-	src/feature/client/circpathbias.c	\
-	src/feature/client/dnsserv.c		\
-	src/feature/client/entrynodes.c		\
-	src/feature/client/transports.c		\
-	src/feature/control/control.c		\
-	src/feature/control/fmt_serverstatus.c  \
-	src/feature/control/getinfo_geoip.c	\
-	src/feature/dirauth/keypin.c		\
-	src/feature/dircache/conscache.c	\
-	src/feature/dircache/consdiffmgr.c	\
-	src/feature/dircache/dircache.c		\
-	src/feature/dircache/dirserv.c		\
-	src/feature/dirclient/dirclient.c	\
-	src/feature/dirclient/dlstatus.c	\
-	src/feature/dircommon/consdiff.c	\
-	src/feature/dircommon/directory.c	\
-	src/feature/dircommon/fp_pair.c		\
-	src/feature/dircommon/voting_schedule.c	\
-	src/feature/dirparse/authcert_parse.c	\
-	src/feature/dirparse/microdesc_parse.c	\
-	src/feature/dirparse/ns_parse.c		\
-	src/feature/dirparse/parsecommon.c	\
-	src/feature/dirparse/policy_parse.c	\
-	src/feature/dirparse/routerparse.c	\
-	src/feature/dirparse/sigcommon.c	\
-	src/feature/dirparse/signing.c		\
-	src/feature/dirparse/unparseable.c	\
-	src/feature/hibernate/hibernate.c	\
-	src/feature/hs/hs_cache.c		\
-	src/feature/hs/hs_cell.c		\
-	src/feature/hs/hs_circuit.c		\
-	src/feature/hs/hs_circuitmap.c		\
-	src/feature/hs/hs_client.c		\
-	src/feature/hs/hs_common.c		\
-	src/feature/hs/hs_config.c		\
-	src/feature/hs/hs_control.c		\
-	src/feature/hs/hs_descriptor.c		\
-	src/feature/hs/hs_ident.c		\
-	src/feature/hs/hs_intropoint.c		\
-	src/feature/hs/hs_service.c		\
-	src/feature/hs/hs_stats.c		\
-	src/feature/hs_common/replaycache.c	\
-	src/feature/hs_common/shared_random_client.c	\
-	src/feature/keymgt/loadkey.c		\
-	src/feature/dirauth/keypin.c		\
-	src/feature/nodelist/authcert.c		\
-	src/feature/nodelist/describe.c		\
-	src/feature/nodelist/dirlist.c		\
-	src/feature/nodelist/microdesc.c	\
-	src/feature/nodelist/networkstatus.c	\
-	src/feature/nodelist/nickname.c		\
-	src/feature/nodelist/nodelist.c		\
-	src/feature/nodelist/node_select.c	\
-	src/feature/nodelist/routerinfo.c	\
-	src/feature/nodelist/routerlist.c	\
-	src/feature/nodelist/routerset.c	\
-	src/feature/nodelist/fmt_routerstatus.c	\
-	src/feature/nodelist/torcert.c		\
-	src/feature/relay/dns.c			\
-	src/feature/relay/ext_orport.c		\
-	src/feature/relay/onion_queue.c		\
-	src/feature/relay/router.c		\
-	src/feature/relay/routerkeys.c		\
-	src/feature/relay/routermode.c		\
-	src/feature/relay/selftest.c		\
-	src/feature/rend/rendcache.c		\
-	src/feature/rend/rendclient.c		\
-	src/feature/rend/rendcommon.c		\
-	src/feature/rend/rendmid.c		\
-	src/feature/rend/rendparse.c		\
-	src/feature/rend/rendservice.c		\
-	src/feature/stats/geoip_stats.c		\
-	src/feature/stats/rephist.c		\
-	src/feature/stats/predict_ports.c
-
-# These should eventually move into module_dirauth_sources, but for now
-# the separation is only in the code location.
-LIBTOR_APP_A_SOURCES += 			\
-	src/feature/dirauth/bwauth.c		\
-	src/feature/dirauth/dsigs_parse.c	\
-	src/feature/dirauth/guardfraction.c	\
-	src/feature/dirauth/reachability.c	\
-	src/feature/dirauth/recommend_pkg.c	\
-	src/feature/dirauth/process_descs.c	\
-	src/feature/dirauth/voteflags.c
-
-if BUILD_NT_SERVICES
-LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
-endif
-
 #
-# Modules are conditionnally compiled in tor starting here. We add the C files
+# Modules are conditionally compiled in tor starting here. We add the C files
 # only if the modules has been enabled at configure time. We always add the
 # source files of every module to libtor-testing.a so we can build the unit
 # tests for everything. See the UNITTESTS_ENABLED branch below.
 #
 LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES)
 
-# The Directory Authority module.
-MODULE_DIRAUTH_SOURCES = 					\
-	src/feature/dirauth/authmode.c				\
-	src/feature/dirauth/dircollate.c			\
-	src/feature/dirauth/dirvote.c				\
-	src/feature/dirauth/shared_random.c			\
-	src/feature/dirauth/shared_random_state.c
-
-if BUILD_MODULE_DIRAUTH
-LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
-endif
+src_core_libtor_app_a_SOURCES = \
+	$(LIBTOR_APP_A_SOURCES) \
+	$(LIBTOR_APP_A_STUB_SOURCES)
 
-src_core_libtor_app_a_SOURCES = $(LIBTOR_APP_A_SOURCES)
 if UNITTESTS_ENABLED
 
 # Add the sources of the modules that are needed for tests to work here.
+LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_RELAY_SOURCES)
+LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_DIRCACHE_SOURCES)
 LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
 
 src_core_libtor_app_testing_a_SOURCES = $(LIBTOR_APP_TESTING_A_SOURCES)
@@ -183,204 +30,6 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\""		\
 src_core_libtor_app_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_core_libtor_app_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
-noinst_HEADERS +=					\
-	src/app/config/config.h				\
-	src/app/config/confparse.h			\
-	src/app/config/or_options_st.h			\
-	src/app/config/or_state_st.h			\
-	src/app/config/statefile.h			\
-	src/app/main/main.h				\
-	src/app/main/ntmain.h				\
-	src/core/crypto/hs_ntor.h			\
-	src/core/crypto/onion_crypto.h	        	\
-	src/core/crypto/onion_fast.h			\
-	src/core/crypto/onion_ntor.h			\
-	src/core/crypto/onion_tap.h			\
-	src/core/crypto/relay_crypto.h			\
-	src/core/mainloop/connection.h			\
-	src/core/mainloop/cpuworker.h			\
-	src/core/mainloop/mainloop.h			\
-	src/core/mainloop/netstatus.h			\
-	src/core/mainloop/periodic.h			\
-	src/core/or/addr_policy_st.h			\
-	src/core/or/address_set.h			\
-	src/core/or/cell_queue_st.h			\
-	src/core/or/cell_st.h				\
-	src/core/or/channel.h				\
-	src/core/or/channelpadding.h			\
-	src/core/or/channeltls.h			\
-	src/core/or/circuit_st.h			\
-	src/core/or/circuitbuild.h			\
-	src/core/or/circuitlist.h			\
-	src/core/or/circuitmux.h			\
-	src/core/or/circuitmux_ewma.h			\
-	src/core/or/circuitstats.h			\
-	src/core/or/circuituse.h			\
-	src/core/or/command.h				\
-	src/core/or/connection_edge.h			\
-	src/core/or/connection_or.h			\
-	src/core/or/connection_st.h			\
-	src/core/or/cpath_build_state_st.h		\
-	src/core/or/crypt_path_reference_st.h		\
-	src/core/or/crypt_path_st.h			\
-	src/core/or/destroy_cell_queue_st.h		\
-	src/core/or/dos.h				\
-	src/core/or/edge_connection_st.h		\
-	src/core/or/half_edge_st.h		\
-	src/core/or/entry_connection_st.h		\
-	src/core/or/entry_port_cfg_st.h			\
-	src/core/or/extend_info_st.h			\
-	src/core/or/listener_connection_st.h		\
-	src/core/or/onion.h				\
-	src/core/or/or.h				\
-	src/core/or/or_circuit_st.h			\
-	src/core/or/or_connection_st.h			\
-	src/core/or/or_handshake_certs_st.h		\
-	src/core/or/or_handshake_state_st.h		\
-	src/core/or/origin_circuit_st.h			\
-	src/core/or/policies.h				\
-	src/core/or/port_cfg_st.h			\
-	src/core/or/protover.h				\
-	src/core/or/reasons.h				\
-	src/core/or/relay.h				\
-	src/core/or/relay_crypto_st.h			\
-	src/core/or/scheduler.h				\
-	src/core/or/server_port_cfg_st.h		\
-	src/core/or/socks_request_st.h			\
-	src/core/or/status.h				\
-	src/core/or/tor_version_st.h			\
-	src/core/or/var_cell_st.h			\
-	src/core/or/versions.h				\
-	src/core/proto/proto_cell.h			\
-	src/core/proto/proto_control0.h			\
-	src/core/proto/proto_ext_or.h			\
-	src/core/proto/proto_http.h			\
-	src/core/proto/proto_socks.h			\
-	src/feature/api/tor_api_internal.h		\
-	src/feature/client/addressmap.h			\
-	src/feature/client/bridges.h			\
-	src/feature/client/circpathbias.h		\
-	src/feature/client/dnsserv.h			\
-	src/feature/client/entrynodes.h			\
-	src/feature/client/transports.h			\
-	src/feature/control/control.h			\
-	src/feature/control/control_connection_st.h	\
-	src/feature/control/fmt_serverstatus.h		\
-	src/feature/control/getinfo_geoip.h		\
-	src/feature/dirauth/authmode.h			\
-	src/feature/dirauth/bwauth.h			\
-	src/feature/dirauth/dircollate.h		\
-	src/feature/dirauth/dirvote.h			\
-	src/feature/dirauth/dsigs_parse.h		\
-	src/feature/dirauth/guardfraction.h		\
-	src/feature/dirauth/keypin.h			\
-	src/feature/dirauth/ns_detached_signatures_st.h	\
-	src/feature/dirauth/reachability.h		\
-	src/feature/dirauth/recommend_pkg.h		\
-	src/feature/dirauth/process_descs.h		\
-	src/feature/dirauth/shared_random.h		\
-	src/feature/dirauth/shared_random_state.h	\
-	src/feature/dirauth/vote_microdesc_hash_st.h	\
-	src/feature/dirauth/voteflags.h			\
-	src/feature/dircache/cached_dir_st.h		\
-	src/feature/dircache/conscache.h		\
-	src/feature/dircache/consdiffmgr.h		\
-	src/feature/dircache/dircache.h			\
-	src/feature/dircache/dirserv.h			\
-	src/feature/dirclient/dir_server_st.h		\
-	src/feature/dirclient/dirclient.h		\
-	src/feature/dirclient/dlstatus.h		\
-	src/feature/dirclient/download_status_st.h	\
-	src/feature/dircommon/consdiff.h		\
-	src/feature/dircommon/dir_connection_st.h	\
-	src/feature/dircommon/directory.h		\
-	src/feature/dircommon/fp_pair.h			\
-	src/feature/dircommon/vote_timing_st.h		\
-	src/feature/dircommon/voting_schedule.h		\
-	src/feature/dirparse/authcert_members.i		\
-	src/feature/dirparse/authcert_parse.h		\
-	src/feature/dirparse/microdesc_parse.h		\
-	src/feature/dirparse/ns_parse.h			\
-	src/feature/dirparse/parsecommon.h		\
-	src/feature/dirparse/policy_parse.h		\
-	src/feature/dirparse/routerparse.h		\
-	src/feature/dirparse/sigcommon.h		\
-	src/feature/dirparse/signing.h			\
-	src/feature/dirparse/unparseable.h		\
-	src/feature/hibernate/hibernate.h		\
-	src/feature/hs/hs_cache.h			\
-	src/feature/hs/hs_cell.h			\
-	src/feature/hs/hs_circuit.h			\
-	src/feature/hs/hs_circuitmap.h			\
-	src/feature/hs/hs_client.h			\
-	src/feature/hs/hs_common.h			\
-	src/feature/hs/hs_config.h			\
-	src/feature/hs/hs_control.h			\
-	src/feature/hs/hs_descriptor.h			\
-	src/feature/hs/hs_ident.h			\
-	src/feature/hs/hs_intropoint.h			\
-	src/feature/hs/hs_service.h			\
-	src/feature/hs/hs_stats.h			\
-	src/feature/hs/hsdir_index_st.h			\
-	src/feature/hs_common/replaycache.h		\
-	src/feature/hs_common/shared_random_client.h	\
-	src/feature/keymgt/loadkey.h			\
-	src/feature/nodelist/authcert.h			\
-	src/feature/nodelist/authority_cert_st.h	\
-	src/feature/nodelist/describe.h			\
-	src/feature/nodelist/desc_store_st.h		\
-	src/feature/nodelist/dirlist.h			\
-	src/feature/nodelist/document_signature_st.h	\
-	src/feature/nodelist/extrainfo_st.h		\
-	src/feature/nodelist/microdesc.h		\
-	src/feature/nodelist/microdesc_st.h		\
-	src/feature/nodelist/networkstatus.h		\
-	src/feature/nodelist/networkstatus_sr_info_st.h	\
-	src/feature/nodelist/networkstatus_st.h		\
-	src/feature/nodelist/networkstatus_voter_info_st.h	\
-	src/feature/nodelist/nickname.h			\
-	src/feature/nodelist/node_st.h			\
-	src/feature/nodelist/nodelist.h			\
-	src/feature/nodelist/node_select.h		\
-	src/feature/nodelist/routerinfo.h		\
-	src/feature/nodelist/routerinfo_st.h		\
-	src/feature/nodelist/routerlist.h		\
-	src/feature/nodelist/routerlist_st.h		\
-	src/feature/nodelist/routerset.h		\
-	src/feature/nodelist/fmt_routerstatus.h		\
-	src/feature/nodelist/routerstatus_st.h		\
-	src/feature/nodelist/signed_descriptor_st.h	\
-	src/feature/nodelist/torcert.h			\
-	src/feature/nodelist/vote_routerstatus_st.h	\
-	src/feature/relay/dns.h				\
-	src/feature/relay/dns_structs.h			\
-	src/feature/relay/ext_orport.h			\
-	src/feature/relay/onion_queue.h			\
-	src/feature/relay/router.h			\
-	src/feature/relay/routerkeys.h			\
-	src/feature/relay/routermode.h			\
-	src/feature/relay/selftest.h			\
-	src/feature/rend/rend_authorized_client_st.h	\
-	src/feature/rend/rend_encoded_v2_service_descriptor_st.h	\
-	src/feature/rend/rend_intro_point_st.h		\
-	src/feature/rend/rend_service_descriptor_st.h	\
-	src/feature/rend/rendcache.h			\
-	src/feature/rend/rendclient.h			\
-	src/feature/rend/rendcommon.h			\
-	src/feature/rend/rendmid.h			\
-	src/feature/rend/rendparse.h			\
-	src/feature/rend/rendservice.h			\
-	src/feature/stats/geoip_stats.h			\
-	src/feature/stats/rephist.h			\
-	src/feature/stats/predict_ports.h
-
-noinst_HEADERS +=			\
-	src/app/config/auth_dirs.inc	\
-	src/app/config/fallback_dirs.inc
-
-# This may someday want to be an installed file?
-noinst_HEADERS += src/feature/api/tor_api.h
-
 micro-revision.i: FORCE
 	$(AM_V_at)rm -f micro-revision.tmp; \
 	if test -r "$(top_srcdir)/.git" && \
diff --git a/src/core/mainloop/.may_include b/src/core/mainloop/.may_include
new file mode 100644
index 0000000000..8e01cf910e
--- /dev/null
+++ b/src/core/mainloop/.may_include
@@ -0,0 +1,24 @@
+!advisory
+
+orconfig.h
+
+lib/conf/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/evloop/*.h
+lib/pubsub/*.h
+lib/subsys/*.h
+lib/buf/*.h
+lib/crypt_ops/*.h
+lib/err/*.h
+lib/tls/*.h
+lib/net/*.h
+lib/evloop/*.h
+lib/geoip/*.h
+lib/sandbox/*.h
+lib/smartlist_core/*.h
+lib/compress/*.h
+lib/log/*.h
+
+core/mainloop/*.h
+core/mainloop/*.inc
\ No newline at end of file
diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c
index 21d4332758..36b2c6ef63 100644
--- a/src/core/mainloop/connection.c
+++ b/src/core/mainloop/connection.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -57,7 +57,7 @@
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/err/backtrace.h"
 
@@ -65,9 +65,9 @@
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
-#define TOR_CHANNEL_INTERNAL_
-#define CONNECTION_PRIVATE
+#define CHANNEL_OBJECT_PRIVATE
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/netstatus.h"
@@ -82,13 +82,17 @@
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
+#include "core/proto/proto_haproxy.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/dnsserv.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_config.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
@@ -105,6 +109,7 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/geoip/geoip.h"
 
+#include "lib/cc/ctassert.h"
 #include "lib/sandbox/sandbox.h"
 #include "lib/net/buffers_net.h"
 #include "lib/tls/tortls.h"
@@ -641,7 +646,7 @@ connection_free_minimal(connection_t *conn)
     }
   }
 
-  tor_free(conn->address);
+  tor_str_wipe_and_free(conn->address);
 
   if (connection_speaks_cells(conn)) {
     or_connection_t *or_conn = TO_OR_CONN(conn);
@@ -661,7 +666,7 @@ connection_free_minimal(connection_t *conn)
     }
     or_handshake_state_free(or_conn->handshake_state);
     or_conn->handshake_state = NULL;
-    tor_free(or_conn->nickname);
+    tor_str_wipe_and_free(or_conn->nickname);
     if (or_conn->chan) {
       /* Owww, this shouldn't happen, but... */
       channel_t *base_chan = TLS_CHAN_TO_BASE(or_conn->chan);
@@ -681,8 +686,8 @@ connection_free_minimal(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_AP) {
     entry_connection_t *entry_conn = TO_ENTRY_CONN(conn);
-    tor_free(entry_conn->chosen_exit_name);
-    tor_free(entry_conn->original_dest_address);
+    tor_str_wipe_and_free(entry_conn->chosen_exit_name);
+    tor_str_wipe_and_free(entry_conn->original_dest_address);
     if (entry_conn->socks_request)
       socks_request_free(entry_conn->socks_request);
     if (entry_conn->pending_optimistic_data) {
@@ -700,6 +705,7 @@ connection_free_minimal(connection_t *conn)
     control_connection_t *control_conn = TO_CONTROL_CONN(conn);
     tor_free(control_conn->safecookie_client_hash);
     tor_free(control_conn->incoming_cmd);
+    tor_free(control_conn->current_cmd);
     if (control_conn->ephemeral_onion_services) {
       SMARTLIST_FOREACH(control_conn->ephemeral_onion_services, char *, cp, {
         memwipe(cp, 0, strlen(cp));
@@ -719,11 +725,7 @@ connection_free_minimal(connection_t *conn)
     tor_free(dir_conn->requested_resource);
 
     tor_compress_free(dir_conn->compress_state);
-    if (dir_conn->spool) {
-      SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled,
-                        spooled_resource_free(spooled));
-      smartlist_free(dir_conn->spool);
-    }
+    dir_conn_clear_spool(dir_conn);
 
     rend_data_free(dir_conn->rend_data);
     hs_ident_dir_conn_free(dir_conn->hs_ident);
@@ -1199,7 +1201,7 @@ make_win32_socket_exclusive(tor_socket_t sock)
     return -1;
   }
   return 0;
-#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */
+#else /* !defined(SO_EXCLUSIVEADDRUSE) */
   (void) sock;
   return 0;
 #endif /* defined(SO_EXCLUSIVEADDRUSE) */
@@ -1470,6 +1472,20 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                tor_socket_strerror(tor_socket_errno(s)));
       goto err;
     }
+
+#ifndef __APPLE__
+    /* This code was introduced to help debug #28229. */
+    int value;
+    socklen_t len = sizeof(value);
+
+    if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) {
+      if (value == 0) {
+        log_err(LD_NET, "Could not listen on %s - "
+                        "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address);
+        goto err;
+      }
+    }
+#endif /* !defined(__APPLE__) */
 #endif /* defined(HAVE_SYS_UN_H) */
   } else {
     log_err(LD_BUG, "Got unexpected address family %d.",
@@ -1503,10 +1519,11 @@ connection_listener_new(const struct sockaddr *listensockaddr,
     }
   }
 
+  /* Force IPv4 and IPv6 traffic on for non-SOCKSPorts.
+   * Forcing options on isn't a good idea, see #32994 and #33607. */
   if (type != CONN_TYPE_AP_LISTENER) {
     lis_conn->entry_cfg.ipv4_traffic = 1;
     lis_conn->entry_cfg.ipv6_traffic = 1;
-    lis_conn->entry_cfg.prefer_ipv6 = 0;
   }
 
   if (connection_add(conn) < 0) { /* no space, forget it */
@@ -1648,7 +1665,7 @@ check_sockaddr(const struct sockaddr *sa, int len, int level)
              len,(int)sizeof(struct sockaddr_in6));
       ok = 0;
     }
-    if (tor_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
+    if (fast_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
         sin6->sin6_port == 0) {
       log_fn(level, LD_NET,
              "Address for new connection has address/port equal to zero.");
@@ -1871,7 +1888,7 @@ connection_init_accepted_conn(connection_t *conn,
       /* Initiate Extended ORPort authentication. */
       return connection_ext_or_start_auth(TO_OR_CONN(conn));
     case CONN_TYPE_OR:
-      control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
+      connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
       rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1);
       if (rv < 0) {
         connection_or_close_for_error(TO_OR_CONN(conn), 0);
@@ -1884,11 +1901,16 @@ connection_init_accepted_conn(connection_t *conn,
       TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch();
       TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type;
 
+      /* Any incoming connection on an entry port counts as user activity. */
+      note_user_activity(approx_time());
+
       switch (TO_CONN(listener)->type) {
         case CONN_TYPE_AP_LISTENER:
           conn->state = AP_CONN_STATE_SOCKS_WAIT;
           TO_ENTRY_CONN(conn)->socks_request->socks_prefer_no_auth =
             listener->entry_cfg.socks_prefer_no_auth;
+          TO_ENTRY_CONN(conn)->socks_request->socks_use_extended_errors =
+            listener->entry_cfg.extended_socks5_codes;
           break;
         case CONN_TYPE_AP_TRANS_LISTENER:
           TO_ENTRY_CONN(conn)->is_transparent_ap = 1;
@@ -2083,6 +2105,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn)
     return;
   }
 
+  if (fascist_firewall_use_ipv6(options)) {
+    log_info(LD_NET, "Our outgoing connection is using IPv%d.",
+             tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4);
+  }
+
   /* Check if we couldn't satisfy an address family preference */
   if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6)
       || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) {
@@ -2261,9 +2288,12 @@ connection_proxy_state_to_string(int state)
     "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929",
     "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK",
     "PROXY_SOCKS5_WANT_CONNECT_OK",
+    "PROXY_HAPROXY_WAIT_FOR_FLUSH",
     "PROXY_CONNECTED",
   };
 
+  CTASSERT(ARRAY_LENGTH(states) == PROXY_CONNECTED+1);
+
   if (state < PROXY_NONE || state > PROXY_CONNECTED)
     return unknown;
 
@@ -2296,7 +2326,11 @@ conn_get_proxy_type(const connection_t *conn)
     return PROXY_SOCKS4;
   else if (options->Socks5Proxy)
     return PROXY_SOCKS5;
-  else
+  else if (options->TCPProxy) {
+    /* The only supported protocol in TCPProxy is haproxy. */
+    tor_assert(options->TCPProxyProtocol == TCP_PROXY_PROTOCOL_HAPROXY);
+    return PROXY_HAPROXY;
+  } else
     return PROXY_NONE;
 }
 
@@ -2305,165 +2339,245 @@ conn_get_proxy_type(const connection_t *conn)
    username NUL: */
 #define SOCKS4_STANDARD_BUFFER_SIZE (1 + 1 + 2 + 4 + 1)
 
-/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn
- * for conn->addr:conn->port, authenticating with the auth details given
- * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies
- * support authentication.
+/** Write a proxy request of https to conn for conn->addr:conn->port,
+ * authenticating with the auth details given in the configuration
+ * (if available).
  *
  * Returns -1 if conn->addr is incompatible with the proxy protocol, and
  * 0 otherwise.
- *
- * Use connection_read_proxy_handshake() to complete the handshake.
  */
-int
-connection_proxy_connect(connection_t *conn, int type)
+static int
+connection_https_proxy_connect(connection_t *conn)
 {
-  const or_options_t *options;
+  tor_assert(conn);
+
+  const or_options_t *options = get_options();
+  char buf[1024];
+  char *base64_authenticator = NULL;
+  const char *authenticator = options->HTTPSProxyAuthenticator;
+
+  /* Send HTTP CONNECT and authentication (if available) in
+   * one request */
+
+  if (authenticator) {
+    base64_authenticator = alloc_http_authenticator(authenticator);
+    if (!base64_authenticator)
+      log_warn(LD_OR, "Encoding https authenticator failed");
+  }
+
+  if (base64_authenticator) {
+    const char *addrport = fmt_addrport(&conn->addr, conn->port);
+    tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.1\r\n"
+        "Host: %s\r\n"
+        "Proxy-Authorization: Basic %s\r\n\r\n",
+        addrport,
+        addrport,
+        base64_authenticator);
+    tor_free(base64_authenticator);
+  } else {
+    tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.0\r\n\r\n",
+        fmt_addrport(&conn->addr, conn->port));
+  }
+
+  connection_buf_add(buf, strlen(buf), conn);
+  conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK;
+
+  return 0;
+}
 
+/** Write a proxy request of socks4 to conn for conn->addr:conn->port.
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ */
+static int
+connection_socks4_proxy_connect(connection_t *conn)
+{
   tor_assert(conn);
 
-  options = get_options();
+  unsigned char *buf;
+  uint16_t portn;
+  uint32_t ip4addr;
+  size_t buf_size = 0;
+  char *socks_args_string = NULL;
 
-  switch (type) {
-    case PROXY_CONNECT: {
-      char buf[1024];
-      char *base64_authenticator=NULL;
-      const char *authenticator = options->HTTPSProxyAuthenticator;
-
-      /* Send HTTP CONNECT and authentication (if available) in
-       * one request */
-
-      if (authenticator) {
-        base64_authenticator = alloc_http_authenticator(authenticator);
-        if (!base64_authenticator)
-          log_warn(LD_OR, "Encoding https authenticator failed");
-      }
+  /* Send a SOCKS4 connect request */
 
-      if (base64_authenticator) {
-        const char *addrport = fmt_addrport(&conn->addr, conn->port);
-        tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.1\r\n"
-                     "Host: %s\r\n"
-                     "Proxy-Authorization: Basic %s\r\n\r\n",
-                     addrport,
-                     addrport,
-                     base64_authenticator);
-        tor_free(base64_authenticator);
-      } else {
-        tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.0\r\n\r\n",
-                     fmt_addrport(&conn->addr, conn->port));
-      }
+  if (tor_addr_family(&conn->addr) != AF_INET) {
+    log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6");
+    return -1;
+  }
 
-      connection_buf_add(buf, strlen(buf), conn);
-      conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK;
-      break;
+  { /* If we are here because we are trying to connect to a
+       pluggable transport proxy, check if we have any SOCKS
+       arguments to transmit. If we do, compress all arguments to
+       a single string in 'socks_args_string': */
+
+    if (conn_get_proxy_type(conn) == PROXY_PLUGGABLE) {
+      socks_args_string =
+        pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port);
+      if (socks_args_string)
+        log_debug(LD_NET, "Sending out '%s' as our SOCKS argument string.",
+            socks_args_string);
     }
+  }
 
-    case PROXY_SOCKS4: {
-      unsigned char *buf;
-      uint16_t portn;
-      uint32_t ip4addr;
-      size_t buf_size = 0;
-      char *socks_args_string = NULL;
+  { /* Figure out the buffer size we need for the SOCKS message: */
 
-      /* Send a SOCKS4 connect request */
+    buf_size = SOCKS4_STANDARD_BUFFER_SIZE;
 
-      if (tor_addr_family(&conn->addr) != AF_INET) {
-        log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6");
-        return -1;
-      }
+    /* If we have a SOCKS argument string, consider its size when
+       calculating the buffer size: */
+    if (socks_args_string)
+      buf_size += strlen(socks_args_string);
+  }
 
-      { /* If we are here because we are trying to connect to a
-           pluggable transport proxy, check if we have any SOCKS
-           arguments to transmit. If we do, compress all arguments to
-           a single string in 'socks_args_string': */
+  buf = tor_malloc_zero(buf_size);
 
-        if (conn_get_proxy_type(conn) == PROXY_PLUGGABLE) {
-          socks_args_string =
-            pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port);
-          if (socks_args_string)
-            log_debug(LD_NET, "Sending out '%s' as our SOCKS argument string.",
-                      socks_args_string);
-        }
-      }
+  ip4addr = tor_addr_to_ipv4n(&conn->addr);
+  portn = htons(conn->port);
 
-      { /* Figure out the buffer size we need for the SOCKS message: */
+  buf[0] = 4; /* version */
+  buf[1] = SOCKS_COMMAND_CONNECT; /* command */
+  memcpy(buf + 2, &portn, 2); /* port */
+  memcpy(buf + 4, &ip4addr, 4); /* addr */
+
+  /* Next packet field is the userid. If we have pluggable
+     transport SOCKS arguments, we have to embed them
+     there. Otherwise, we use an empty userid.  */
+  if (socks_args_string) { /* place the SOCKS args string: */
+    tor_assert(strlen(socks_args_string) > 0);
+    tor_assert(buf_size >=
+        SOCKS4_STANDARD_BUFFER_SIZE + strlen(socks_args_string));
+    strlcpy((char *)buf + 8, socks_args_string, buf_size - 8);
+    tor_free(socks_args_string);
+  } else {
+    buf[8] = 0; /* no userid */
+  }
 
-        buf_size = SOCKS4_STANDARD_BUFFER_SIZE;
+  connection_buf_add((char *)buf, buf_size, conn);
+  tor_free(buf);
 
-        /* If we have a SOCKS argument string, consider its size when
-           calculating the buffer size: */
-        if (socks_args_string)
-          buf_size += strlen(socks_args_string);
-      }
+  conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK;
+  return 0;
+}
 
-      buf = tor_malloc_zero(buf_size);
-
-      ip4addr = tor_addr_to_ipv4n(&conn->addr);
-      portn = htons(conn->port);
-
-      buf[0] = 4; /* version */
-      buf[1] = SOCKS_COMMAND_CONNECT; /* command */
-      memcpy(buf + 2, &portn, 2); /* port */
-      memcpy(buf + 4, &ip4addr, 4); /* addr */
-
-      /* Next packet field is the userid. If we have pluggable
-         transport SOCKS arguments, we have to embed them
-         there. Otherwise, we use an empty userid.  */
-      if (socks_args_string) { /* place the SOCKS args string: */
-        tor_assert(strlen(socks_args_string) > 0);
-        tor_assert(buf_size >=
-                   SOCKS4_STANDARD_BUFFER_SIZE + strlen(socks_args_string));
-        strlcpy((char *)buf + 8, socks_args_string, buf_size - 8);
-        tor_free(socks_args_string);
-      } else {
-        buf[8] = 0; /* no userid */
-      }
+/** Write a proxy request of socks5 to conn for conn->addr:conn->port,
+ * authenticating with the auth details given in the configuration
+ * (if available).
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ */
+static int
+connection_socks5_proxy_connect(connection_t *conn)
+{
+  tor_assert(conn);
 
-      connection_buf_add((char *)buf, buf_size, conn);
-      tor_free(buf);
+  const or_options_t *options = get_options();
+  unsigned char buf[4]; /* fields: vers, num methods, method list */
 
-      conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK;
-      break;
-    }
+  /* Send a SOCKS5 greeting (connect request must wait) */
 
-    case PROXY_SOCKS5: {
-      unsigned char buf[4]; /* fields: vers, num methods, method list */
+  buf[0] = 5; /* version */
 
-      /* Send a SOCKS5 greeting (connect request must wait) */
+  /* We have to use SOCKS5 authentication, if we have a
+     Socks5ProxyUsername or if we want to pass arguments to our
+     pluggable transport proxy: */
+  if ((options->Socks5ProxyUsername) ||
+      (conn_get_proxy_type(conn) == PROXY_PLUGGABLE &&
+       (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) {
+  /* number of auth methods */
+    buf[1] = 2;
+    buf[2] = 0x00; /* no authentication */
+    buf[3] = 0x02; /* rfc1929 Username/Passwd auth */
+    conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929;
+  } else {
+    buf[1] = 1;
+    buf[2] = 0x00; /* no authentication */
+    conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE;
+  }
 
-      buf[0] = 5; /* version */
+  connection_buf_add((char *)buf, 2 + buf[1], conn);
+  return 0;
+}
 
-      /* We have to use SOCKS5 authentication, if we have a
-         Socks5ProxyUsername or if we want to pass arguments to our
-         pluggable transport proxy: */
-      if ((options->Socks5ProxyUsername) ||
-          (conn_get_proxy_type(conn) == PROXY_PLUGGABLE &&
-           (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) {
-      /* number of auth methods */
-        buf[1] = 2;
-        buf[2] = 0x00; /* no authentication */
-        buf[3] = 0x02; /* rfc1929 Username/Passwd auth */
-        conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929;
-      } else {
-        buf[1] = 1;
-        buf[2] = 0x00; /* no authentication */
-        conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE;
-      }
+/** Write a proxy request of haproxy to conn for conn->addr:conn->port.
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ */
+static int
+connection_haproxy_proxy_connect(connection_t *conn)
+{
+  int ret = 0;
+  tor_addr_port_t *addr_port = tor_addr_port_new(&conn->addr, conn->port);
+  char *buf = haproxy_format_proxy_header_line(addr_port);
+
+  if (buf == NULL) {
+    ret = -1;
+    goto done;
+  }
+
+  connection_buf_add(buf, strlen(buf), conn);
+  /* In haproxy, we don't have to wait for the response, but we wait for ack.
+   * So we can set the state to be PROXY_HAPROXY_WAIT_FOR_FLUSH. */
+  conn->proxy_state = PROXY_HAPROXY_WAIT_FOR_FLUSH;
+
+  ret = 0;
+ done:
+  tor_free(buf);
+  tor_free(addr_port);
+  return ret;
+}
+
+/** Write a proxy request of <b>type</b> (socks4, socks5, https, haproxy)
+ * to conn for conn->addr:conn->port, authenticating with the auth details
+ * given in the configuration (if available). SOCKS 5 and HTTP CONNECT
+ * proxies support authentication.
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ *
+ * Use connection_read_proxy_handshake() to complete the handshake.
+ */
+int
+connection_proxy_connect(connection_t *conn, int type)
+{
+  int ret = 0;
+
+  tor_assert(conn);
+
+  switch (type) {
+    case PROXY_CONNECT:
+      ret = connection_https_proxy_connect(conn);
+      break;
 
-      connection_buf_add((char *)buf, 2 + buf[1], conn);
+    case PROXY_SOCKS4:
+      ret = connection_socks4_proxy_connect(conn);
+      break;
+
+    case PROXY_SOCKS5:
+      ret = connection_socks5_proxy_connect(conn);
+      break;
+
+    case PROXY_HAPROXY:
+      ret = connection_haproxy_proxy_connect(conn);
       break;
-    }
 
     default:
       log_err(LD_BUG, "Invalid proxy protocol, %d", type);
       tor_fragile_assert();
-      return -1;
+      ret = -1;
+      break;
   }
 
-  log_debug(LD_NET, "set state %s",
-            connection_proxy_state_to_string(conn->proxy_state));
+  if (ret == 0) {
+    log_debug(LD_NET, "set state %s",
+              connection_proxy_state_to_string(conn->proxy_state));
+  }
 
-  return 0;
+  return ret;
 }
 
 /** Read conn's inbuf. If the http response from the proxy is all
@@ -2841,7 +2955,7 @@ retry_listener_ports(smartlist_t *old_conns,
           SMARTLIST_DEL_CURRENT(old_conns, conn);
           break;
         }
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
       }
     } SMARTLIST_FOREACH_END(wanted);
 
@@ -2905,6 +3019,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
     retval = -1;
 
 #ifdef ENABLE_LISTENER_REBIND
+  if (smartlist_len(replacements))
+    log_debug(LD_NET, "%d replacements - starting rebinding loop.",
+              smartlist_len(replacements));
+
   SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) {
     int addr_in_use = 0;
     int skip = 0;
@@ -2916,8 +3034,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
       connection_listener_new_for_port(r->new_port, &skip, &addr_in_use);
     connection_t *old_conn = r->old_conn;
 
-    if (skip)
+    if (skip) {
+      log_debug(LD_NET, "Skipping creating new listener for %s:%d",
+                old_conn->address, old_conn->port);
       continue;
+    }
 
     connection_close_immediate(old_conn);
     connection_mark_for_close(old_conn);
@@ -2943,7 +3064,7 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
                conn_type_to_string(old_conn->type), old_conn->address,
                old_conn->port, new_conn->address, new_conn->port);
   } SMARTLIST_FOREACH_END(r);
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
 
   /* Any members that were still in 'listeners' don't correspond to
    * any configured port.  Kill 'em. */
@@ -3030,7 +3151,7 @@ connection_mark_all_noncontrol_connections(void)
  * uses pluggable transports, since we should then limit it even if it
  * comes from an internal IP address. */
 static int
-connection_is_rate_limited(connection_t *conn)
+connection_is_rate_limited(const connection_t *conn)
 {
   const or_options_t *options = get_options();
   if (conn->linked)
@@ -3166,14 +3287,14 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
                                      global_bucket_val, conn_bucket);
 }
 
-/** Return 1 if the global write buckets are low enough that we
+/** Return true iff the global write buckets are low enough that we
  * shouldn't send <b>attempt</b> bytes of low-priority directory stuff
- * out to <b>conn</b>. Else return 0.
-
- * Priority was 1 for v1 requests (directories and running-routers),
- * and 2 for v2 requests and later (statuses and descriptors).
+ * out to <b>conn</b>.
+ *
+ * If we are a directory authority, always answer dir requests thus true is
+ * always returned.
  *
- * There are a lot of parameters we could use here:
+ * Note: There are a lot of parameters we could use here:
  * - global_relayed_write_bucket. Low is bad.
  * - global_write_bucket. Low is bad.
  * - bandwidthrate. Low is bad.
@@ -3185,39 +3306,40 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
  *   mean is "total directory bytes added to outbufs recently", but
  *   that's harder to quantify and harder to keep track of.
  */
-int
-global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
+bool
+connection_dir_is_global_write_low(const connection_t *conn, size_t attempt)
 {
   size_t smaller_bucket =
     MIN(token_bucket_rw_get_write(&global_bucket),
         token_bucket_rw_get_write(&global_relayed_bucket));
-  if (authdir_mode(get_options()) && priority>1)
-    return 0; /* there's always room to answer v2 if we're an auth dir */
+
+  /* Special case for authorities (directory only). */
+  if (authdir_mode_v3(get_options())) {
+    /* Are we configured to possibly reject requests under load? */
+    if (!dirauth_should_reject_requests_under_load()) {
+      /* Answer request no matter what. */
+      return false;
+    }
+    /* Always answer requests from a known relay which includes the other
+     * authorities. The following looks up the addresses for relays that we
+     * have their descriptor _and_ any configured trusted directories. */
+    if (nodelist_probably_contains_address(&conn->addr)) {
+      return false;
+    }
+  }
 
   if (!connection_is_rate_limited(conn))
-    return 0; /* local conns don't get limited */
+    return false; /* local conns don't get limited */
 
   if (smaller_bucket < attempt)
-    return 1; /* not enough space no matter the priority */
+    return true; /* not enough space. */
 
   {
     const time_t diff = approx_time() - write_buckets_last_empty_at;
     if (diff <= 1)
-      return 1; /* we're already hitting our limits, no more please */
+      return true; /* we're already hitting our limits, no more please */
   }
-
-  if (priority == 1) { /* old-style v1 query */
-    /* Could we handle *two* of these requests within the next two seconds? */
-    const or_options_t *options = get_options();
-    size_t can_write = (size_t) (smaller_bucket
-      + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate :
-           options->BandwidthRate));
-    if (can_write < 2*attempt)
-      return 1;
-  } else { /* v2 query */
-    /* no further constraints yet */
-  }
-  return 0;
+  return false;
 }
 
 /** When did we last tell the accounting subsystem about transmitted
@@ -3239,8 +3361,17 @@ record_num_bytes_transferred_impl(connection_t *conn,
       rep_hist_note_dir_bytes_written(num_written, now);
   }
 
+  /* Linked connections and internal IPs aren't counted for statistics or
+   * accounting:
+   *  - counting linked connections would double-count BEGINDIR bytes, because
+   *    they are sent as Dir bytes on the linked connection, and OR bytes on
+   *    the OR connection;
+   *  - relays and clients don't connect to internal IPs, unless specifically
+   *    configured to do so. If they are configured that way, we don't count
+   *    internal bytes.
+   */
   if (!connection_is_rate_limited(conn))
-    return; /* local IPs are free */
+    return;
 
   if (conn->type == CONN_TYPE_OR)
     rep_hist_note_or_conn_bytes(conn->global_identifier, num_read,
@@ -3696,6 +3827,12 @@ connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read,
     at_most = connection_bucket_read_limit(conn, approx_time());
   }
 
+  /* Do not allow inbuf to grow past BUF_MAX_LEN. */
+  const ssize_t maximum = BUF_MAX_LEN - buf_datalen(conn->inbuf);
+  if (at_most > maximum) {
+    at_most = maximum;
+  }
+
   slack_in_buf = buf_slack(conn->inbuf);
  again:
   if ((size_t)at_most > slack_in_buf && slack_in_buf >= 1024) {
@@ -3943,9 +4080,9 @@ update_send_buffer_size(tor_socket_t sock)
       &isb, sizeof(isb), &bytesReturned, NULL, NULL)) {
     setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb));
   }
-#else
+#else /* !defined(_WIN32) */
   (void) sock;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Try to flush more bytes onto <b>conn</b>-\>s.
@@ -4080,6 +4217,7 @@ connection_handle_write_impl(connection_t *conn, int force)
     switch (result) {
       CASE_TOR_TLS_ERROR_ANY:
       case TOR_TLS_CLOSE:
+        or_conn->tls_error = result;
         log_info(LD_NET, result != TOR_TLS_CLOSE ?
                  "tls error. breaking.":"TLS connection closed on flush");
         /* Don't flush; connection is dead. */
@@ -4343,6 +4481,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
   connection_write_to_buf_commit(conn, written);
 }
 
+/**
+ * Write a <b>string</b> (of size <b>len</b> to directory connection
+ * <b>dir_conn</b>. Apply compression if connection is configured to use
+ * it and finalize it if <b>done</b> is true.
+ */
+void
+connection_dir_buf_add(const char *string, size_t len,
+                       dir_connection_t *dir_conn, int done)
+{
+  if (dir_conn->compress_state != NULL) {
+    connection_buf_add_compress(string, len, dir_conn, done);
+    return;
+  }
+
+  connection_buf_add(string, len, TO_CONN(dir_conn));
+}
+
 void
 connection_buf_add_compress(const char *string, size_t len,
                             dir_connection_t *conn, int done)
@@ -4457,6 +4612,16 @@ connection_get_by_type_state(int type, int state)
   CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state);
 }
 
+/**
+ * Return a connection of type <b>type</b> that is not an internally linked
+ * connection, and is not marked for close.
+ **/
+MOCK_IMPL(connection_t *,
+connection_get_by_type_nonlinked,(int type))
+{
+  CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked);
+}
+
 /** Return a connection of type <b>type</b> that has rendquery equal
  * to <b>rendquery</b>, and that is not marked for close. If state
  * is non-zero, conn must be of that state too.
@@ -4837,10 +5002,10 @@ connection_finished_flushing(connection_t *conn)
   }
 }
 
-/** Called when our attempt to connect() to another server has just
- * succeeded.
+/** Called when our attempt to connect() to a server has just succeeded.
  *
- * This function just passes conn to the connection-specific
+ * This function checks if the interface address has changed (clients only),
+ * and then passes conn to the connection-specific
  * connection_*_finished_connecting() function.
  */
 static int
@@ -5299,7 +5464,7 @@ assert_connection_ok(connection_t *conn, time_t now)
         tor_assert(entry_conn->socks_request->has_finished);
         if (!conn->marked_for_close) {
           tor_assert(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
-          assert_cpath_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
+          cpath_assert_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
         }
       }
     }
@@ -5353,17 +5518,20 @@ assert_connection_ok(connection_t *conn, time_t now)
 }
 
 /** Fills <b>addr</b> and <b>port</b> with the details of the global
- *  proxy server we are using.
- *  <b>conn</b> contains the connection we are using the proxy for.
+ *  proxy server we are using. Store a 1 to the int pointed to by
+ *  <b>is_put_out</b> if the connection is using a pluggable
+ *  transport; store 0 otherwise. <b>conn</b> contains the connection
+ *  we are using the proxy for.
  *
  *  Return 0 on success, -1 on failure.
  */
 int
 get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                   const connection_t *conn)
+                   int *is_pt_out, const connection_t *conn)
 {
   const or_options_t *options = get_options();
 
+  *is_pt_out = 0;
   /* Client Transport Plugins can use another proxy, but that should be hidden
    * from the rest of tor (as the plugin is responsible for dealing with the
    * proxy), check it first, then check the rest of the proxy types to allow
@@ -5379,6 +5547,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
       tor_addr_copy(addr, &transport->addr);
       *port = transport->port;
       *proxy_type = transport->socks_version;
+      *is_pt_out = 1;
       return 0;
     }
 
@@ -5400,6 +5569,13 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
     *port = options->Socks5ProxyPort;
     *proxy_type = PROXY_SOCKS5;
     return 0;
+  } else if (options->TCPProxy) {
+    tor_addr_copy(addr, &options->TCPProxyAddr);
+    *port = options->TCPProxyPort;
+    /* The only supported protocol in TCPProxy is haproxy. */
+    tor_assert(options->TCPProxyProtocol == TCP_PROXY_PROTOCOL_HAPROXY);
+    *proxy_type = PROXY_HAPROXY;
+    return 0;
   }
 
   tor_addr_make_unspec(addr);
@@ -5415,11 +5591,13 @@ log_failed_proxy_connection(connection_t *conn)
 {
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt;
 
-  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, conn) != 0)
+  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         conn) != 0)
     return; /* if we have no proxy set up, leave this function. */
 
+  (void)is_pt;
   log_warn(LD_NET,
            "The connection to the %s proxy server at %s just failed. "
            "Make sure that the proxy server is up and running.",
@@ -5435,6 +5613,7 @@ proxy_type_to_string(int proxy_type)
   case PROXY_CONNECT:   return "HTTP";
   case PROXY_SOCKS4:    return "SOCKS4";
   case PROXY_SOCKS5:    return "SOCKS5";
+  case PROXY_HAPROXY:   return "HAPROXY";
   case PROXY_PLUGGABLE: return "pluggable transports SOCKS";
   case PROXY_NONE:      return "NULL";
   default:              tor_assert(0);
diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h
index 8ecdd6f06f..bcd3d590a5 100644
--- a/src/core/mainloop/connection.h
+++ b/src/core/mainloop/connection.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,25 @@
 #ifndef TOR_CONNECTION_H
 #define TOR_CONNECTION_H
 
-listener_connection_t *TO_LISTENER_CONN(connection_t *);
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/log/log.h"
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+struct listener_connection_t;
+struct connection_t;
+struct dir_connection_t;
+struct or_connection_t;
+struct edge_connection_t;
+struct entry_connection_t;
+struct control_connection_t;
+struct port_cfg_t;
+struct tor_addr_t;
+struct or_options_t;
+
+struct listener_connection_t *TO_LISTENER_CONN(struct connection_t *);
 
 struct buf_t;
 
@@ -56,7 +74,7 @@ struct buf_t;
 
 #define CONN_TYPE_MAX_ 19
 /* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in
- * connection_t. */
+ * struct connection_t. */
 
 /* Proxy client handshake states */
 /* We use a proxy but we haven't even connected to it yet. */
@@ -75,8 +93,10 @@ struct buf_t;
 #define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 6
 /* We use a SOCKS5 proxy and we just sent our CONNECT command. */
 #define PROXY_SOCKS5_WANT_CONNECT_OK 7
+/* We use an HAPROXY proxy and we just sent the proxy header. */
+#define PROXY_HAPROXY_WAIT_FOR_FLUSH 8
 /* We use a proxy and we CONNECTed successfully!. */
-#define PROXY_CONNECTED 8
+#define PROXY_CONNECTED 9
 
 /** State for any listener connection. */
 #define LISTENER_STATE_READY 0
@@ -88,34 +108,36 @@ struct buf_t;
  */
 typedef struct
 {
-  connection_t *old_conn; /* Old listener connection to be replaced */
-  const port_cfg_t *new_port; /* New port configuration */
+  struct connection_t *old_conn; /* Old listener connection to be replaced */
+  const struct port_cfg_t *new_port; /* New port configuration */
 } listener_replacement_t;
 
 const char *conn_type_to_string(int type);
 const char *conn_state_to_string(int type, int state);
 int conn_listener_type_supports_af_unix(int type);
 
-dir_connection_t *dir_connection_new(int socket_family);
-or_connection_t *or_connection_new(int type, int socket_family);
-edge_connection_t *edge_connection_new(int type, int socket_family);
-entry_connection_t *entry_connection_new(int type, int socket_family);
-control_connection_t *control_connection_new(int socket_family);
-listener_connection_t *listener_connection_new(int type, int socket_family);
-connection_t *connection_new(int type, int socket_family);
-int connection_init_accepted_conn(connection_t *conn,
-                                  const listener_connection_t *listener);
-void connection_link_connections(connection_t *conn_a, connection_t *conn_b);
-MOCK_DECL(void,connection_free_,(connection_t *conn));
+struct dir_connection_t *dir_connection_new(int socket_family);
+struct or_connection_t *or_connection_new(int type, int socket_family);
+struct edge_connection_t *edge_connection_new(int type, int socket_family);
+struct entry_connection_t *entry_connection_new(int type, int socket_family);
+struct control_connection_t *control_connection_new(int socket_family);
+struct listener_connection_t *listener_connection_new(int type,
+                                                      int socket_family);
+struct connection_t *connection_new(int type, int socket_family);
+int connection_init_accepted_conn(struct connection_t *conn,
+                                const struct listener_connection_t *listener);
+void connection_link_connections(struct connection_t *conn_a,
+                                 struct connection_t *conn_b);
+MOCK_DECL(void,connection_free_,(struct connection_t *conn));
 #define connection_free(conn) \
-  FREE_AND_NULL(connection_t, connection_free_, (conn))
+  FREE_AND_NULL(struct connection_t, connection_free_, (conn))
 void connection_free_all(void);
-void connection_about_to_close_connection(connection_t *conn);
-void connection_close_immediate(connection_t *conn);
-void connection_mark_for_close_(connection_t *conn,
+void connection_about_to_close_connection(struct connection_t *conn);
+void connection_close_immediate(struct connection_t *conn);
+void connection_mark_for_close_(struct connection_t *conn,
                                 int line, const char *file);
 MOCK_DECL(void, connection_mark_for_close_internal_,
-          (connection_t *conn, int line, const char *file));
+          (struct connection_t *conn, int line, const char *file));
 
 #define connection_mark_for_close(c) \
   connection_mark_for_close_((c), __LINE__, SHORT_FILE__)
@@ -130,11 +152,11 @@ MOCK_DECL(void, connection_mark_for_close_internal_,
  * connection_or_notify_error()), or you actually are the
  * connection_or_close_for_error() or connection_or_close_normally function.
  * For all other cases, use connection_mark_and_flush() instead, which
- * checks for or_connection_t properly, instead.  See below.
+ * checks for struct or_connection_t properly, instead.  See below.
  */
 #define connection_mark_and_flush_internal_(c,line,file)                \
   do {                                                                  \
-    connection_t *tmp_conn__ = (c);                                     \
+    struct connection_t *tmp_conn__ = (c);                              \
     connection_mark_for_close_internal_(tmp_conn__, (line), (file));    \
     tmp_conn__->hold_open_until_flushed = 1;                            \
   } while (0)
@@ -147,7 +169,7 @@ MOCK_DECL(void, connection_mark_for_close_internal_,
  */
 #define connection_mark_and_flush_(c,line,file)                           \
   do {                                                                    \
-    connection_t *tmp_conn_ = (c);                                        \
+    struct connection_t *tmp_conn_ = (c);                                 \
     if (tmp_conn_->type == CONN_TYPE_OR) {                                \
       log_warn(LD_CHANNEL | LD_BUG,                                       \
                "Something tried to close (and flush) an or_connection_t"  \
@@ -164,13 +186,13 @@ MOCK_DECL(void, connection_mark_for_close_internal_,
 
 void connection_expire_held_open(void);
 
-int connection_connect(connection_t *conn, const char *address,
-                       const tor_addr_t *addr,
+int connection_connect(struct connection_t *conn, const char *address,
+                       const struct tor_addr_t *addr,
                        uint16_t port, int *socket_error);
 
 #ifdef HAVE_SYS_UN_H
 
-int connection_connect_unix(connection_t *conn, const char *socket_path,
+int connection_connect_unix(struct connection_t *conn, const char *socket_path,
                             int *socket_error);
 
 #endif /* defined(HAVE_SYS_UN_H) */
@@ -183,75 +205,86 @@ int connection_connect_unix(connection_t *conn, const char *socket_path,
     username and password fields. */
 #define MAX_SOCKS5_AUTH_SIZE_TOTAL 2*MAX_SOCKS5_AUTH_FIELD_SIZE
 
-int connection_proxy_connect(connection_t *conn, int type);
-int connection_read_proxy_handshake(connection_t *conn);
-void log_failed_proxy_connection(connection_t *conn);
-int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                       const connection_t *conn);
+int connection_proxy_connect(struct connection_t *conn, int type);
+int connection_read_proxy_handshake(struct connection_t *conn);
+void log_failed_proxy_connection(struct connection_t *conn);
+int get_proxy_addrport(struct tor_addr_t *addr, uint16_t *port,
+                       int *proxy_type,
+                       int *is_pt_out, const struct connection_t *conn);
 
-int retry_all_listeners(smartlist_t *new_conns,
+int retry_all_listeners(struct smartlist_t *new_conns,
                         int close_all_noncontrol);
 
 void connection_mark_all_noncontrol_listeners(void);
 void connection_mark_all_noncontrol_connections(void);
 
-ssize_t connection_bucket_write_limit(connection_t *conn, time_t now);
-int global_write_bucket_low(connection_t *conn, size_t attempt, int priority);
+ssize_t connection_bucket_write_limit(struct connection_t *conn, time_t now);
+bool connection_dir_is_global_write_low(const struct connection_t *conn,
+                                        size_t attempt);
 void connection_bucket_init(void);
-void connection_bucket_adjust(const or_options_t *options);
+void connection_bucket_adjust(const struct or_options_t *options);
 void connection_bucket_refill_all(time_t now,
                                   uint32_t now_ts);
-void connection_read_bw_exhausted(connection_t *conn, bool is_global_bw);
-void connection_write_bw_exhausted(connection_t *conn, bool is_global_bw);
-void connection_consider_empty_read_buckets(connection_t *conn);
-void connection_consider_empty_write_buckets(connection_t *conn);
-
-int connection_handle_read(connection_t *conn);
-
-int connection_buf_get_bytes(char *string, size_t len, connection_t *conn);
-int connection_buf_get_line(connection_t *conn, char *data,
-                                   size_t *data_len);
-int connection_fetch_from_buf_http(connection_t *conn,
+void connection_read_bw_exhausted(struct connection_t *conn,
+                                  bool is_global_bw);
+void connection_write_bw_exhausted(struct connection_t *conn,
+                                   bool is_global_bw);
+void connection_consider_empty_read_buckets(struct connection_t *conn);
+void connection_consider_empty_write_buckets(struct connection_t *conn);
+
+int connection_handle_read(struct connection_t *conn);
+
+int connection_buf_get_bytes(char *string, size_t len,
+                             struct connection_t *conn);
+int connection_buf_get_line(struct connection_t *conn, char *data,
+                            size_t *data_len);
+int connection_fetch_from_buf_http(struct connection_t *conn,
                                char **headers_out, size_t max_headerlen,
                                char **body_out, size_t *body_used,
                                size_t max_bodylen, int force_complete);
 
-int connection_wants_to_flush(connection_t *conn);
-int connection_outbuf_too_full(connection_t *conn);
-int connection_handle_write(connection_t *conn, int force);
-int connection_flush(connection_t *conn);
+int connection_wants_to_flush(struct connection_t *conn);
+int connection_outbuf_too_full(struct connection_t *conn);
+int connection_handle_write(struct connection_t *conn, int force);
+int connection_flush(struct connection_t *conn);
 
 MOCK_DECL(void, connection_write_to_buf_impl_,
-          (const char *string, size_t len, connection_t *conn, int zlib));
+          (const char *string, size_t len, struct connection_t *conn,
+           int zlib));
 /* DOCDOC connection_write_to_buf */
 static void connection_buf_add(const char *string, size_t len,
-                                    connection_t *conn);
+                                    struct connection_t *conn);
+void connection_dir_buf_add(const char *string, size_t len,
+                            struct dir_connection_t *dir_conn, int done);
 static inline void
-connection_buf_add(const char *string, size_t len, connection_t *conn)
+connection_buf_add(const char *string, size_t len, struct connection_t *conn)
 {
   connection_write_to_buf_impl_(string, len, conn, 0);
 }
 void connection_buf_add_compress(const char *string, size_t len,
-                                 dir_connection_t *conn, int done);
-void connection_buf_add_buf(connection_t *conn, struct buf_t *buf);
-
-size_t connection_get_inbuf_len(connection_t *conn);
-size_t connection_get_outbuf_len(connection_t *conn);
-connection_t *connection_get_by_global_id(uint64_t id);
-
-connection_t *connection_get_by_type(int type);
-MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type,
-                                                  const tor_addr_t *addr,
-                                                  uint16_t port, int purpose));
-connection_t *connection_get_by_type_state(int type, int state);
-connection_t *connection_get_by_type_state_rendquery(int type, int state,
+                                 struct dir_connection_t *conn, int done);
+void connection_buf_add_buf(struct connection_t *conn, struct buf_t *buf);
+
+size_t connection_get_inbuf_len(struct connection_t *conn);
+size_t connection_get_outbuf_len(struct connection_t *conn);
+struct connection_t *connection_get_by_global_id(uint64_t id);
+
+struct connection_t *connection_get_by_type(int type);
+MOCK_DECL(struct connection_t *,connection_get_by_type_nonlinked,(int type));
+MOCK_DECL(struct connection_t *,connection_get_by_type_addr_port_purpose,
+                                               (int type,
+                                                const struct tor_addr_t *addr,
+                                                uint16_t port, int purpose));
+struct connection_t *connection_get_by_type_state(int type, int state);
+struct connection_t *connection_get_by_type_state_rendquery(
+                                                     int type, int state,
                                                      const char *rendquery);
-smartlist_t *connection_list_by_type_state(int type, int state);
-smartlist_t *connection_list_by_type_purpose(int type, int purpose);
-smartlist_t *connection_dir_list_by_purpose_and_resource(
+struct smartlist_t *connection_list_by_type_state(int type, int state);
+struct smartlist_t *connection_list_by_type_purpose(int type, int purpose);
+struct smartlist_t *connection_dir_list_by_purpose_and_resource(
                                                   int purpose,
                                                   const char *resource);
-smartlist_t *connection_dir_list_by_purpose_resource_and_state(
+struct smartlist_t *connection_dir_list_by_purpose_resource_and_state(
                                                   int purpose,
                                                   const char *resource,
                                                   int state);
@@ -270,7 +303,7 @@ connection_dir_count_by_purpose_and_resource(
                                              int purpose,
                                              const char *resource)
 {
-  smartlist_t *conns = connection_dir_list_by_purpose_and_resource(
+  struct smartlist_t *conns = connection_dir_list_by_purpose_and_resource(
                                                                    purpose,
                                                                    resource);
   CONN_LEN_AND_FREE_TEMPLATE(conns);
@@ -284,7 +317,7 @@ connection_dir_count_by_purpose_resource_and_state(
                                                    const char *resource,
                                                    int state)
 {
-  smartlist_t *conns =
+  struct smartlist_t *conns =
     connection_dir_list_by_purpose_resource_and_state(
                                                       purpose,
                                                       resource,
@@ -294,26 +327,26 @@ connection_dir_count_by_purpose_resource_and_state(
 
 #undef CONN_LEN_AND_FREE_TEMPLATE
 
-int any_other_active_or_conns(const or_connection_t *this_conn);
+int any_other_active_or_conns(const struct or_connection_t *this_conn);
 
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
 #define connection_speaks_cells(conn) (((conn)->type == CONN_TYPE_OR) || 0)
-int connection_is_listener(connection_t *conn);
-int connection_state_is_open(connection_t *conn);
-int connection_state_is_connecting(connection_t *conn);
+int connection_is_listener(struct connection_t *conn);
+int connection_state_is_open(struct connection_t *conn);
+int connection_state_is_connecting(struct connection_t *conn);
 
 char *alloc_http_authenticator(const char *authenticator);
 
-void assert_connection_ok(connection_t *conn, time_t now);
-int connection_or_nonopen_was_started_here(or_connection_t *conn);
+void assert_connection_ok(struct connection_t *conn, time_t now);
+int connection_or_nonopen_was_started_here(struct or_connection_t *conn);
 void connection_dump_buffer_mem_stats(int severity);
 
 MOCK_DECL(void, clock_skew_warning,
-          (const connection_t *conn, long apparent_skew, int trusted,
+          (const struct connection_t *conn, long apparent_skew, int trusted,
            log_domain_mask_t domain, const char *received,
            const char *source));
 
-int connection_is_moribund(connection_t *conn);
+int connection_is_moribund(struct connection_t *conn);
 void connection_check_oos(int n_socks, int failed);
 
 /** Execute the statement <b>stmt</b>, which may log events concerning the
@@ -335,18 +368,18 @@ void connection_check_oos(int n_socks, int failed);
   STMT_END
 
 #ifdef CONNECTION_PRIVATE
-STATIC void connection_free_minimal(connection_t *conn);
+STATIC void connection_free_minimal(struct connection_t *conn);
 
 /* Used only by connection.c and test*.c */
 MOCK_DECL(STATIC int,connection_connect_sockaddr,
-                                            (connection_t *conn,
+                                            (struct connection_t *conn,
                                              const struct sockaddr *sa,
                                              socklen_t sa_len,
                                              const struct sockaddr *bindaddr,
                                              socklen_t bindaddr_len,
                                              int *socket_error));
-MOCK_DECL(STATIC void, kill_conn_list_for_oos, (smartlist_t *conns));
-MOCK_DECL(STATIC smartlist_t *, pick_oos_victims, (int n));
+MOCK_DECL(STATIC void, kill_conn_list_for_oos, (struct smartlist_t *conns));
+MOCK_DECL(STATIC struct smartlist_t *, pick_oos_victims, (int n));
 
 #endif /* defined(CONNECTION_PRIVATE) */
 
diff --git a/src/core/mainloop/core_mainloop.md b/src/core/mainloop/core_mainloop.md
new file mode 100644
index 0000000000..fee8a8179c
--- /dev/null
+++ b/src/core/mainloop/core_mainloop.md
@@ -0,0 +1,10 @@
+@dir /core/mainloop
+@brief core/mainloop: Non-onion-routing mainloop functionality
+
+This module uses the event-loop code of \refdir{lib/evloop} to implement an
+asynchronous connection-oriented protocol handler.
+
+The layering here is imperfect: the code here was split from \refdir{core/or}
+without refactoring how the two modules call one another.  Probably many
+functions should be moved and refactored.
+
diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c
index e704d55642..485ddb9741 100644
--- a/src/core/mainloop/cpuworker.c
+++ b/src/core/mainloop/cpuworker.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,6 @@
  **/
 #include "core/or/or.h"
 #include "core/or/channel.h"
-#include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
@@ -27,6 +26,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "core/or/onion.h"
+#include "feature/relay/circuitbuild_relay.h"
 #include "feature/relay/onion_queue.h"
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
@@ -34,11 +34,10 @@
 #include "core/crypto/onion_crypto.h"
 
 #include "core/or/or_circuit_st.h"
-#include "lib/intmath/weakrng.h"
 
 static void queue_pending_tasks(void);
 
-typedef struct worker_state_s {
+typedef struct worker_state_t {
   int generation;
   server_onion_keys_t *onion_keys;
 } worker_state_t;
@@ -74,8 +73,6 @@ worker_state_free_void(void *arg)
 static replyqueue_t *replyqueue = NULL;
 static threadpool_t *threadpool = NULL;
 
-static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT;
-
 static int total_pending_tasks = 0;
 static int max_pending_tasks = 128;
 
@@ -109,7 +106,6 @@ cpu_init(void)
 
   /* Total voodoo. Can we make this more sensible? */
   max_pending_tasks = get_num_cpus(get_options()) * 64;
-  crypto_seed_weak_rng(&request_sample_rng);
 }
 
 /** Magic numbers to make sure our cpuworker_requests don't grow any
@@ -164,7 +160,7 @@ typedef struct cpuworker_reply_t {
   uint8_t rend_auth_material[DIGEST_LEN];
 } cpuworker_reply_t;
 
-typedef struct cpuworker_job_u {
+typedef struct cpuworker_job_u_t {
   or_circuit_t *circ;
   union {
     cpuworker_request_t request;
@@ -235,9 +231,10 @@ should_time_request(uint16_t onionskin_type)
    * sample */
   if (onionskins_n_processed[onionskin_type] < 4096)
     return 1;
+
   /** Otherwise, measure with P=1/128.  We avoid doing this for every
    * handshake, since the measurement itself can take a little time. */
-  return tor_weak_random_one_in_n(&request_sample_rng, 128);
+  return crypto_fast_rng_one_in_n(get_thread_fast_rng(), 128);
 }
 
 /** Return an estimate of how many microseconds we will need for a single
diff --git a/src/core/mainloop/cpuworker.h b/src/core/mainloop/cpuworker.h
index 77e2c42508..7e71961750 100644
--- a/src/core/mainloop/cpuworker.h
+++ b/src/core/mainloop/cpuworker.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,10 +14,10 @@
 
 void cpu_init(void);
 void cpuworkers_rotate_keyinfo(void);
-struct workqueue_entry_s;
+struct workqueue_entry_t;
 enum workqueue_reply_t;
 enum workqueue_priority_t;
-MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, (
+MOCK_DECL(struct workqueue_entry_t *, cpuworker_queue_work, (
                     enum workqueue_priority_t priority,
                     enum workqueue_reply_t (*fn)(void *, void *),
                     void (*reply_fn)(void *),
diff --git a/src/core/mainloop/include.am b/src/core/mainloop/include.am
new file mode 100644
index 0000000000..63643127f3
--- /dev/null
+++ b/src/core/mainloop/include.am
@@ -0,0 +1,22 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/mainloop/connection.c		\
+	src/core/mainloop/cpuworker.c		\
+	src/core/mainloop/mainloop.c		\
+	src/core/mainloop/mainloop_pubsub.c	\
+	src/core/mainloop/mainloop_sys.c	\
+	src/core/mainloop/netstatus.c		\
+	src/core/mainloop/periodic.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/mainloop/connection.h			\
+	src/core/mainloop/cpuworker.h			\
+	src/core/mainloop/mainloop.h			\
+	src/core/mainloop/mainloop_pubsub.h		\
+	src/core/mainloop/mainloop_state.inc    	\
+	src/core/mainloop/mainloop_state_st.h    	\
+	src/core/mainloop/mainloop_sys.h		\
+	src/core/mainloop/netstatus.h			\
+	src/core/mainloop/periodic.h
diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c
index e67e1299b2..b4dbedbfe4 100644
--- a/src/core/mainloop/mainloop.c
+++ b/src/core/mainloop/mainloop.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,12 +71,13 @@
 #include "feature/client/bridges.h"
 #include "feature/client/dnsserv.h"
 #include "feature/client/entrynodes.h"
+#include "feature/client/proxymode.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
-#include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_cache.h"
@@ -95,7 +96,7 @@
 #include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/err/backtrace.h"
 #include "lib/tls/buffers_tls.h"
@@ -105,9 +106,6 @@
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/authmode.h"
-
 #include "core/or/cell_st.h"
 #include "core/or/entry_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -200,12 +198,10 @@ static int can_complete_circuits = 0;
 #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60)
 
 static int conn_close_if_marked(int i);
-static int run_main_loop_until_done(void);
 static void connection_start_reading_from_linked_conn(connection_t *conn);
 static int connection_should_read_from_linked_conn(connection_t *conn);
 static void conn_read_callback(evutil_socket_t fd, short event, void *_conn);
 static void conn_write_callback(evutil_socket_t fd, short event, void *_conn);
-static void second_elapsed_callback(periodic_timer_t *timer, void *args);
 static void shutdown_did_not_work_callback(evutil_socket_t fd, short event,
                                            void *arg) ATTR_NORETURN;
 
@@ -759,7 +755,7 @@ tor_shutdown_event_loop_for_restart_cb(
   tor_event_free(tor_shutdown_event_loop_for_restart_event);
   tor_shutdown_event_loop_and_exit(0);
 }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
 /**
  * After finishing the current callback (if any), shut down the main loop,
@@ -970,7 +966,6 @@ conn_close_if_marked(int i)
     return 0; /* nothing to see here, move along */
   now = time(NULL);
   assert_connection_ok(conn, now);
-  /* assert_all_pending_dns_resolves_ok(); */
 
   log_debug(LD_NET,"Cleaning up connection (fd "TOR_SOCKET_T_FORMAT").",
             conn->s);
@@ -1137,14 +1132,14 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
 
   if (!router_have_minimum_dir_info()) {
     int quiet = suppress_logs || from_cache ||
-                directory_too_idle_to_fetch_descriptors(options, now);
+                dirclient_too_idle_to_fetch_descriptors(options, now);
     tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR,
         "I learned some more directory information, but not enough to "
         "build a circuit: %s", get_dir_info_status_string());
     update_all_descriptor_downloads(now);
     return;
   } else {
-    if (directory_fetches_from_authorities(options)) {
+    if (dirclient_fetches_from_authorities(options)) {
       update_all_descriptor_downloads(now);
     }
 
@@ -1357,123 +1352,101 @@ get_signewnym_epoch(void)
 static int periodic_events_initialized = 0;
 
 /* Declare all the timer callback functions... */
+#ifndef COCCI
 #undef CALLBACK
 #define CALLBACK(name) \
   static int name ## _callback(time_t, const or_options_t *)
+
 CALLBACK(add_entropy);
-CALLBACK(check_authority_cert);
-CALLBACK(check_canonical_channels);
-CALLBACK(check_descriptor);
-CALLBACK(check_dns_honesty);
-CALLBACK(check_ed_keys);
 CALLBACK(check_expired_networkstatus);
-CALLBACK(check_for_reachability_bw);
-CALLBACK(check_onion_keys_expiry_time);
 CALLBACK(clean_caches);
 CALLBACK(clean_consdiffmgr);
-CALLBACK(dirvote);
-CALLBACK(downrate_stability);
-CALLBACK(expire_old_ciruits_serverside);
 CALLBACK(fetch_networkstatus);
 CALLBACK(heartbeat);
 CALLBACK(hs_service);
 CALLBACK(launch_descriptor_fetches);
-CALLBACK(launch_reachability_tests);
-CALLBACK(reachability_warnings);
+CALLBACK(prune_old_routers);
 CALLBACK(record_bridge_stats);
 CALLBACK(rend_cache_failure_clean);
 CALLBACK(reset_padding_counts);
-CALLBACK(retry_dns);
 CALLBACK(retry_listeners);
-CALLBACK(rotate_onion_key);
 CALLBACK(rotate_x509_certificate);
-CALLBACK(save_stability);
 CALLBACK(save_state);
-CALLBACK(write_bridge_ns);
 CALLBACK(write_stats_file);
+CALLBACK(control_per_second_events);
+CALLBACK(second_elapsed);
 
 #undef CALLBACK
 
 /* Now we declare an array of periodic_event_item_t for each periodic event */
-#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f)
-
-STATIC periodic_event_item_t periodic_events[] = {
-  /* Everyone needs to run those. */
-  CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0),
-
-  /* Routers (bridge and relay) only. */
-  CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0),
-
-  /* Authorities (bridge and directory) only. */
-  CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-  CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-
-  /* Directory authority only. */
-  CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0),
-  CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET),
-
-  /* Relay only. */
-  CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+#define CALLBACK(name, r, f)                            \
+  PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f)
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+#endif /* !defined(COCCI) */
+
+STATIC periodic_event_item_t mainloop_periodic_events[] = {
+
+  /* Everyone needs to run these. They need to have very long timeouts for
+   * that to be safe. */
+  CALLBACK(add_entropy, ALL, 0),
+  CALLBACK(heartbeat, ALL, 0),
+  CALLBACK(reset_padding_counts, ALL, 0),
+
+  /* This is a legacy catch-all callback that runs once per second if
+   * we are online and active. */
+  CALLBACK(second_elapsed, NET_PARTICIPANT,
+           FL(RUN_ON_DISABLE)),
+
+  /* XXXX Do we have a reason to do this on a callback? Does it do any good at
+   * all?  For now, if we're dormant, we can let our listeners decay. */
+  CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)),
+
+  /* We need to do these if we're participating in the Tor network. */
+  CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)),
+  CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0),
+  CALLBACK(check_network_participation, NET_PARTICIPANT, 0),
+
+  /* We need to do these if we're participating in the Tor network, and
+   * immediately before we stop. */
+  CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Hidden Service service only. */
-  CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more
 
   /* Bridge only. */
-  CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0),
+  CALLBACK(record_bridge_stats, BRIDGE, 0),
 
   /* Client only. */
-  CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0),
-
-  /* Bridge Authority only. */
-  CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0),
+  /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */
+  CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Directory server only. */
-  CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0),
+  CALLBACK(clean_consdiffmgr, DIRSERVER, 0),
+
+  /* Controller with per-second events only. */
+  CALLBACK(control_per_second_events, CONTROLEV, 0),
 
   END_OF_PERIODIC_EVENTS
 };
+#ifndef COCCI
 #undef CALLBACK
+#undef FL
+#endif
 
 /* These are pointers to members of periodic_events[] that are used to
  * implement particular callbacks.  We keep them separate here so that we
  * can access them by name.  We also keep them inside periodic_events[]
  * so that we can implement "reset all timers" in a reasonable way. */
-static periodic_event_item_t *check_descriptor_event=NULL;
-static periodic_event_item_t *dirvote_event=NULL;
 static periodic_event_item_t *fetch_networkstatus_event=NULL;
 static periodic_event_item_t *launch_descriptor_fetches_event=NULL;
 static periodic_event_item_t *check_dns_honesty_event=NULL;
 static periodic_event_item_t *save_state_event=NULL;
+static periodic_event_item_t *prune_old_routers_event=NULL;
 
 /** Reset all the periodic events so we'll do all our actions again as if we
  * just started up.
@@ -1483,24 +1456,7 @@ static periodic_event_item_t *save_state_event=NULL;
 void
 reset_all_main_loop_timers(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_reschedule(&periodic_events[i]);
-  }
-}
-
-/** Return the member of periodic_events[] whose name is <b>name</b>.
- * Return NULL if no such event is found.
- */
-static periodic_event_item_t *
-find_periodic_event(const char *name)
-{
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    if (strcmp(name, periodic_events[i].name) == 0)
-      return &periodic_events[i];
-  }
-  return NULL;
+  periodic_events_reset_all();
 }
 
 /** Return a bitmask of the roles this tor instance is configured for using
@@ -1510,7 +1466,7 @@ get_my_roles(const or_options_t *options)
 {
   tor_assert(options);
 
-  int roles = 0;
+  int roles = PERIODIC_EVENT_ROLE_ALL;
   int is_bridge = options->BridgeRelay;
   int is_relay = server_mode(options);
   int is_dirauth = authdir_mode_v3(options);
@@ -1518,6 +1474,8 @@ get_my_roles(const or_options_t *options)
   int is_hidden_service = !!hs_service_get_num_services() ||
                           !!rend_num_services();
   int is_dirserver = dir_server_mode(options);
+  int sending_control_events = control_any_per_second_event_enabled();
+
   /* We also consider tor to have the role of a client if the ControlPort is
    * set because a lot of things can be done over the control port which
    * requires tor to have basic functionnalities. */
@@ -1525,6 +1483,9 @@ get_my_roles(const or_options_t *options)
                   options->ControlPort_set ||
                   options->OwningControllerFD != UINT64_MAX;
 
+  int is_net_participant = is_participating_on_network() ||
+    is_relay || is_hidden_service;
+
   if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE;
   if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT;
   if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY;
@@ -1532,6 +1493,8 @@ get_my_roles(const or_options_t *options)
   if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH;
   if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE;
   if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER;
+  if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV;
 
   return roles;
 }
@@ -1556,9 +1519,9 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data)
   rescan_periodic_events(get_options());
 }
 
-/** Set up all the members of periodic_events[], and configure them all to be
- * launched from a callback. */
-STATIC void
+/** Set up all the members of mainloop_periodic_events[], and configure them
+ * all to be launched from a callback. */
+void
 initialize_periodic_events(void)
 {
   if (periodic_events_initialized)
@@ -1566,39 +1529,60 @@ initialize_periodic_events(void)
 
   periodic_events_initialized = 1;
 
-  /* Set up all periodic events. We'll launch them by roles. */
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_setup(&periodic_events[i]);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_events_register(&mainloop_periodic_events[i]);
   }
 
+  /* Set up all periodic events. We'll launch them by roles. */
+
+#ifndef COCCI
 #define NAMED_CALLBACK(name) \
-  STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END
+  STMT_BEGIN name ## _event = periodic_events_find( #name ); STMT_END
+#endif
 
-  NAMED_CALLBACK(check_descriptor);
-  NAMED_CALLBACK(dirvote);
+  NAMED_CALLBACK(prune_old_routers);
   NAMED_CALLBACK(fetch_networkstatus);
   NAMED_CALLBACK(launch_descriptor_fetches);
   NAMED_CALLBACK(check_dns_honesty);
   NAMED_CALLBACK(save_state);
-
-  struct timeval one_second = { 1, 0 };
-  initialize_periodic_events_event = tor_evtimer_new(
-                  tor_libevent_get_base(),
-                  initialize_periodic_events_cb, NULL);
-  event_add(initialize_periodic_events_event, &one_second);
 }
 
 STATIC void
 teardown_periodic_events(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_destroy(&periodic_events[i]);
-  }
+  periodic_events_disconnect_all();
+  fetch_networkstatus_event = NULL;
+  launch_descriptor_fetches_event = NULL;
+  check_dns_honesty_event = NULL;
+  save_state_event = NULL;
+  prune_old_routers_event = NULL;
   periodic_events_initialized = 0;
 }
 
+static mainloop_event_t *rescan_periodic_events_ev = NULL;
+
+/** Callback: rescan the periodic event list. */
+static void
+rescan_periodic_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+  rescan_periodic_events(get_options());
+}
+
+/**
+ * Schedule an event that will rescan which periodic events should run.
+ **/
+MOCK_IMPL(void,
+schedule_rescan_periodic_events,(void))
+{
+  if (!rescan_periodic_events_ev) {
+    rescan_periodic_events_ev =
+      mainloop_event_new(rescan_periodic_events_cb, NULL);
+  }
+  mainloop_event_activate(rescan_periodic_events_ev);
+}
+
 /** Do a pass at all our periodic events, disable those we don't need anymore
  * and enable those we need now using the given options. */
 void
@@ -1606,36 +1590,7 @@ rescan_periodic_events(const or_options_t *options)
 {
   tor_assert(options);
 
-  /* Avoid scanning the event list if we haven't initialized it yet. This is
-   * particularly useful for unit tests in order to avoid initializing main
-   * loop events everytime. */
-  if (!periodic_events_initialized) {
-    return;
-  }
-
-  int roles = get_my_roles(options);
-
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-
-    int enable = !!(item->roles & roles);
-
-    /* Handle the event flags. */
-    if (net_is_disabled() &&
-        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
-      enable = 0;
-    }
-
-    /* Enable the event if needed. It is safe to enable an event that was
-     * already enabled. Same goes for disabling it. */
-    if (enable) {
-      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
-      periodic_event_enable(item);
-    } else {
-      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
-      periodic_event_disable(item);
-    }
-  }
+  periodic_events_rescan_by_roles(get_my_roles(options), net_is_disabled());
 }
 
 /* We just got new options globally set, see if we need to enabled or disable
@@ -1643,26 +1598,7 @@ rescan_periodic_events(const or_options_t *options)
 void
 periodic_events_on_new_options(const or_options_t *options)
 {
-  /* Only if we've already initialized the events, rescan the list which will
-   * enable or disable events depending on our roles. This will be called at
-   * bootup and we don't want this function to initialize the events because
-   * they aren't set up at this stage. */
-  if (periodic_events_initialized) {
-    rescan_periodic_events(options);
-  }
-}
-
-/**
- * Update our schedule so that we'll check whether we need to update our
- * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
- * seconds.
- */
-void
-reschedule_descriptor_update_check(void)
-{
-  if (check_descriptor_event) {
-    periodic_event_reschedule(check_descriptor_event);
-  }
+  rescan_periodic_events(options);
 }
 
 /**
@@ -1708,40 +1644,41 @@ mainloop_schedule_postloop_cleanup(void)
   mainloop_event_activate(postloop_cleanup_ev);
 }
 
-#define LONGEST_TIMER_PERIOD (30 * 86400)
-/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
- * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */
-static inline int
-safe_timer_diff(time_t now, time_t next)
-{
-  if (next > now) {
-    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
-     * and nothing that could run Tor. It's a bug if 'next' is around then.
-     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
-     * Bang. We cannot extrapolate past a singularity, but there was probably
-     * nothing that could run Tor then, either.
-     **/
-    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
-
-    if (next - LONGEST_TIMER_PERIOD > now)
-      return LONGEST_TIMER_PERIOD;
-    return (int)(next - now);
-  } else {
-    return 1;
+/** Event to run 'scheduled_shutdown_cb' */
+static mainloop_event_t *scheduled_shutdown_ev=NULL;
+
+/** Callback: run a scheduled shutdown */
+static void
+scheduled_shutdown_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+  tor_shutdown_event_loop_and_exit(0);
+}
+
+/** Schedule the mainloop to exit after <b>delay_sec</b> seconds. */
+void
+mainloop_schedule_shutdown(int delay_sec)
+{
+  const struct timeval delay_tv = { delay_sec, 0 };
+  if (! scheduled_shutdown_ev) {
+    scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL);
   }
+  mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv);
 }
 
 /** Perform regular maintenance tasks.  This function gets run once per
- * second by second_elapsed_callback().
+ * second.
  */
-static void
-run_scheduled_events(time_t now)
+static int
+second_elapsed_callback(time_t now, const or_options_t *options)
 {
-  const or_options_t *options = get_options();
-
-  /* 0. See if we've been asked to shut down and our timeout has
-   * expired; or if our bandwidth limits are exhausted and we
-   * should hibernate; or if it's time to wake up from hibernation.
+  /* 0. See if our bandwidth limits are exhausted and we should hibernate
+   *
+   * Note: we have redundant mechanisms to handle the case where it's
+   * time to wake up from hibernation; or where we have a scheduled
+   * shutdown and it's time to run it, but this will also handle those.
    */
   consider_hibernation(now);
 
@@ -1751,10 +1688,13 @@ run_scheduled_events(time_t now)
   if (options->UseBridges && !net_is_disabled()) {
     /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches()
      * -- the latter is only for fetching consensus-derived directory info. */
+    // TODO: client
+    //     Also, schedule this rather than probing 1x / sec
     fetch_bridge_descriptors(options, now);
   }
 
   if (accounting_is_enabled(options)) {
+    // TODO: refactor or rewrite?
     accounting_run_housekeeping(now);
   }
 
@@ -1765,6 +1705,7 @@ run_scheduled_events(time_t now)
    */
   /* (If our circuit build timeout can ever become lower than a second (which
    * it can't, currently), we should do this more often.) */
+  // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE
   circuit_expire_building();
   circuit_expire_waiting_for_better_guard();
 
@@ -1798,80 +1739,8 @@ run_scheduled_events(time_t now)
     run_connection_housekeeping(i, now);
   }
 
-  /* 11b. check pending unconfigured managed proxies */
-  if (!net_is_disabled() && pt_proxies_configuration_pending())
-    pt_configure_remaining_proxies();
-}
-
-/* Periodic callback: rotate the onion keys after the period defined by the
- * "onion-key-rotation-days" consensus parameter, shut down and restart all
- * cpuworkers, and update our descriptor if necessary.
- */
-static int
-rotate_onion_key_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_lifetime = get_onion_key_lifetime();
-    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
-    if (rotation_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL,"Rotating onion key.");
-    rotate_onion_key();
-    cpuworkers_rotate_keyinfo();
-    if (router_rebuild_descriptor(1)<0) {
-      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
-    }
-    if (advertised_server_mode() && !net_is_disabled())
-      router_upload_dir_desc_to_dirservers(0);
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Period callback: Check if our old onion keys are still valid after the
- * period of time defined by the consensus parameter
- * "onion-key-grace-period-days", otherwise expire them by setting them to
- * NULL.
- */
-static int
-check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_grace_period = get_onion_key_grace_period();
-    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
-    if (expiry_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL, "Expiring old onion keys.");
-    expire_old_onion_keys();
-    cpuworkers_rotate_keyinfo();
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Periodic callback: Every 30 seconds, check whether it's time to make new
- * Ed25519 subkeys.
- */
-static int
-check_ed_keys_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    if (should_make_new_ed_keys(options, now)) {
-      int new_signing_key = load_ed_keys(options, now);
-      if (new_signing_key < 0 ||
-          generate_ed_link_cert(options, now, new_signing_key > 0)) {
-        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
-        tor_shutdown_event_loop_and_exit(1);
-      }
-    }
-    return 30;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
+  /* Run again in a second. */
+  return 1;
 }
 
 /**
@@ -1947,100 +1816,53 @@ add_entropy_callback(time_t now, const or_options_t *options)
   return ENTROPY_INTERVAL;
 }
 
-/**
- * Periodic callback: if we're an authority, make sure we test
- * the routers on the network for reachability.
- */
-static int
-launch_reachability_tests_callback(time_t now, const or_options_t *options)
+/** Periodic callback: if there has been no network usage in a while,
+ * enter a dormant state. */
+STATIC int
+check_network_participation_callback(time_t now, const or_options_t *options)
 {
-  if (authdir_mode_tests_reachability(options) &&
-      !net_is_disabled()) {
-    /* try to determine reachability of the other Tor relays */
-    dirserv_test_reachability(now);
+  /* If we're a server, we can't become dormant. */
+  if (server_mode(options)) {
+    goto found_activity;
   }
-  return REACHABILITY_TEST_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, discount the stability
- * information (and other rephist information) that's older.
- */
-static int
-downrate_stability_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* 1d. Periodically, we discount older stability information so that new
-   * stability info counts more, and save the stability information to disk as
-   * appropriate. */
-  time_t next = rep_hist_downrate_old_runs(now);
-  return safe_timer_diff(now, next);
-}
+  /* If we're running an onion service, we can't become dormant. */
+  /* XXXX this would be nice to change, so that we can be dormant with a
+   * service. */
+  if (hs_service_get_num_services() || rend_num_services()) {
+    goto found_activity;
+  }
 
-/**
- * Periodic callback: if we're an authority, record our measured stability
- * information from rephist in an mtbf file.
- */
-static int
-save_stability_callback(time_t now, const or_options_t *options)
-{
-  if (authdir_mode_tests_reachability(options)) {
-    if (rep_hist_record_mtbf_data(now, 1)<0) {
-      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+  /* If we have any currently open entry streams other than "linked"
+   * connections used for directory requests, those count as user activity.
+   */
+  if (options->DormantTimeoutDisabledByIdleStreams) {
+    if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) {
+      goto found_activity;
     }
   }
-#define SAVE_STABILITY_INTERVAL (30*60)
-  return SAVE_STABILITY_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, check on our authority
- * certificate (the one that authenticates our authority signing key).
- */
-static int
-check_authority_cert_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  (void)options;
-  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
-   * close to expiring and warn the admin if it is. */
-  v3_authority_check_key_expiry();
-#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
-  return CHECK_V3_CERTIFICATE_INTERVAL;
-}
+  /* XXXX Make this configurable? */
+/** How often do we check whether we have had network activity? */
+#define CHECK_PARTICIPATION_INTERVAL (5*60)
 
-/**
- * Scheduled callback: Run directory-authority voting functionality.
- *
- * The schedule is a bit complicated here, so dirvote_act() manages the
- * schedule itself.
- **/
-static int
-dirvote_callback(time_t now, const or_options_t *options)
-{
-  if (!authdir_mode_v3(options)) {
-    tor_assert_nonfatal_unreached();
-    return 3600;
+  /* Become dormant if there has been no user activity in a long time.
+   * (The funny checks below are in order to prevent overflow.) */
+  time_t time_since_last_activity = 0;
+  if (get_last_user_activity_time() < now)
+    time_since_last_activity = now - get_last_user_activity_time();
+  if (time_since_last_activity >= options->DormantClientTimeout) {
+    log_notice(LD_GENERAL, "No user activity in a long time: becoming"
+               " dormant.");
+    set_network_participation(false);
+    rescan_periodic_events(options);
   }
 
-  time_t next = dirvote_act(options, now);
-  if (BUG(next == TIME_MAX)) {
-    /* This shouldn't be returned unless we called dirvote_act() without
-     * being an authority.  If it happens, maybe our configuration will
-     * fix itself in an hour or so? */
-    return 3600;
-  }
-  return safe_timer_diff(now, next);
-}
+  return CHECK_PARTICIPATION_INTERVAL;
 
-/** Reschedule the directory-authority voting event.  Run this whenever the
- * schedule has changed. */
-void
-reschedule_dirvote(const or_options_t *options)
-{
-  if (periodic_events_initialized && authdir_mode_v3(options)) {
-    periodic_event_reschedule(dirvote_event);
-  }
+ found_activity:
+  note_user_activity(now);
+  return CHECK_PARTICIPATION_INTERVAL;
 }
 
 /**
@@ -2053,11 +1875,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   (void)options;
   /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
-  /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
-   * networkstatus_get_reasonably_live_consensus(), but that value is way
-   * way too high.  Arma: is the bridge issue there resolved yet? -NM */
-#define NS_EXPIRY_SLOP (24*60*60)
-  if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) &&
+  /* Use reasonably live consensuses until they are no longer reasonably live.
+   */
+  if (ns && !networkstatus_consensus_reasonably_live(ns, now) &&
       router_have_minimum_dir_info()) {
     router_dir_info_changed();
   }
@@ -2143,17 +1963,6 @@ write_stats_file_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next_time_to_write_stats_files);
 }
 
-#define CHANNEL_CHECK_INTERVAL (60*60)
-static int
-check_canonical_channels_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  if (public_server_mode(options))
-    channel_check_for_duplicates();
-
-  return CHANNEL_CHECK_INTERVAL;
-}
-
 static int
 reset_padding_counts_callback(time_t now, const or_options_t *options)
 {
@@ -2228,87 +2037,24 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
 }
 
 /**
- * Periodic callback: If we're a server and initializing dns failed, retry.
+ * Periodic callback: prune routerlist of old information about Tor network.
  */
 static int
-retry_dns_callback(time_t now, const or_options_t *options)
+prune_old_routers_callback(time_t now, const or_options_t *options)
 {
+#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour.
   (void)now;
-#define RETRY_DNS_INTERVAL (10*60)
-  if (server_mode(options) && has_dns_init_failed())
-    dns_init();
-  return RETRY_DNS_INTERVAL;
-}
-
-/** Periodic callback: consider rebuilding or and re-uploading our descriptor
- * (if we've passed our internal checks). */
-static int
-check_descriptor_callback(time_t now, const or_options_t *options)
-{
-/** How often do we check whether part of our router info has changed in a
- * way that would require an upload? That includes checking whether our IP
- * address has changed. */
-#define CHECK_DESCRIPTOR_INTERVAL (60)
-
   (void)options;
 
-  /* 2b. Once per minute, regenerate and upload the descriptor if the old
-   * one is inaccurate. */
   if (!net_is_disabled()) {
-    check_descriptor_bandwidth_changed(now);
-    check_descriptor_ipaddress_changed(now);
-    mark_my_descriptor_dirty_if_too_old(now);
-    consider_publishable_server(0);
     /* If any networkstatus documents are no longer recent, we need to
      * update all the descriptors' running status. */
     /* Remove dead routers. */
-    /* XXXX This doesn't belong here, but it was here in the pre-
-     * XXXX refactoring code. */
+    log_debug(LD_GENERAL, "Pruning routerlist...");
     routerlist_remove_old_routers();
   }
 
-  return CHECK_DESCRIPTOR_INTERVAL;
-}
-
-/**
- * Periodic callback: check whether we're reachable (as a relay), and
- * whether our bandwidth has changed enough that we need to
- * publish a new descriptor.
- */
-static int
-check_for_reachability_bw_callback(time_t now, const or_options_t *options)
-{
-  /* XXXX This whole thing was stuck in the middle of what is now
-   * XXXX check_descriptor_callback.  I'm not sure it's right. */
-
-  static int dirport_reachability_count = 0;
-  /* also, check religiously for reachability, if it's within the first
-   * 20 minutes of our uptime. */
-  if (server_mode(options) &&
-      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
-      !net_is_disabled()) {
-    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-      router_do_reachability_checks(1, dirport_reachability_count==0);
-      if (++dirport_reachability_count > 5)
-        dirport_reachability_count = 0;
-      return 1;
-    } else {
-      /* If we haven't checked for 12 hours and our bandwidth estimate is
-       * low, do another bandwidth test. This is especially important for
-       * bridges, since they might go long periods without much use. */
-      const routerinfo_t *me = router_get_my_routerinfo();
-      static int first_time = 1;
-      if (!first_time && me &&
-          me->bandwidthcapacity < me->bandwidthrate &&
-          me->bandwidthcapacity < 51200) {
-        reset_bandwidth_test();
-      }
-      first_time = 0;
-#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
-      return BANDWIDTH_RECHECK_INTERVAL;
-    }
-  }
-  return CHECK_DESCRIPTOR_INTERVAL;
+  return ROUTERLIST_PRUNING_INTERVAL;
 }
 
 /**
@@ -2322,7 +2068,7 @@ fetch_networkstatus_callback(time_t now, const or_options_t *options)
    * documents? */
   const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping(
                                                                         now);
-  const int prefer_mirrors = !directory_fetches_from_authorities(
+  const int prefer_mirrors = !dirclient_fetches_from_authorities(
                                                               get_options());
   int networkstatus_dl_check_interval = 60;
   /* check more often when testing, or when bootstrapping from mirrors
@@ -2353,109 +2099,6 @@ retry_listeners_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
-/**
- * Periodic callback: as a server, see if we have any old unused circuits
- * that should be expired */
-static int
-expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* every 11 seconds, so not usually the same second as other such events */
-  circuit_expire_old_circuits_serverside(now);
-  return 11;
-}
-
-/**
- * Callback: Send warnings if Tor doesn't find its ports reachable.
- */
-static int
-reachability_warnings_callback(time_t now, const or_options_t *options)
-{
-  (void) now;
-
-  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
-  }
-
-  if (server_mode(options) &&
-      !net_is_disabled() &&
-      have_completed_a_circuit()) {
-    /* every 20 minutes, check and complain if necessary */
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (me && !check_whether_orport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
-               "its ORPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->or_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
-                                  address, me->or_port);
-      tor_free(address);
-    }
-
-    if (me && !check_whether_dirport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,
-               "Your server (%s:%d) has not managed to confirm that its "
-               "DirPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->dir_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
-                                  address, me->dir_port);
-      tor_free(address);
-    }
-  }
-
-  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
-}
-
-static int dns_honesty_first_time = 1;
-
-/**
- * Periodic event: if we're an exit, see if our DNS server is telling us
- * obvious lies.
- */
-static int
-check_dns_honesty_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  /* 9. and if we're an exit node, check whether our DNS is telling stories
-   * to us. */
-  if (net_is_disabled() ||
-      ! public_server_mode(options) ||
-      router_my_exit_policy_is_reject_star())
-    return PERIODIC_EVENT_NO_UPDATE;
-
-  if (dns_honesty_first_time) {
-    /* Don't launch right when we start */
-    dns_honesty_first_time = 0;
-    return crypto_rand_int_range(60, 180);
-  }
-
-  dns_launch_correctness_checks();
-  return 12*3600 + crypto_rand_int(12*3600);
-}
-
-/**
- * Periodic callback: if we're the bridge authority, write a networkstatus
- * file to disk.
- */
-static int
-write_bridge_ns_callback(time_t now, const or_options_t *options)
-{
-  /* 10. write bridge networkstatus file to disk */
-  if (options->BridgeAuthoritativeDir) {
-    networkstatus_dump_bridge_status_to_file(now);
-#define BRIDGE_STATUSFILE_INTERVAL (30*60)
-     return BRIDGE_STATUSFILE_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
 static int heartbeat_callback_first_time = 1;
 
 /**
@@ -2523,36 +2166,19 @@ hs_service_callback(time_t now, const or_options_t *options)
   return 1;
 }
 
-/** Timer: used to invoke second_elapsed_callback() once per second. */
-static periodic_timer_t *second_timer = NULL;
-
-/**
- * Enable or disable the per-second timer as appropriate, creating it if
- * necessary.
+/*
+ * Periodic callback: Send once-per-second events to the controller(s).
+ * This is called every second.
  */
-void
-reschedule_per_second_timer(void)
+static int
+control_per_second_events_callback(time_t now, const or_options_t *options)
 {
-  struct timeval one_second;
-  one_second.tv_sec = 1;
-  one_second.tv_usec = 0;
-
-  if (! second_timer) {
-    second_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &one_second,
-                                      second_elapsed_callback,
-                                      NULL);
-    tor_assert(second_timer);
-  }
+  (void) options;
+  (void) now;
 
-  const bool run_per_second_events =
-    control_any_per_second_event_enabled() || ! net_is_completely_disabled();
+  control_per_second_events();
 
-  if (run_per_second_events) {
-    periodic_timer_launch(second_timer, &one_second);
-  } else {
-    periodic_timer_disable(second_timer);
-  }
+  return 1;
 }
 
 /** Last time that update_current_time was called. */
@@ -2582,6 +2208,17 @@ update_current_time(time_t now)
   memcpy(&last_updated, &current_second_last_changed, sizeof(last_updated));
   monotime_coarse_get(&current_second_last_changed);
 
+  /** How much clock jumping means that we should adjust our idea of when
+   * to go dormant? */
+#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20
+
+  /* Don't go dormant early or late just because we jumped in time. */
+  if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) {
+    if (is_participating_on_network()) {
+      netstatus_note_clock_jumped(seconds_elapsed);
+    }
+  }
+
   /** How much clock jumping do we tolerate? */
 #define NUM_JUMPED_SECONDS_BEFORE_WARN 100
 
@@ -2591,6 +2228,7 @@ update_current_time(time_t now)
   if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) {
     // moving back in time is always a bad sign.
     circuit_note_clock_jumped(seconds_elapsed, false);
+
   } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
     /* Compare the monotonic clock to the result of time(). */
     const int32_t monotime_msec_passed =
@@ -2620,31 +2258,6 @@ update_current_time(time_t now)
   current_second = now;
 }
 
-/** Libevent callback: invoked once every second. */
-static void
-second_elapsed_callback(periodic_timer_t *timer, void *arg)
-{
-  /* XXXX This could be sensibly refactored into multiple callbacks, and we
-   * could use Libevent's timers for this rather than checking the current
-   * time against a bunch of timeouts every second. */
-  time_t now;
-  (void)timer;
-  (void)arg;
-
-  now = time(NULL);
-
-  /* We don't need to do this once-per-second any more: time-updating is
-   * only in this callback _because it is a callback_. It should be fine
-   * to disable this callback, and the time will still get updated.
-   */
-  update_current_time(now);
-
-  /* Maybe some controller events are ready to fire */
-  control_per_second_events();
-
-  run_scheduled_events(now);
-}
-
 #ifdef HAVE_SYSTEMD_209
 static periodic_timer_t *systemd_watchdog_timer = NULL;
 
@@ -2660,18 +2273,23 @@ systemd_watchdog_callback(periodic_timer_t *timer, void *arg)
 
 #define UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST (6*60*60)
 
-/** Called when our IP address seems to have changed. <b>at_interface</b>
- * should be true if we detected a change in our interface, and false if we
- * detected a change in our published address. */
+/** Called when our IP address seems to have changed. <b>on_client_conn</b>
+ * should be true if:
+ *   - we detected a change in our interface address, using an outbound
+ *     connection, and therefore
+ *   - our client TLS keys need to be rotated.
+ * Otherwise, it should be false, and:
+ *   - we detected a change in our published address
+ *     (using some other method), and therefore
+ *   - the published addresses in our descriptor need to change.
+ */
 void
-ip_address_changed(int at_interface)
+ip_address_changed(int on_client_conn)
 {
   const or_options_t *options = get_options();
   int server = server_mode(options);
-  int exit_reject_interfaces = (server && options->ExitRelay
-                                && options->ExitPolicyRejectLocalInterfaces);
 
-  if (at_interface) {
+  if (on_client_conn) {
     if (! server) {
       /* Okay, change our keys. */
       if (init_keys_client() < 0)
@@ -2683,15 +2301,14 @@ ip_address_changed(int at_interface)
         reset_bandwidth_test();
       reset_uptime();
       router_reset_reachability();
+      /* All relays include their IP addresses as their ORPort addresses in
+       * their descriptor.
+       * Exit relays also incorporate interface addresses in their exit
+       * policies, when ExitPolicyRejectLocalInterfaces is set. */
+      mark_my_descriptor_dirty("IP address changed");
     }
   }
 
-  /* Exit relays incorporate interface addresses in their exit policies when
-   * ExitPolicyRejectLocalInterfaces is set */
-  if (exit_reject_interfaces || (server && !at_interface)) {
-    mark_my_descriptor_dirty("IP address changed");
-  }
-
   dns_servers_relaunch_checks();
 }
 
@@ -2702,8 +2319,7 @@ dns_servers_relaunch_checks(void)
 {
   if (server_mode(get_options())) {
     dns_reset_correctness_checks();
-    if (periodic_events_initialized) {
-      tor_assert(check_dns_honesty_event);
+    if (check_dns_honesty_event) {
       periodic_event_reschedule(check_dns_honesty_event);
     }
   }
@@ -2713,8 +2329,6 @@ dns_servers_relaunch_checks(void)
 void
 initialize_mainloop_events(void)
 {
-  initialize_periodic_events();
-
   if (!schedule_active_linked_connections_event) {
     schedule_active_linked_connections_event =
       mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL);
@@ -2732,11 +2346,16 @@ do_main_loop(void)
   /* initialize the periodic events first, so that code that depends on the
    * events being present does not assert.
    */
-  initialize_periodic_events();
+  tor_assert(periodic_events_initialized);
   initialize_mainloop_events();
 
-  /* set up once-a-second callback. */
-  reschedule_per_second_timer();
+  periodic_events_connect_all();
+
+  struct timeval one_second = { 1, 0 };
+  initialize_periodic_events_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  initialize_periodic_events_cb, NULL);
+  event_add(initialize_periodic_events_event, &one_second);
 
 #ifdef HAVE_SYSTEMD_209
   uint64_t watchdog_delay;
@@ -2759,10 +2378,6 @@ do_main_loop(void)
     }
   }
 #endif /* defined(HAVE_SYSTEMD_209) */
-
-  main_loop_should_exit = 0;
-  main_loop_exit_value = 0;
-
 #ifdef ENABLE_RESTART_DEBUGGING
   {
     static int first_time = 1;
@@ -2786,7 +2401,7 @@ do_main_loop(void)
       event_add(tor_shutdown_event_loop_for_restart_event, &restart_after);
     }
   }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
   return run_main_loop_until_done();
 }
@@ -2888,10 +2503,14 @@ run_main_loop_once(void)
  *
  * Shadow won't invoke this function, so don't fill it up with things.
  */
-static int
+STATIC int
 run_main_loop_until_done(void)
 {
   int loop_result = 1;
+
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
+
   do {
     loop_result = run_main_loop_once();
   } while (loop_result == 1);
@@ -2922,7 +2541,6 @@ tor_mainloop_free_all(void)
   smartlist_free(connection_array);
   smartlist_free(closeable_connection_lst);
   smartlist_free(active_linked_connection_lst);
-  periodic_timer_free(second_timer);
   teardown_periodic_events();
   tor_event_free(shutdown_did_not_work_event);
   tor_event_free(initialize_periodic_events_event);
@@ -2930,6 +2548,8 @@ tor_mainloop_free_all(void)
   mainloop_event_free(schedule_active_linked_connections_event);
   mainloop_event_free(postloop_cleanup_ev);
   mainloop_event_free(handle_deferred_signewnym_ev);
+  mainloop_event_free(scheduled_shutdown_ev);
+  mainloop_event_free(rescan_periodic_events_ev);
 
 #ifdef HAVE_SYSTEMD_209
   periodic_timer_free(systemd_watchdog_timer);
@@ -2949,7 +2569,6 @@ tor_mainloop_free_all(void)
   can_complete_circuits = 0;
   quiet_level = 0;
   should_init_bridge_stats = 1;
-  dns_honesty_first_time = 1;
   heartbeat_callback_first_time = 1;
   current_second = 0;
   memset(&current_second_last_changed, 0,
diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h
index c5669fc4e0..1ddfec2162 100644
--- a/src/core/mainloop/mainloop.h
+++ b/src/core/mainloop/mainloop.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -56,15 +56,14 @@ MOCK_DECL(int, connection_count_moribund, (void));
 void directory_all_unreachable(time_t now);
 void directory_info_has_arrived(time_t now, int from_cache, int suppress_logs);
 
-void ip_address_changed(int at_interface);
+void ip_address_changed(int on_client_conn);
 void dns_servers_relaunch_checks(void);
 void reset_all_main_loop_timers(void);
-void reschedule_descriptor_update_check(void);
 void reschedule_directory_downloads(void);
 void reschedule_or_state_save(void);
-void reschedule_dirvote(const or_options_t *options);
 void mainloop_schedule_postloop_cleanup(void);
 void rescan_periodic_events(const or_options_t *options);
+MOCK_DECL(void, schedule_rescan_periodic_events,(void));
 
 void update_current_time(time_t now);
 
@@ -81,34 +80,38 @@ uint64_t get_main_loop_error_count(void);
 uint64_t get_main_loop_idle_count(void);
 
 void periodic_events_on_new_options(const or_options_t *options);
-void reschedule_per_second_timer(void);
 
 void do_signewnym(time_t);
 time_t get_last_signewnym_time(void);
 
+void mainloop_schedule_shutdown(int delay_sec);
+
 void tor_init_connection_lists(void);
 void initialize_mainloop_events(void);
+void initialize_periodic_events(void);
 void tor_mainloop_free_all(void);
 
 struct token_bucket_rw_t;
 
 extern time_t time_of_process_start;
-extern int quiet_level;
 extern struct token_bucket_rw_t global_bucket;
 extern struct token_bucket_rw_t global_relayed_bucket;
 
 #ifdef MAINLOOP_PRIVATE
+STATIC int run_main_loop_until_done(void);
 STATIC void close_closeable_connections(void);
-STATIC void initialize_periodic_events(void);
 STATIC void teardown_periodic_events(void);
 STATIC int get_my_roles(const or_options_t *);
+STATIC int check_network_participation_callback(time_t now,
+                                                const or_options_t *options);
+
 #ifdef TOR_UNIT_TESTS
 extern smartlist_t *connection_array;
 
 /* We need the periodic_event_item_t definition. */
 #include "core/mainloop/periodic.h"
-extern periodic_event_item_t periodic_events[];
-#endif
-#endif /* defined(MAIN_PRIVATE) */
+extern periodic_event_item_t mainloop_periodic_events[];
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(MAINLOOP_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_MAINLOOP_H) */
diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c
new file mode 100644
index 0000000000..0e982d4c40
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.c
@@ -0,0 +1,179 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_pubsub.c
+ * @brief Connect the publish-subscribe code to the main-loop.
+ *
+ * This module is responsible for instantiating all the channels used by the
+ * publish-subscribe code, and making sure that each one's messages are
+ * processed when appropriate.
+ **/
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/pubsub/pubsub_build.h"
+
+/**
+ * Dispatcher to use for delivering messages.
+ **/
+static dispatch_t *the_dispatcher = NULL;
+static pubsub_items_t *the_pubsub_items = NULL;
+/**
+ * A list of mainloop_event_t, indexed by channel ID, to flush the messages
+ * on a channel.
+ **/
+static smartlist_t *alert_events = NULL;
+
+/**
+ * Mainloop event callback: flush all the messages in a channel.
+ *
+ * The channel is encoded as a pointer, and passed via arg.
+ **/
+static void
+flush_channel_event(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  if (!the_dispatcher)
+    return;
+
+  channel_id_t chan = (channel_id_t)(uintptr_t)(arg);
+  dispatch_flush(the_dispatcher, chan, INT_MAX);
+}
+
+/**
+ * Construct our global pubsub object from <b>builder</b>. Return 0 on
+ * success, -1 on failure. */
+int
+tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder)
+{
+  int rv = -1;
+  tor_mainloop_disconnect_pubsub();
+
+  the_dispatcher = pubsub_builder_finalize(builder, &the_pubsub_items);
+  if (! the_dispatcher)
+    goto err;
+
+  rv = 0;
+  goto done;
+ err:
+  tor_mainloop_disconnect_pubsub();
+ done:
+  return rv;
+}
+
+/**
+ * Install libevent events for all of the pubsub channels.
+ *
+ * Invoke this after tor_mainloop_connect_pubsub, and after libevent has been
+ * initialized.
+ */
+void
+tor_mainloop_connect_pubsub_events(void)
+{
+  tor_assert(the_dispatcher);
+  tor_assert(! alert_events);
+
+  const size_t num_channels = get_num_channel_ids();
+  alert_events = smartlist_new();
+  for (size_t i = 0; i < num_channels; ++i) {
+    smartlist_add(alert_events,
+                  mainloop_event_postloop_new(flush_channel_event,
+                                              (void*)(uintptr_t)(i)));
+  }
+}
+
+/**
+ * Dispatch alertfn callback: do nothing. Implements DELIV_NEVER.
+ **/
+static void
+alertfn_never(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  (void)arg;
+}
+
+/**
+ * Dispatch alertfn callback: activate a mainloop event. Implements
+ * DELIV_PROMPT.
+ **/
+static void
+alertfn_prompt(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  mainloop_event_t *event = arg;
+  mainloop_event_activate(event);
+}
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Set the strategy to be used for delivering messages on the named channel.
+ *
+ * This function needs to be called once globally for each channel, to
+ * set up how messages are delivered.
+ **/
+int
+tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                   deliv_strategy_t strategy)
+{
+  channel_id_t chan = get_channel_id(msg_channel_name);
+  if (BUG(chan == ERROR_ID) ||
+      BUG(chan >= smartlist_len(alert_events)))
+    return -1;
+
+  switch (strategy) {
+    case DELIV_NEVER:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_never, NULL);
+      break;
+    case DELIV_PROMPT:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_prompt,
+                            smartlist_get(alert_events, chan));
+      break;
+    case DELIV_IMMEDIATE:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_immediate, NULL);
+      break;
+  }
+  return 0;
+}
+
+/**
+ * Remove all pubsub dispatchers and events from the mainloop.
+ **/
+void
+tor_mainloop_disconnect_pubsub(void)
+{
+  if (the_pubsub_items) {
+    pubsub_items_clear_bindings(the_pubsub_items);
+    pubsub_items_free(the_pubsub_items);
+  }
+  if (alert_events) {
+    SMARTLIST_FOREACH(alert_events, mainloop_event_t *, ev,
+                      mainloop_event_free(ev));
+    smartlist_free(alert_events);
+  }
+  dispatch_free(the_dispatcher);
+}
diff --git a/src/core/mainloop/mainloop_pubsub.h b/src/core/mainloop/mainloop_pubsub.h
new file mode 100644
index 0000000000..3698fd8d03
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_pubsub.h
+ * @brief Header for mainloop_pubsub.c
+ **/
+
+#ifndef TOR_MAINLOOP_PUBSUB_H
+#define TOR_MAINLOOP_PUBSUB_H
+
+struct pubsub_builder_t;
+
+/**
+ * Describe when and how messages are delivered on message channel.
+ *
+ * Every message channel must be associated with one of these strategies.
+ **/
+typedef enum {
+   /**
+    * Never deliver messages automatically.
+    *
+    * If a message channel uses this strategy, then no matter now many
+    * messages are published on it, they are not delivered until something
+    * manually calls dispatch_flush() for that channel
+    **/
+   DELIV_NEVER=0,
+   /**
+    * Deliver messages promptly, via the event loop.
+    *
+    * If a message channel uses this strategy, then publishing a messages
+    * that channel activates an event that causes messages to be handled
+    * later in the mainloop.  The messages will be processed at some point
+    * very soon, delaying only for pending IO events and the like.
+    *
+    * Generally this is the best choice for a delivery strategy, since
+    * it avoids stack explosion.
+    **/
+   DELIV_PROMPT,
+   /**
+    * Deliver messages immediately, skipping the event loop.
+    *
+    * Every event on this channel is flushed immediately after it is queued,
+    * using the stack.
+    *
+    * This delivery type should be used with caution, since it can cause
+    * unexpected call chains, resource starvation, and the like.
+    **/
+   DELIV_IMMEDIATE,
+} deliv_strategy_t;
+
+int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder);
+void tor_mainloop_connect_pubsub_events(void);
+int tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                        deliv_strategy_t strategy);
+void tor_mainloop_disconnect_pubsub(void);
+
+#endif /* !defined(TOR_MAINLOOP_PUBSUB_H) */
diff --git a/src/core/mainloop/mainloop_state.inc b/src/core/mainloop/mainloop_state.inc
new file mode 100644
index 0000000000..34a37caaa2
--- /dev/null
+++ b/src/core/mainloop/mainloop_state.inc
@@ -0,0 +1,19 @@
+
+/**
+ * @file mainloop_state.inc
+ * @brief Declare configuration options for the crypto_ops module.
+ **/
+
+/** Holds state for the mainloop, corresponding to part of the state
+ * file in Tor's DataDirectory. */
+BEGIN_CONF_STRUCT(mainloop_state_t)
+
+/** Number of minutes since the last user-initiated request (as defined by
+ * the dormant net-status system.) Set to zero if we are dormant. */
+CONF_VAR(MinutesSinceUserActivity, POSINT, 0, NULL)
+
+/** True if we were dormant when we last wrote the file; false if we
+ * weren't.  "auto" on initial startup. */
+CONF_VAR(Dormant, AUTOBOOL, 0, "auto")
+
+END_CONF_STRUCT(mainloop_state_t)
diff --git a/src/core/mainloop/mainloop_state_st.h b/src/core/mainloop/mainloop_state_st.h
new file mode 100644
index 0000000000..5649b536f9
--- /dev/null
+++ b/src/core/mainloop/mainloop_state_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_state_st.h
+ * @brief Declare a state structure for mainloop-relevant fields
+ **/
+
+#ifndef TOR_CORE_MAINLOOP_MAINLOOP_STATE_ST_H
+#define TOR_CORE_MAINLOOP_MAINLOOP_STATE_ST_H
+
+#include "lib/conf/confdecl.h"
+
+#define CONF_CONTEXT STRUCT
+#include "core/mainloop/mainloop_state.inc"
+#undef CONF_CONTEXT
+
+typedef struct mainloop_state_t mainloop_state_t;
+
+#endif /* !defined(TOR_CORE_MAINLOOP_MAINLOOP_STATE_ST_H) */
diff --git a/src/core/mainloop/mainloop_sys.c b/src/core/mainloop/mainloop_sys.c
new file mode 100644
index 0000000000..884bae1c59
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.c
@@ -0,0 +1,90 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_sys.c
+ * @brief Declare the "mainloop" subsystem.
+ **/
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop_sys.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_state_st.h"
+#include "core/mainloop/netstatus.h"
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confdecl.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_mainloop_initialize(void)
+{
+  initialize_periodic_events();
+  return 0;
+}
+
+static void
+subsys_mainloop_shutdown(void)
+{
+  tor_mainloop_free_all();
+}
+
+/** Declare a list of state variables for mainloop state. */
+#define CONF_CONTEXT TABLE
+#include "core/mainloop/mainloop_state.inc"
+#undef CONF_CONTEXT
+
+/** Magic number for mainloop state objects */
+#define MAINLOOP_STATE_MAGIC 0x59455449
+
+/**
+ * Format object for mainloop state.
+ **/
+static config_format_t mainloop_state_fmt = {
+  .size = sizeof(mainloop_state_t),
+  .magic = { "mainloop_state",
+             MAINLOOP_STATE_MAGIC,
+             offsetof(mainloop_state_t, magic)
+            },
+  .vars = mainloop_state_t_vars,
+};
+
+/**
+ */
+static int
+mainloop_set_state(void *arg)
+{
+  const mainloop_state_t *state = arg;
+  tor_assert(state->magic == MAINLOOP_STATE_MAGIC);
+
+  netstatus_load_from_state(state, approx_time());
+
+  return 0;
+}
+
+static int
+mainloop_flush_state(void *arg)
+{
+  mainloop_state_t *state = arg;
+  tor_assert(state->magic == MAINLOOP_STATE_MAGIC);
+
+  netstatus_flush_to_state(state, approx_time());
+
+  return 0;
+}
+
+const struct subsys_fns_t sys_mainloop = {
+  .name = "mainloop",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = 5,
+  .initialize = subsys_mainloop_initialize,
+  .shutdown = subsys_mainloop_shutdown,
+
+  .state_format = &mainloop_state_fmt,
+  .set_state = mainloop_set_state,
+  .flush_state = mainloop_flush_state,
+};
diff --git a/src/core/mainloop/mainloop_sys.h b/src/core/mainloop/mainloop_sys.h
new file mode 100644
index 0000000000..b3ade33cd1
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_sys.h
+ * @brief Header for mainloop_sys.c
+ **/
+
+#ifndef MAINLOOP_SYS_H
+#define MAINLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_mainloop;
+
+#endif /* !defined(MAINLOOP_SYS_H) */
diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c
index 1444ca5db2..61a3469eaa 100644
--- a/src/core/mainloop/netstatus.c
+++ b/src/core/mainloop/netstatus.c
@@ -1,14 +1,23 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file netstatus.c
+ * @brief Track whether the network is disabled, dormant, etc.
+ **/
+
 #include "core/or/or.h"
 #include "core/mainloop/netstatus.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_state_st.h"
 #include "app/config/config.h"
 #include "feature/hibernate/hibernate.h"
 
+#include "app/config/or_state_st.h"
+
 /** Return true iff our network is in some sense disabled or shutting down:
  * either we're hibernating, entering hibernation, or the network is turned
  * off with DisableNetwork. */
@@ -26,3 +35,136 @@ net_is_completely_disabled(void)
 {
   return get_options()->DisableNetwork || we_are_fully_hibernating();
 }
+
+/**
+ * The time at which we've last seen "user activity" -- that is, any activity
+ * that should keep us as a participant on the network.
+ *
+ * This is not actually the true time.  We will adjust this forward if
+ * our clock jumps, or if Tor is shut down for a while, so that the time
+ * since our last activity remains as it was before the jump or shutdown.
+ */
+static time_t last_user_activity_seen = 0;
+
+/**
+ * True iff we are currently a "network participant" -- that is, we
+ * are building circuits, fetching directory information, and so on.
+ **/
+static bool participating_on_network = false;
+
+/**
+ * Record the fact that we have seen "user activity" at the time now.  Move
+ * "last activity seen" time forwards, but never backwards.
+ *
+ * If we were previously not participating on the network, set our
+ * participation status to true, and launch periodic events as appropriate.
+ **/
+void
+note_user_activity(time_t now)
+{
+  last_user_activity_seen = MAX(now, last_user_activity_seen);
+
+  if (! participating_on_network) {
+    log_notice(LD_GENERAL, "Tor is no longer dormant.");
+    set_network_participation(true);
+    schedule_rescan_periodic_events();
+  }
+}
+
+/**
+ * Change the time at which "user activitiy" was last seen to <b>now</b>.
+ *
+ * Unlike note_user_actity, this function sets the time without checking
+ * whether it is in the past, and without causing any rescan of periodic events
+ * or change in participation status.
+ */
+void
+reset_user_activity(time_t now)
+{
+  last_user_activity_seen = now;
+}
+
+/**
+ * Return the most recent time at which we recorded "user activity".
+ **/
+time_t
+get_last_user_activity_time(void)
+{
+  return last_user_activity_seen;
+}
+
+/**
+ * Set the field that remembers whether we are currently participating on the
+ * network.  Does not schedule or un-schedule periodic events.
+ **/
+void
+set_network_participation(bool participation)
+{
+  participating_on_network = participation;
+}
+
+/**
+ * Return true iff we are currently participating on the network.
+ **/
+bool
+is_participating_on_network(void)
+{
+  return participating_on_network;
+}
+
+/**
+ * Update 'state' with the last time at which we were active on the network.
+ **/
+void
+netstatus_flush_to_state(mainloop_state_t *state, time_t now)
+{
+  state->Dormant = ! participating_on_network;
+  if (participating_on_network) {
+    time_t sec_since_activity = MAX(0, now - last_user_activity_seen);
+    state->MinutesSinceUserActivity = (int)(sec_since_activity / 60);
+  } else {
+    state->MinutesSinceUserActivity = 0;
+  }
+}
+
+/**
+ * Update our current view of network participation from an or_state_t object.
+ **/
+void
+netstatus_load_from_state(const mainloop_state_t *state, time_t now)
+{
+  time_t last_activity;
+  if (state->Dormant == -1) { // Initial setup.
+    if (get_options()->DormantOnFirstStartup) {
+      last_activity = 0;
+      participating_on_network = false;
+    } else {
+      // Start up as active, treat activity as happening now.
+      last_activity = now;
+      participating_on_network = true;
+    }
+  } else if (state->Dormant) {
+    last_activity = 0;
+    participating_on_network = false;
+  } else {
+    last_activity = now - 60 * state->MinutesSinceUserActivity;
+    participating_on_network = true;
+  }
+  if (get_options()->DormantCanceledByStartup) {
+    last_activity = now;
+    participating_on_network = true;
+  }
+  reset_user_activity(last_activity);
+}
+
+/**
+ * Adjust the time at which the user was last active by <b>seconds_diff</b>
+ * in response to a clock jump.
+ */
+void
+netstatus_note_clock_jumped(time_t seconds_diff)
+{
+  time_t last_active = get_last_user_activity_time();
+  if (last_active)
+    reset_user_activity(last_active + seconds_diff);
+}
diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h
index ae8547b30d..5f54e54553 100644
--- a/src/core/mainloop/netstatus.h
+++ b/src/core/mainloop/netstatus.h
@@ -1,13 +1,32 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file netstatus.h
+ * @brief Header for netstatus.c
+ **/
+
 #ifndef TOR_NETSTATUS_H
 #define TOR_NETSTATUS_H
 
 int net_is_disabled(void);
 int net_is_completely_disabled(void);
 
-#endif
+void note_user_activity(time_t now);
+void reset_user_activity(time_t now);
+time_t get_last_user_activity_time(void);
+
+void set_network_participation(bool participation);
+bool is_participating_on_network(void);
+
+struct mainloop_state_t;
+
+void netstatus_flush_to_state(struct mainloop_state_t *state, time_t now);
+void netstatus_load_from_state(const struct mainloop_state_t *state,
+                               time_t now);
+void netstatus_note_clock_jumped(time_t seconds_diff);
+
+#endif /* !defined(TOR_NETSTATUS_H) */
diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c
index 2651bbbc89..b5fd8fab61 100644
--- a/src/core/mainloop/periodic.c
+++ b/src/core/mainloop/periodic.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,9 +6,22 @@
  *
  * \brief Generic backend for handling periodic events.
  *
- * The events in this module are used by main.c to track items that need
+ * The events in this module are used to track items that need
  * to fire once every N seconds, possibly picking a new interval each time
- * that they fire.  See periodic_events[] in main.c for examples.
+ * that they fire.  See periodic_events[] in mainloop.c for examples.
+ *
+ * This module manages a global list of periodic_event_item_t objects,
+ * each corresponding to a single event.  To register an event, pass it to
+ * periodic_events_register() when initializing your subsystem.
+ *
+ * Registering an event makes the periodic event subsystem know about it, but
+ * doesn't cause the event to get created immediately.  Before the event can
+ * be started, periodic_event_connect_all() must be called by mainloop.c to
+ * connect all the events to Libevent.
+ *
+ * We expect that periodic_event_item_t objects will be statically allocated;
+ * we set them up and tear them down here, but we don't take ownership of
+ * them.
  */
 
 #include "core/or/or.h"
@@ -16,7 +29,6 @@
 #include "app/config/config.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/periodic.h"
-#include "lib/evloop/compat_libevent.h"
 
 /** We disable any interval greater than this number of seconds, on the
  * grounds that it is probably an absolute time mistakenly passed in as a
@@ -24,6 +36,12 @@
  */
 static const int MAX_INTERVAL = 10 * 365 * 86400;
 
+/**
+ * Global list of periodic events that have been registered with
+ * <b>periodic_event_register</b>.
+ **/
+static smartlist_t *the_periodic_events = NULL;
+
 /** Set the event <b>event</b> to run in <b>next_interval</b> seconds from
  * now. */
 static void
@@ -45,10 +63,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   periodic_event_item_t *event = data;
   tor_assert(ev == event->ev);
 
-  if (BUG(!periodic_event_is_enabled(event))) {
-    return;
-  }
-
   time_t now = time(NULL);
   update_current_time(now);
   const or_options_t *options = get_options();
@@ -57,7 +71,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   int next_interval = 0;
 
   if (!periodic_event_is_enabled(event)) {
-    /* The event got disabled from inside its callback; no need to
+    /* The event got disabled from inside its callback, or before: no need to
      * reschedule. */
     return;
   }
@@ -91,15 +105,16 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
 void
 periodic_event_reschedule(periodic_event_item_t *event)
 {
-  /* Don't reschedule a disabled event. */
-  if (periodic_event_is_enabled(event)) {
+  /* Don't reschedule a disabled or uninitialized event. */
+  if (event->ev && periodic_event_is_enabled(event)) {
     periodic_event_set_interval(event, 1);
   }
 }
 
-/** Initializes the libevent backend for a periodic event. */
+/** Connects a periodic event to the Libevent backend.  Does not launch the
+ * event immediately. */
 void
-periodic_event_setup(periodic_event_item_t *event)
+periodic_event_connect(periodic_event_item_t *event)
 {
   if (event->ev) { /* Already setup? This is a bug */
     log_err(LD_BUG, "Initial dispatch should only be done once.");
@@ -117,7 +132,7 @@ void
 periodic_event_launch(periodic_event_item_t *event)
 {
   if (! event->ev) { /* Not setup? This is a bug */
-    log_err(LD_BUG, "periodic_event_launch without periodic_event_setup");
+    log_err(LD_BUG, "periodic_event_launch without periodic_event_connect");
     tor_assert(0);
   }
   /* Event already enabled? This is a bug */
@@ -131,9 +146,9 @@ periodic_event_launch(periodic_event_item_t *event)
   periodic_event_dispatch(event->ev, event);
 }
 
-/** Release all storage associated with <b>event</b> */
-void
-periodic_event_destroy(periodic_event_item_t *event)
+/** Disconnect and unregister the periodic event in <b>event</b> */
+static void
+periodic_event_disconnect(periodic_event_item_t *event)
 {
   if (!event)
     return;
@@ -177,3 +192,177 @@ periodic_event_disable(periodic_event_item_t *event)
   mainloop_event_cancel(event->ev);
   event->enabled = 0;
 }
+
+/**
+ * Disable an event, then schedule it to run once.
+ * Do nothing if the event was already disabled.
+ */
+void
+periodic_event_schedule_and_disable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  if (!periodic_event_is_enabled(event))
+    return;
+
+  periodic_event_disable(event);
+
+  mainloop_event_activate(event->ev);
+}
+
+/**
+ * Add <b>item</b> to the list of periodic events.
+ *
+ * Note that <b>item</b> should be statically allocated: we do not
+ * take ownership of it.
+ **/
+void
+periodic_events_register(periodic_event_item_t *item)
+{
+  if (!the_periodic_events)
+    the_periodic_events = smartlist_new();
+
+  if (BUG(smartlist_contains(the_periodic_events, item)))
+    return;
+
+  smartlist_add(the_periodic_events, item);
+}
+
+/**
+ * Make all registered periodic events connect to the libevent backend.
+ */
+void
+periodic_events_connect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (item->ev)
+      continue;
+    periodic_event_connect(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Reset all the registered periodic events so we'll do all our actions again
+ * as if we just started up.
+ *
+ * Useful if our clock just moved back a long time from the future,
+ * so we don't wait until that future arrives again before acting.
+ */
+void
+periodic_events_reset_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    periodic_event_reschedule(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Return the registered periodic event whose name is <b>name</b>.
+ * Return NULL if no such event is found.
+ */
+periodic_event_item_t *
+periodic_events_find(const char *name)
+{
+  if (! the_periodic_events)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (strcmp(name, item->name) == 0)
+      return item;
+  } SMARTLIST_FOREACH_END(item);
+  return NULL;
+}
+
+/**
+ * Start or stop registered periodic events, depending on our current set of
+ * roles.
+ *
+ * Invoked when our list of roles, or the net_disabled flag has changed.
+ **/
+void
+periodic_events_rescan_by_roles(int roles, bool net_disabled)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    int enable = !!(item->roles & roles);
+
+    /* Handle the event flags. */
+    if (net_disabled &&
+        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
+      enable = 0;
+    }
+
+    /* Enable the event if needed. It is safe to enable an event that was
+     * already enabled. Same goes for disabling it. */
+    if (enable) {
+      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
+      periodic_event_enable(item);
+    } else {
+      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
+      if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) {
+        periodic_event_schedule_and_disable(item);
+      } else {
+        periodic_event_disable(item);
+      }
+    }
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Invoked at shutdown: disconnect and unregister all periodic events.
+ *
+ * Does not free the periodic_event_item_t object themselves, because we do
+ * not own them.
+ */
+void
+periodic_events_disconnect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    periodic_event_disconnect(item);
+  } SMARTLIST_FOREACH_END(item);
+
+  smartlist_free(the_periodic_events);
+}
+
+#define LONGEST_TIMER_PERIOD (30 * 86400)
+/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
+ * clipped to the range [1 second, LONGEST_TIMER_PERIOD].
+ *
+ * We use this to answer the question, "how many seconds is it from now until
+ * next" in periodic timer callbacks.  Don't use it for other purposes
+ **/
+int
+safe_timer_diff(time_t now, time_t next)
+{
+  if (next > now) {
+    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
+     * and nothing that could run Tor. It's a bug if 'next' is around then.
+     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
+     * Bang. We cannot extrapolate past a singularity, but there was probably
+     * nothing that could run Tor then, either.
+     **/
+    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
+
+    if (next - LONGEST_TIMER_PERIOD > now)
+      return LONGEST_TIMER_PERIOD;
+    return (int)(next - now);
+  } else {
+    return 1;
+  }
+}
diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h
index e49fafd174..de556a6bdb 100644
--- a/src/core/mainloop/periodic.h
+++ b/src/core/mainloop/periodic.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file periodic.h
+ * @brief Header for periodic.c
+ **/
+
 #ifndef TOR_PERIODIC_H
 #define TOR_PERIODIC_H
 
@@ -15,6 +20,10 @@
 #define PERIODIC_EVENT_ROLE_BRIDGEAUTH  (1U << 4)
 #define PERIODIC_EVENT_ROLE_HS_SERVICE  (1U << 5)
 #define PERIODIC_EVENT_ROLE_DIRSERVER   (1U << 6)
+#define PERIODIC_EVENT_ROLE_CONTROLEV   (1U << 7)
+
+#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8)
+#define PERIODIC_EVENT_ROLE_ALL         (1U << 9)
 
 /* Helper macro to make it a bit less annoying to defined groups of roles that
  * are often used. */
@@ -25,10 +34,6 @@
 /* Authorities that is both bridge and directory. */
 #define PERIODIC_EVENT_ROLE_AUTHORITIES \
   (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH)
-/* All roles. */
-#define PERIODIC_EVENT_ROLE_ALL \
-  (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \
-   PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER)
 
 /*
  * Event flags which can change the behavior of an event.
@@ -39,6 +44,11 @@
  * the net_is_disabled() check. */
 #define PERIODIC_EVENT_FLAG_NEED_NET  (1U << 0)
 
+/* Indicate that if the event is enabled, it needs to be run once before
+ * it becomes disabled.
+ */
+#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE  (1U << 1)
+
 /** Callback function for a periodic event to take action.  The return value
 * influences the next time the function will get called.  Return
 * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled
@@ -66,8 +76,10 @@ typedef struct periodic_event_item_t {
 } periodic_event_item_t;
 
 /** events will get their interval from first execution */
+#ifndef COCCI
 #define PERIODIC_EVENT(fn, r, f) { fn##_callback, 0, NULL, #fn, r, f, 0 }
 #define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL, 0, 0, 0 }
+#endif
 
 /* Return true iff the given event was setup before thus is enabled to be
  * scheduled. */
@@ -78,11 +90,20 @@ periodic_event_is_enabled(const periodic_event_item_t *item)
 }
 
 void periodic_event_launch(periodic_event_item_t *event);
-void periodic_event_setup(periodic_event_item_t *event);
-void periodic_event_destroy(periodic_event_item_t *event);
+void periodic_event_connect(periodic_event_item_t *event);
+//void periodic_event_disconnect(periodic_event_item_t *event);
 void periodic_event_reschedule(periodic_event_item_t *event);
 void periodic_event_enable(periodic_event_item_t *event);
 void periodic_event_disable(periodic_event_item_t *event);
+void periodic_event_schedule_and_disable(periodic_event_item_t *event);
 
-#endif /* !defined(TOR_PERIODIC_H) */
+void periodic_events_register(periodic_event_item_t *item);
+void periodic_events_connect_all(void);
+void periodic_events_reset_all(void);
+periodic_event_item_t *periodic_events_find(const char *name);
+void periodic_events_rescan_by_roles(int roles, bool net_disabled);
+void periodic_events_disconnect_all(void);
+
+int safe_timer_diff(time_t now, time_t next);
 
+#endif /* !defined(TOR_PERIODIC_H) */
diff --git a/src/core/or/.may_include b/src/core/or/.may_include
new file mode 100644
index 0000000000..beb12f155d
--- /dev/null
+++ b/src/core/or/.may_include
@@ -0,0 +1,40 @@
+!advisory
+
+orconfig.h
+
+lib/arch/*.h
+lib/buf/*.h
+lib/cc/*.h
+lib/compress/*.h
+lib/container/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/defs/*.h
+lib/encoding/*.h
+lib/err/*.h
+lib/evloop/*.h
+lib/fs/*.h
+lib/geoip/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/math/*.h
+lib/net/*.h
+lib/pubsub/*.h
+lib/string/*.h
+lib/subsys/*.h
+lib/test/*.h
+lib/testsupport/*.h
+lib/thread/*.h
+lib/time/*.h
+lib/tls/*.h
+lib/wallclock/*.h
+
+trunnel/*.h
+
+core/mainloop/*.h
+core/proto/*.h
+core/crypto/*.h
+core/or/*.h
+
+ext/*.h
diff --git a/src/core/or/addr_policy_st.h b/src/core/or/addr_policy_st.h
index a75f1a731d..08d16ee616 100644
--- a/src/core/or/addr_policy_st.h
+++ b/src/core/or/addr_policy_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file addr_policy_st.h
+ * @brief Address policy structures.
+ **/
+
 #ifndef TOR_ADDR_POLICY_ST_H
 #define TOR_ADDR_POLICY_ST_H
 
@@ -33,7 +38,7 @@ struct addr_policy_t {
   /** Base address to accept or reject.
    *
    * Note that wildcards are treated
-   * differntly depending on address family. An AF_UNSPEC address means
+   * differently depending on address family. An AF_UNSPEC address means
    * "All addresses, IPv4 or IPv6." An AF_INET address with maskbits==0 means
    * "All IPv4 addresses" and an AF_INET6 address with maskbits == 0 means
    * "All IPv6 addresses".
@@ -43,4 +48,4 @@ struct addr_policy_t {
   uint16_t prt_max; /**< Highest port number to accept/reject. */
 };
 
-#endif
+#endif /* !defined(TOR_ADDR_POLICY_ST_H) */
diff --git a/src/core/or/address_set.c b/src/core/or/address_set.c
index 7ada4446c4..9bd3cc0f2d 100644
--- a/src/core/or/address_set.c
+++ b/src/core/or/address_set.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/container/bloomfilt.h"
 #include "lib/crypt_ops/crypto_rand.h"
 
-/* Wrap our hash function to have the signature that the bloom filter
+/** Wrap our hash function to have the signature that the bloom filter
  * needs. */
 static uint64_t
 bloomfilt_addr_hash(const struct sipkey *key,
diff --git a/src/core/or/address_set.h b/src/core/or/address_set.h
index a505d31628..b4d94b65a9 100644
--- a/src/core/or/address_set.h
+++ b/src/core/or/address_set.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/or/cell_queue_st.h b/src/core/or/cell_queue_st.h
index 130b95a011..0681dba1b8 100644
--- a/src/core/or/cell_queue_st.h
+++ b/src/core/or/cell_queue_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cell_queue_st.h
+ * @brief Cell queue structures
+ **/
+
 #ifndef PACKED_CELL_ST_H
 #define PACKED_CELL_ST_H
 
@@ -22,8 +27,8 @@ struct packed_cell_t {
  * or_connection_t's outbuf. */
 struct cell_queue_t {
   /** Linked list of packed_cell_t*/
-  TOR_SIMPLEQ_HEAD(cell_simpleq, packed_cell_t) head;
+  TOR_SIMPLEQ_HEAD(cell_simpleq_t, packed_cell_t) head;
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
+#endif /* !defined(PACKED_CELL_ST_H) */
diff --git a/src/core/or/cell_st.h b/src/core/or/cell_st.h
index 7ab7eceb50..a640d6a456 100644
--- a/src/core/or/cell_st.h
+++ b/src/core/or/cell_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cell_st.h
+ * @brief Fixed-size cell structure.
+ **/
+
 #ifndef CELL_ST_H
 #define CELL_ST_H
 
@@ -16,5 +21,4 @@ struct cell_t {
   uint8_t payload[CELL_PAYLOAD_SIZE]; /**< Cell body. */
 };
 
-#endif
-
+#endif /* !defined(CELL_ST_H) */
diff --git a/src/core/or/channel.c b/src/core/or/channel.c
index 9649bdf278..50c03de846 100644
--- a/src/core/or/channel.c
+++ b/src/core/or/channel.c
@@ -1,5 +1,4 @@
-
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -52,10 +51,10 @@
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 
 /* This one's for stuff only channel.c and the test suite should see */
-#define CHANNEL_PRIVATE_
+#define CHANNEL_FILE_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
@@ -85,6 +84,13 @@
 
 #include "core/or/cell_queue_st.h"
 
+/* Static function prototypes */
+
+static bool channel_matches_target_addr_for_extend(
+                                          channel_t *chan,
+                                          const tor_addr_t *target_ipv4_addr,
+                                          const tor_addr_t *target_ipv6_addr);
+
 /* Global lists of channels */
 
 /* All channel_t instances */
@@ -107,7 +113,7 @@ static smartlist_t *finished_listeners = NULL;
 
 /** Map from channel->global_identifier to channel.  Contains the same
  * elements as all_channels. */
-static HT_HEAD(channel_gid_map, channel_s) channel_gid_map = HT_INITIALIZER();
+static HT_HEAD(channel_gid_map, channel_t) channel_gid_map = HT_INITIALIZER();
 
 static unsigned
 channel_id_hash(const channel_t *chan)
@@ -119,13 +125,13 @@ channel_id_eq(const channel_t *a, const channel_t *b)
 {
   return a->global_identifier == b->global_identifier;
 }
-HT_PROTOTYPE(channel_gid_map, channel_s, gidmap_node,
-             channel_id_hash, channel_id_eq)
-HT_GENERATE2(channel_gid_map, channel_s, gidmap_node,
+HT_PROTOTYPE(channel_gid_map, channel_t, gidmap_node,
+             channel_id_hash, channel_id_eq);
+HT_GENERATE2(channel_gid_map, channel_t, gidmap_node,
              channel_id_hash, channel_id_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
-HANDLE_IMPL(channel, channel_s,)
+HANDLE_IMPL(channel, channel_t,)
 
 /* Counter for ID numbers */
 static uint64_t n_channels_allocated = 0;
@@ -138,13 +144,13 @@ static uint64_t n_channels_allocated = 0;
  * If more than one channel exists, follow the next_with_same_id pointer
  * as a linked list.
  */
-static HT_HEAD(channel_idmap, channel_idmap_entry_s) channel_identity_map =
+static HT_HEAD(channel_idmap, channel_idmap_entry_t) channel_identity_map =
   HT_INITIALIZER();
 
-typedef struct channel_idmap_entry_s {
-  HT_ENTRY(channel_idmap_entry_s) node;
+typedef struct channel_idmap_entry_t {
+  HT_ENTRY(channel_idmap_entry_t) node;
   uint8_t digest[DIGEST_LEN];
-  TOR_LIST_HEAD(channel_list_s, channel_s) channel_list;
+  TOR_LIST_HEAD(channel_list_t, channel_t) channel_list;
 } channel_idmap_entry_t;
 
 static inline unsigned
@@ -160,10 +166,10 @@ channel_idmap_eq(const channel_idmap_entry_t *a,
   return tor_memeq(a->digest, b->digest, DIGEST_LEN);
 }
 
-HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
-             channel_idmap_eq)
-HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
-             channel_idmap_eq, 0.5,  tor_reallocarray_, tor_free_)
+HT_PROTOTYPE(channel_idmap, channel_idmap_entry_t, node, channel_idmap_hash,
+             channel_idmap_eq);
+HT_GENERATE2(channel_idmap, channel_idmap_entry_t, node, channel_idmap_hash,
+             channel_idmap_eq, 0.5,  tor_reallocarray_, tor_free_);
 
 /* Functions to maintain the digest map */
 static void channel_remove_from_digest_map(channel_t *chan);
@@ -1092,23 +1098,6 @@ channel_get_cell_handler(channel_t *chan)
 }
 
 /**
- * Return the variable-length cell handler for a channel.
- *
- * This function gets the handler for incoming variable-length cells
- * installed on a channel.
- */
-channel_var_cell_handler_fn_ptr
-channel_get_var_cell_handler(channel_t *chan)
-{
-  tor_assert(chan);
-
-  if (CHANNEL_CAN_HANDLE_CELLS(chan))
-    return chan->var_cell_handler;
-
-  return NULL;
-}
-
-/**
  * Set both cell handlers for a channel.
  *
  * This function sets both the fixed-length and variable length cell handlers
@@ -1116,9 +1105,7 @@ channel_get_var_cell_handler(channel_t *chan)
  */
 void
 channel_set_cell_handlers(channel_t *chan,
-                          channel_cell_handler_fn_ptr cell_handler,
-                          channel_var_cell_handler_fn_ptr
-                            var_cell_handler)
+                          channel_cell_handler_fn_ptr cell_handler)
 {
   tor_assert(chan);
   tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan));
@@ -1126,13 +1113,9 @@ channel_set_cell_handlers(channel_t *chan,
   log_debug(LD_CHANNEL,
            "Setting cell_handler callback for channel %p to %p",
            chan, cell_handler);
-  log_debug(LD_CHANNEL,
-           "Setting var_cell_handler callback for channel %p to %p",
-           chan, var_cell_handler);
 
   /* Change them */
   chan->cell_handler = cell_handler;
-  chan->var_cell_handler = var_cell_handler;
 }
 
 /*
@@ -1443,6 +1426,7 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
   int ret = -1;
   size_t cell_bytes;
+  uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids);
 
   tor_assert(chan);
   tor_assert(cell);
@@ -1477,6 +1461,16 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
   /* Successfully sent the cell. */
   ret = 0;
 
+  /* Update padding statistics for the packed codepath.. */
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
+  if (command == CELL_PADDING)
+    rep_hist_padding_count_write(PADDING_TYPE_CELL);
+  if (chan->padding_enabled) {
+    rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
+    if (command == CELL_PADDING)
+      rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
+  }
+
  done:
   return ret;
 }
@@ -1893,11 +1887,11 @@ channel_do_open_actions(channel_t *chan)
         geoip_note_client_seen(GEOIP_CLIENT_CONNECT,
                                &remote_addr, transport_name,
                                now);
-        tor_free(transport_name);
         /* Notify the DoS subsystem of a new client. */
         if (tlschan && tlschan->conn) {
           dos_new_client_conn(tlschan->conn, transport_name);
         }
+        tor_free(transport_name);
       }
       /* Otherwise the underlying transport can't tell us this, so skip it */
     }
@@ -2372,7 +2366,7 @@ channel_is_better(channel_t *a, channel_t *b)
   if (!a->is_canonical_to_peer && b->is_canonical_to_peer) return 0;
 
   /*
-   * Okay, if we're here they tied on canonicity, the prefer the older
+   * Okay, if we're here they tied on canonicity. Prefer the older
    * connection, so that the adversary can't create a new connection
    * and try to switch us over to it (which will leak information
    * about long-lived circuits). Additionally, switching connections
@@ -2397,19 +2391,23 @@ channel_is_better(channel_t *a, channel_t *b)
 /**
  * Get a channel to extend a circuit.
  *
- * Pick a suitable channel to extend a circuit to given the desired digest
- * the address we believe is correct for that digest; this tries to see
- * if we already have one for the requested endpoint, but if there is no good
- * channel, set *msg_out to a message describing the channel's state
- * and our next action, and set *launch_out to a boolean indicated whether
- * the caller should try to launch a new channel with channel_connect().
+ * Given the desired relay identity, pick a suitable channel to extend a
+ * circuit to the target IPv4 or IPv6 address requsted by the client. Search
+ * for an existing channel for the requested endpoint. Make sure the channel
+ * is usable for new circuits, and matches one of the target addresses.
+ *
+ * Try to return the best channel. But if there is no good channel, set
+ * *msg_out to a message describing the channel's state and our next action,
+ * and set *launch_out to a boolean indicated whether the caller should try to
+ * launch a new channel with channel_connect().
  */
-channel_t *
-channel_get_for_extend(const char *rsa_id_digest,
-                       const ed25519_public_key_t *ed_id,
-                       const tor_addr_t *target_addr,
-                       const char **msg_out,
-                       int *launch_out)
+MOCK_IMPL(channel_t *,
+channel_get_for_extend,(const char *rsa_id_digest,
+                        const ed25519_public_key_t *ed_id,
+                        const tor_addr_t *target_ipv4_addr,
+                        const tor_addr_t *target_ipv6_addr,
+                        const char **msg_out,
+                        int *launch_out))
 {
   channel_t *chan, *best = NULL;
   int n_inprogress_goodaddr = 0, n_old = 0;
@@ -2420,9 +2418,7 @@ channel_get_for_extend(const char *rsa_id_digest,
 
   chan = channel_find_by_remote_identity(rsa_id_digest, ed_id);
 
-  /* Walk the list, unrefing the old one and refing the new at each
-   * iteration.
-   */
+  /* Walk the list of channels */
   for (; chan; chan = channel_next_with_rsa_identity(chan)) {
     tor_assert(tor_memeq(chan->identity_digest,
                          rsa_id_digest, DIGEST_LEN));
@@ -2441,11 +2437,15 @@ channel_get_for_extend(const char *rsa_id_digest,
       continue;
     }
 
+    const bool matches_target =
+      channel_matches_target_addr_for_extend(chan,
+                                             target_ipv4_addr,
+                                             target_ipv6_addr);
     /* Never return a non-open connection. */
     if (!CHANNEL_IS_OPEN(chan)) {
       /* If the address matches, don't launch a new connection for this
        * circuit. */
-      if (channel_matches_target_addr_for_extend(chan, target_addr))
+      if (matches_target)
         ++n_inprogress_goodaddr;
       continue;
     }
@@ -2458,8 +2458,7 @@ channel_get_for_extend(const char *rsa_id_digest,
 
     /* Only return canonical connections or connections where the address
      * is the address we wanted. */
-    if (!channel_is_canonical(chan) &&
-        !channel_matches_target_addr_for_extend(chan, target_addr)) {
+    if (!channel_is_canonical(chan) && !matches_target) {
       ++n_noncanonical;
       continue;
     }
@@ -2841,8 +2840,8 @@ channel_get_actual_remote_address(channel_t *chan)
  * Subsequent calls to channel_get_{actual,canonical}_remote_{address,descr}
  * may invalidate the return value from this function.
  */
-const char *
-channel_get_canonical_remote_descr(channel_t *chan)
+MOCK_IMPL(const char *,
+channel_get_canonical_remote_descr,(channel_t *chan))
 {
   tor_assert(chan);
   tor_assert(chan->get_remote_descr);
@@ -3303,20 +3302,33 @@ channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info)
 }
 
 /**
- * Check if a channel matches a given target address; return true iff we do.
+ * Check if a channel matches the given target IPv4 or IPv6 addresses.
+ * If either address matches, return true. If neither address matches,
+ * return false.
+ *
+ * Both addresses can't be NULL.
  *
  * This function calls into the lower layer and asks if this channel thinks
- * it matches a given target address for circuit extension purposes.
+ * it matches the target addresses for circuit extension purposes.
  */
-int
+static bool
 channel_matches_target_addr_for_extend(channel_t *chan,
-                                       const tor_addr_t *target)
+                                       const tor_addr_t *target_ipv4_addr,
+                                       const tor_addr_t *target_ipv6_addr)
 {
   tor_assert(chan);
   tor_assert(chan->matches_target);
-  tor_assert(target);
 
-  return chan->matches_target(chan, target);
+  IF_BUG_ONCE(!target_ipv4_addr && !target_ipv6_addr)
+    return false;
+
+  if (target_ipv4_addr && chan->matches_target(chan, target_ipv4_addr))
+    return true;
+
+  if (target_ipv6_addr && chan->matches_target(chan, target_ipv6_addr))
+    return true;
+
+  return false;
 }
 
 /**
@@ -3389,7 +3401,7 @@ channel_sort_by_ed25519_identity(const void **a_, const void **b_)
  * all of which MUST have the same RSA ID.  (They MAY have different
  * Ed25519 IDs.) */
 static void
-channel_rsa_id_group_set_badness(struct channel_list_s *lst, int force)
+channel_rsa_id_group_set_badness(struct channel_list_t *lst, int force)
 {
   /*XXXX This function should really be about channels. 15056 */
   channel_t *chan = TOR_LIST_FIRST(lst);
diff --git a/src/core/or/channel.h b/src/core/or/channel.h
index d41f0d70bb..fa4ce4f703 100644
--- a/src/core/or/channel.h
+++ b/src/core/or/channel.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@
 #include "lib/container/handles.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 
+#include "ext/ht.h"
 #include "tor_queue.h"
 
 #define tor_timer_t timeout
@@ -22,7 +23,6 @@ struct tor_timer_t;
 /* Channel handler function pointer typedefs */
 typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *);
 typedef void (*channel_cell_handler_fn_ptr)(channel_t *, cell_t *);
-typedef void (*channel_var_cell_handler_fn_ptr)(channel_t *, var_cell_t *);
 
 /**
  * This enum is used by channelpadding to decide when to pad channels.
@@ -48,7 +48,7 @@ typedef enum {
 /* channel states for channel_t */
 
 typedef enum {
-  /*
+  /**
    * Closed state - channel is inactive
    *
    * Permitted transitions from:
@@ -57,7 +57,7 @@ typedef enum {
    *   - CHANNEL_STATE_OPENING
    */
   CHANNEL_STATE_CLOSED = 0,
-  /*
+  /**
    * Opening state - channel is trying to connect
    *
    * Permitted transitions from:
@@ -68,7 +68,7 @@ typedef enum {
    *   - CHANNEL_STATE_OPEN
    */
   CHANNEL_STATE_OPENING,
-  /*
+  /**
    * Open state - channel is active and ready for use
    *
    * Permitted transitions from:
@@ -80,7 +80,7 @@ typedef enum {
    *   - CHANNEL_STATE_MAINT
    */
   CHANNEL_STATE_OPEN,
-  /*
+  /**
    * Maintenance state - channel is temporarily offline for subclass specific
    *   maintenance activities such as TLS renegotiation.
    *
@@ -92,7 +92,7 @@ typedef enum {
    *   - CHANNEL_STATE_OPEN
    */
   CHANNEL_STATE_MAINT,
-  /*
+  /**
    * Closing state - channel is shutting down
    *
    * Permitted transitions from:
@@ -103,7 +103,7 @@ typedef enum {
    *   - CHANNEL_STATE_ERROR
    */
   CHANNEL_STATE_CLOSING,
-  /*
+  /**
    * Error state - channel has experienced a permanent error
    *
    * Permitted transitions from:
@@ -115,7 +115,7 @@ typedef enum {
    *   - None
    */
   CHANNEL_STATE_ERROR,
-  /*
+  /**
    * Placeholder for maximum state value
    */
   CHANNEL_STATE_LAST
@@ -124,7 +124,7 @@ typedef enum {
 /* channel listener states for channel_listener_t */
 
 typedef enum {
-  /*
+  /**
    * Closed state - channel listener is inactive
    *
    * Permitted transitions from:
@@ -133,7 +133,7 @@ typedef enum {
    *   - CHANNEL_LISTENER_STATE_LISTENING
    */
   CHANNEL_LISTENER_STATE_CLOSED = 0,
-  /*
+  /**
    * Listening state - channel listener is listening for incoming
    * connections
    *
@@ -144,7 +144,7 @@ typedef enum {
    *   - CHANNEL_LISTENER_STATE_ERROR
    */
   CHANNEL_LISTENER_STATE_LISTENING,
-  /*
+  /**
    * Closing state - channel listener is shutting down
    *
    * Permitted transitions from:
@@ -154,7 +154,7 @@ typedef enum {
    *   - CHANNEL_LISTENER_STATE_ERROR
    */
   CHANNEL_LISTENER_STATE_CLOSING,
-  /*
+  /**
    * Error state - channel listener has experienced a permanent error
    *
    * Permitted transitions from:
@@ -164,7 +164,7 @@ typedef enum {
    *   - None
    */
   CHANNEL_LISTENER_STATE_ERROR,
-  /*
+  /**
    * Placeholder for maximum state value
    */
   CHANNEL_LISTENER_STATE_LAST
@@ -178,15 +178,15 @@ typedef enum {
  * to a particular node, and once constructed support the abstract operations
  * defined below.
  */
-struct channel_s {
+struct channel_t {
   /** Magic number for type-checking cast macros */
   uint32_t magic;
 
   /** List entry for hashtable for global-identifier lookup. */
-  HT_ENTRY(channel_s) gidmap_node;
+  HT_ENTRY(channel_t) gidmap_node;
 
   /** Handle entry for handle-based lookup */
-  HANDLE_ENTRY(channel, channel_s);
+  HANDLE_ENTRY(channel, channel_t);
 
   /** Current channel state */
   channel_state_t state;
@@ -267,21 +267,21 @@ struct channel_s {
 
   /** State variable for use by the scheduler */
   enum {
-    /*
+    /**
      * The channel is not open, or it has a full output buffer but no queued
      * cells.
      */
     SCHED_CHAN_IDLE = 0,
-    /*
+    /**
      * The channel has space on its output buffer to write, but no queued
      * cells.
      */
     SCHED_CHAN_WAITING_FOR_CELLS,
-    /*
+    /**
      * The scheduler has queued cells but no output buffer space to write.
      */
     SCHED_CHAN_WAITING_TO_WRITE,
-    /*
+    /**
      * The scheduler has both queued cells and output buffer space, and is
      * eligible for the scheduler loop.
      */
@@ -320,7 +320,6 @@ struct channel_s {
 
   /** Registered handlers for incoming cells */
   channel_cell_handler_fn_ptr cell_handler;
-  channel_var_cell_handler_fn_ptr var_cell_handler;
 
   /* Methods implemented by the lower layer */
 
@@ -395,7 +394,7 @@ struct channel_s {
    * Linked list of channels with the same RSA identity digest, for use with
    * the digest->channel map
    */
-  TOR_LIST_ENTRY(channel_s) next_with_same_id;
+  TOR_LIST_ENTRY(channel_t) next_with_same_id;
 
   /** Circuit mux for circuits sending on this channel */
   circuitmux_t *cmux;
@@ -442,9 +441,9 @@ struct channel_s {
   ratelim_t last_warned_circ_ids_exhausted;
 
   /** Channel timestamps for cell channels */
-  time_t timestamp_client; /* Client used this, according to relay.c */
-  time_t timestamp_recv; /* Cell received from lower layer */
-  time_t timestamp_xmit; /* Cell sent to lower layer */
+  time_t timestamp_client; /*(< Client used this, according to relay.c */
+  time_t timestamp_recv; /**< Cell received from lower layer */
+  time_t timestamp_xmit; /**< Cell sent to lower layer */
 
   /** Timestamp for run_connection_housekeeping(). We update this once a
    * second when we run housekeeping and find a circuit on this channel, and
@@ -456,16 +455,17 @@ struct channel_s {
    * distinct namespace. */
   uint64_t dirreq_id;
 
-  /** Channel counters for cell channels */
+  /** Channel counters for cells and bytes we have received. */
   uint64_t n_cells_recved, n_bytes_recved;
+  /** Channel counters for cells and bytes we have sent. */
   uint64_t n_cells_xmitted, n_bytes_xmitted;
 };
 
-struct channel_listener_s {
-  /* Current channel listener state */
+struct channel_listener_t {
+  /** Current channel listener state */
   channel_listener_state_t state;
 
-  /* Globally unique ID number for a channel over the lifetime of a Tor
+  /** Globally unique ID number for a channel over the lifetime of a Tor
    * process.
    */
   uint64_t global_identifier;
@@ -539,13 +539,8 @@ void channel_listener_set_listener_fn(channel_listener_t *chan,
 /* Incoming cell callbacks */
 channel_cell_handler_fn_ptr channel_get_cell_handler(channel_t *chan);
 
-channel_var_cell_handler_fn_ptr
-channel_get_var_cell_handler(channel_t *chan);
-
 void channel_set_cell_handlers(channel_t *chan,
-                               channel_cell_handler_fn_ptr cell_handler,
-                               channel_var_cell_handler_fn_ptr
-                                 var_cell_handler);
+                               channel_cell_handler_fn_ptr cell_handler);
 
 /* Clean up closed channels and channel listeners periodically; these are
  * called from run_scheduled_events() in main.c.
@@ -560,13 +555,13 @@ void channel_free_all(void);
 void channel_dumpstats(int severity);
 void channel_listener_dumpstats(int severity);
 
-#ifdef TOR_CHANNEL_INTERNAL_
+#ifdef CHANNEL_OBJECT_PRIVATE
 
-#ifdef CHANNEL_PRIVATE_
+#ifdef CHANNEL_FILE_PRIVATE
 
 STATIC void channel_add_to_digest_map(channel_t *chan);
 
-#endif /* defined(CHANNEL_PRIVATE_) */
+#endif /* defined(CHANNEL_FILE_PRIVATE) */
 
 /* Channel operations for subclasses and internal use only */
 
@@ -645,7 +640,7 @@ void channel_notify_flushed(channel_t *chan);
 /* Handle stuff we need to do on open like notifying circuits */
 void channel_do_open_actions(channel_t *chan);
 
-#endif /* defined(TOR_CHANNEL_INTERNAL_) */
+#endif /* defined(CHANNEL_OBJECT_PRIVATE) */
 
 /* Helper functions to perform operations on channels */
 
@@ -661,11 +656,13 @@ channel_t * channel_connect(const tor_addr_t *addr, uint16_t port,
                             const char *rsa_id_digest,
                             const struct ed25519_public_key_t *ed_id);
 
-channel_t * channel_get_for_extend(const char *rsa_id_digest,
+MOCK_DECL(channel_t *, channel_get_for_extend,(
+                                   const char *rsa_id_digest,
                                    const struct ed25519_public_key_t *ed_id,
-                                   const tor_addr_t *target_addr,
+                                   const tor_addr_t *target_ipv4_addr,
+                                   const tor_addr_t *target_ipv6_addr,
                                    const char **msg_out,
-                                   int *launch_out);
+                                   int *launch_out));
 
 /* Ask which of two channels is better for circuit-extension purposes */
 int channel_is_better(channel_t *a, channel_t *b);
@@ -726,7 +723,7 @@ const char * channel_get_actual_remote_descr(channel_t *chan);
 const char * channel_get_actual_remote_address(channel_t *chan);
 MOCK_DECL(int, channel_get_addr_if_possible, (channel_t *chan,
                                               tor_addr_t *addr_out));
-const char * channel_get_canonical_remote_descr(channel_t *chan);
+MOCK_DECL(const char *, channel_get_canonical_remote_descr,(channel_t *chan));
 int channel_has_queued_writes(channel_t *chan);
 int channel_is_bad_for_new_circs(channel_t *chan);
 void channel_mark_bad_for_new_circs(channel_t *chan);
@@ -741,8 +738,6 @@ int channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info);
 int channel_remote_identity_matches(const channel_t *chan,
                                     const char *rsa_id_digest,
                                     const ed25519_public_key_t *ed_id);
-int channel_matches_target_addr_for_extend(channel_t *chan,
-                                           const tor_addr_t *target);
 unsigned int channel_num_circuits(channel_t *chan);
 MOCK_DECL(void,channel_set_circid_type,(channel_t *chan,
                                         crypto_pk_t *identity_rcvd,
@@ -772,7 +767,7 @@ int packed_cell_is_destroy(channel_t *chan,
                            circid_t *circid_out);
 
 /* Declare the handle helpers */
-HANDLE_DECL(channel, channel_s,)
+HANDLE_DECL(channel, channel_t,)
 #define channel_handle_free(h)    \
   FREE_AND_NULL(channel_handle_t, channel_handle_free_, (h))
 #undef tor_timer_t
diff --git a/src/core/or/channelpadding.c b/src/core/or/channelpadding.c
index 4a0f0e00da..be2ce78a17 100644
--- a/src/core/or/channelpadding.c
+++ b/src/core/or/channelpadding.c
@@ -1,12 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-/* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of
+/**
+ * @file channelpadding.c
+ * @brief Link-level padding code.
+ **/
+
+/* CHANNEL_OBJECT_PRIVATE define needed for an O(1) implementation of
  * channelpadding_channel_to_channelinfo() */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 
 #include "core/or/or.h"
 #include "core/or/channel.h"
diff --git a/src/core/or/channelpadding.h b/src/core/or/channelpadding.h
index 48002eedb7..d1c7192ffd 100644
--- a/src/core/or/channelpadding.h
+++ b/src/core/or/channelpadding.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c
index 18025ff73a..f9eb67c399 100644
--- a/src/core/or/channeltls.c
+++ b/src/core/or/channeltls.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,7 +34,7 @@
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 
 #define CHANNELTLS_PRIVATE
 
@@ -45,8 +45,10 @@
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/command.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
+#include "feature/relay/relay_handshake.h"
 #include "feature/control/control.h"
 #include "feature/client/entrynodes.h"
 #include "trunnel/link_handshake.h"
@@ -59,6 +61,7 @@
 #include "feature/nodelist/torcert.h"
 #include "feature/nodelist/networkstatus.h"
 #include "trunnel/channelpadding_negotiation.h"
+#include "trunnel/netinfo.h"
 #include "core/or/channelpadding.h"
 
 #include "core/or/cell_st.h"
@@ -562,9 +565,7 @@ channel_tls_get_transport_name_method(channel_t *chan, char **transport_out)
 static const char *
 channel_tls_get_remote_descr_method(channel_t *chan, int flags)
 {
-#define MAX_DESCR_LEN 32
-
-  static char buf[MAX_DESCR_LEN + 1];
+  static char buf[TOR_ADDRPORT_BUF_LEN];
   channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan);
   connection_t *conn;
   const char *answer = NULL;
@@ -577,15 +578,14 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags)
     switch (flags) {
       case 0:
         /* Canonical address with port*/
-        tor_snprintf(buf, MAX_DESCR_LEN + 1,
+        tor_snprintf(buf, TOR_ADDRPORT_BUF_LEN,
                      "%s:%u", conn->address, conn->port);
         answer = buf;
         break;
       case GRD_FLAG_ORIGINAL:
         /* Actual address with port */
         addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr));
-        tor_snprintf(buf, MAX_DESCR_LEN + 1,
-                     "%s:%u", addr_str, conn->port);
+        tor_snprintf(buf, TOR_ADDRPORT_BUF_LEN, "%s:%u", addr_str, conn->port);
         tor_free(addr_str);
         answer = buf;
         break;
@@ -724,10 +724,13 @@ channel_tls_matches_target_method(channel_t *chan,
    * base_.addr is updated by connection_or_init_conn_from_address()
    * to be the address in the descriptor. It may be tempting to
    * allow either address to be allowed, but if we did so, it would
-   * enable someone who steals a relay's keys to impersonate/MITM it
+   * enable someone who steals a relay's keys to covertly impersonate/MITM it
    * from anywhere on the Internet! (Because they could make long-lived
    * TLS connections from anywhere to all relays, and wait for them to
    * be used for extends).
+   *
+   * An adversary who has stolen a relay's keys could also post a fake relay
+   * descriptor, but that attack is easier to detect.
    */
   return tor_addr_eq(&(tlschan->conn->real_addr), target);
 }
@@ -937,7 +940,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
 void
 channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                           or_connection_t *conn,
-                                          uint8_t old_state,
                                           uint8_t state)
 {
   channel_t *base_chan;
@@ -946,8 +948,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
   tor_assert(conn);
   tor_assert(conn->chan == chan);
   tor_assert(chan->conn == conn);
-  /* Shut the compiler up without triggering -Wtautological-compare */
-  (void)old_state;
 
   base_chan = TLS_CHAN_TO_BASE(chan);
 
@@ -1017,6 +1017,16 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
 }
 #endif /* defined(KEEP_TIMING_STATS) */
 
+#ifdef KEEP_TIMING_STATS
+#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
+    ++num ## tp;                                                \
+    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
+                             channel_tls_process_ ## tp ## _cell);  \
+    } STMT_END
+#else /* !defined(KEEP_TIMING_STATS) */
+#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
+#endif /* defined(KEEP_TIMING_STATS) */
+
 /**
  * Handle an incoming cell on a channel_tls_t.
  *
@@ -1036,16 +1046,6 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   channel_tls_t *chan;
   int handshaking;
 
-#ifdef KEEP_TIMING_STATS
-#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
-    ++num ## tp;                                                \
-    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
-                             channel_tls_process_ ## tp ## _cell);  \
-    } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
-#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
-#endif /* defined(KEEP_TIMING_STATS) */
-
   tor_assert(cell);
   tor_assert(conn);
 
@@ -1063,7 +1063,8 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
     return;
 
   /* Reject all but VERSIONS and NETINFO when handshaking. */
-  /* (VERSIONS should actually be impossible; it's variable-length.) */
+  /* (VERSIONS actually indicates a protocol warning: it's variable-length,
+   * so if it reaches this function, we're on a v1 connection.) */
   if (handshaking && cell->command != CELL_VERSIONS &&
       cell->command != CELL_NETINFO) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
@@ -1084,13 +1085,13 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   entry_guards_note_internet_connectivity(get_guard_selection_info());
   rep_hist_padding_count_read(PADDING_TYPE_TOTAL);
 
-  if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+  if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
     rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL);
 
   switch (cell->command) {
     case CELL_PADDING:
       rep_hist_padding_count_read(PADDING_TYPE_CELL);
-      if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+      if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
         rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL);
       ++stats_n_padding_cells_processed;
       /* do nothing */
@@ -1317,6 +1318,8 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
   }
 }
 
+#undef PROCESS_CELL
+
 /**
  * Update channel marks after connection_or.c has changed an address.
  *
@@ -1632,6 +1635,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan)
 }
 
 /**
+ * Convert <b>netinfo_addr</b> into corresponding <b>tor_addr</b>.
+ * Return 0 on success; on failure, return -1 and log a warning.
+ */
+static int
+tor_addr_from_netinfo_addr(tor_addr_t *tor_addr,
+                           const netinfo_addr_t *netinfo_addr) {
+  tor_assert(tor_addr);
+  tor_assert(netinfo_addr);
+
+  uint8_t type = netinfo_addr_get_addr_type(netinfo_addr);
+  uint8_t len = netinfo_addr_get_len(netinfo_addr);
+
+  if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4)  {
+    uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr);
+    tor_addr_from_ipv4h(tor_addr, ipv4);
+  } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) {
+    const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6(
+                                  netinfo_addr);
+    tor_addr_from_ipv6_bytes(tor_addr, ipv6_bytes);
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO "
+                                     "- wrong type/length.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/**
  * Helper: compute the absolute value of a time_t.
  *
  * (we need this because labs() doesn't always work for time_t, since
@@ -1655,8 +1687,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   time_t timestamp;
   uint8_t my_addr_type;
   uint8_t my_addr_len;
-  const uint8_t *my_addr_ptr;
-  const uint8_t *cp, *end;
   uint8_t n_other_addrs;
   time_t now = time(NULL);
   const routerinfo_t *me = router_get_my_routerinfo();
@@ -1704,7 +1734,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
         tor_assert(tor_digest_is_zero(
                   (const char*)(chan->conn->handshake_state->
                       authenticated_rsa_peer_id)));
-        tor_assert(tor_mem_is_zero(
+        tor_assert(fast_mem_is_zero(
                   (const char*)(chan->conn->handshake_state->
                                 authenticated_ed25519_peer_id.pubkey), 32));
         /* If the client never authenticated, it's a tor client or bridge
@@ -1727,38 +1757,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   }
 
   /* Decode the cell. */
-  timestamp = ntohl(get_uint32(cell->payload));
-  const time_t sent_versions_at =
-    chan->conn->handshake_state->sent_versions_at;
-  if (now > sent_versions_at && (now - sent_versions_at) < 180) {
-    /* If we have gotten the NETINFO cell reasonably soon after having
-     * sent our VERSIONS cell, maybe we can learn skew information from it. */
-    apparent_skew = now - timestamp;
+  netinfo_cell_t *netinfo_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload,
+                                      CELL_PAYLOAD_SIZE);
+
+  if (parsed < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+           "Failed to parse NETINFO cell - closing connection.");
+    connection_or_close_for_error(chan->conn, 0);
+    return;
   }
 
-  my_addr_type = (uint8_t) cell->payload[4];
-  my_addr_len = (uint8_t) cell->payload[5];
-  my_addr_ptr = (uint8_t*) cell->payload + 6;
-  end = cell->payload + CELL_PAYLOAD_SIZE;
-  cp = cell->payload + 6 + my_addr_len;
+  timestamp = netinfo_cell_get_timestamp(netinfo_cell);
 
+  const netinfo_addr_t *my_addr =
+    netinfo_cell_getconst_other_addr(netinfo_cell);
+
+  my_addr_type = netinfo_addr_get_addr_type(my_addr);
+  my_addr_len = netinfo_addr_get_len(my_addr);
+
+  if ((now - chan->conn->handshake_state->sent_versions_at) < 180) {
+    apparent_skew = now - timestamp;
+  }
   /* We used to check:
    *    if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) {
    *
    * This is actually never going to happen, since my_addr_len is at most 255,
    * and CELL_PAYLOAD_LEN - 6 is 503.  So we know that cp is < end. */
 
-  if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) {
-    tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr));
+  if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) {
+    connection_or_close_for_error(chan->conn, 0);
+    netinfo_cell_free(netinfo_cell);
+    return;
+  }
 
+  if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) {
     if (!get_options()->BridgeRelay && me &&
-        get_uint32(my_addr_ptr) == htonl(me->addr)) {
+        tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) {
       TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
     }
-
-  } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) {
-    tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr);
-
+  } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 &&
+             my_addr_len == 16) {
     if (!get_options()->BridgeRelay && me &&
         !tor_addr_is_null(&me->ipv6_addr) &&
         tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) {
@@ -1766,18 +1806,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
     }
   }
 
-  n_other_addrs = (uint8_t) *cp++;
-  while (n_other_addrs && cp < end-2) {
+  n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell);
+  for (uint8_t i = 0; i < n_other_addrs; i++) {
     /* Consider all the other addresses; if any matches, this connection is
      * "canonical." */
+
+    const netinfo_addr_t *netinfo_addr =
+      netinfo_cell_getconst_my_addrs(netinfo_cell, i);
+
     tor_addr_t addr;
-    const uint8_t *next =
-      decode_address_from_payload(&addr, cp, (int)(end-cp));
-    if (next == NULL) {
+
+    if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) {
       log_fn(LOG_PROTOCOL_WARN,  LD_OR,
-             "Bad address in netinfo cell; closing connection.");
-      connection_or_close_for_error(chan->conn, 0);
-      return;
+             "Bad address in netinfo cell; Skipping.");
+      continue;
     }
     /* A relay can connect from anywhere and be canonical, so
      * long as it tells you from where it came. This may sound a bit
@@ -1790,10 +1832,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       connection_or_set_canonical(chan->conn, 1);
       break;
     }
-    cp = next;
-    --n_other_addrs;
   }
 
+  netinfo_cell_free(netinfo_cell);
+
   if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer &&
       channel_is_canonical(TLS_CHAN_TO_BASE(chan))) {
     const char *descr =
diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h
index a2648ff537..f04ce0fa9c 100644
--- a/src/core/or/channeltls.h
+++ b/src/core/or/channeltls.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,16 +22,16 @@ struct curve25519_public_key_t;
 
 #define TLS_CHAN_MAGIC 0x8a192427U
 
-#ifdef TOR_CHANNEL_INTERNAL_
+#ifdef CHANNEL_OBJECT_PRIVATE
 
-struct channel_tls_s {
+struct channel_tls_t {
   /* Base channel_t struct */
   channel_t base_;
   /* or_connection_t pointer */
   or_connection_t *conn;
 };
 
-#endif /* defined(TOR_CHANNEL_INTERNAL_) */
+#endif /* defined(CHANNEL_OBJECT_PRIVATE) */
 
 channel_t * channel_tls_connect(const tor_addr_t *addr, uint16_t port,
                                 const char *id_digest,
@@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan);
 void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn);
 void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                                or_connection_t *conn,
-                                               uint8_t old_state,
                                                uint8_t state);
 void channel_tls_handle_var_cell(var_cell_t *var_cell,
                                  or_connection_t *conn);
diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h
index d4339ff50d..4baafb1848 100644
--- a/src/core/or/circuit_st.h
+++ b/src/core/or/circuit_st.h
@@ -1,17 +1,30 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file circuit_st.h
+ * @brief Base circuit structure.
+ **/
+
 #ifndef CIRCUIT_ST_H
 #define CIRCUIT_ST_H
 
 #include "core/or/or.h"
 
+#include "lib/container/handles.h"
+
 #include "core/or/cell_queue_st.h"
+#include "ext/ht.h"
 
 struct hs_token_t;
+struct circpad_machine_spec_t;
+struct circpad_machine_runtime_t;
+
+/** Number of padding state machines on a circuit. */
+#define CIRCPAD_MAX_MACHINES (2)
 
 /** "magic" value for an origin_circuit_t */
 #define ORIGIN_CIRCUIT_MAGIC 0x35315243u
@@ -49,6 +62,9 @@ struct circuit_t {
   uint32_t magic; /**< For memory and type debugging: must equal
                    * ORIGIN_CIRCUIT_MAGIC or OR_CIRCUIT_MAGIC. */
 
+  /** Handle entry for handle-based lookup */
+  HANDLE_ENTRY(circuit, circuit_t);
+
   /** The channel that is next in this circuit. */
   channel_t *n_chan;
 
@@ -61,12 +77,6 @@ struct circuit_t {
    */
   circid_t n_circ_id;
 
-  /**
-   * Circuit mux associated with n_chan to which this circuit is attached;
-   * NULL if we have no n_chan.
-   */
-  circuitmux_t *n_mux;
-
   /** Queue of cells waiting to be transmitted on n_chan */
   cell_queue_t n_chan_cells;
 
@@ -93,6 +103,10 @@ struct circuit_t {
   /** True iff this circuit has received a DESTROY cell in either direction */
   unsigned int received_destroy : 1;
 
+  /** True iff we have sent a sufficiently random data cell since last
+   * we reset send_randomness_after_n_cells. */
+  unsigned int have_sent_sufficiently_random_cell : 1;
+
   uint8_t state; /**< Current status of this circuit. */
   uint8_t purpose; /**< Why are we creating this circuit? */
 
@@ -105,6 +119,32 @@ struct circuit_t {
    * circuit-level sendme cells to indicate that we're willing to accept
    * more. */
   int deliver_window;
+  /**
+   * How many cells do we have until we need to send one that contains
+   * sufficient randomness?  Used to ensure that authenticated SENDME cells
+   * will reflect some unpredictable information.
+   **/
+  uint16_t send_randomness_after_n_cells;
+
+  /** FIFO containing the digest of the cells that are just before a SENDME is
+   * sent by the client. It is done at the last cell before our package_window
+   * goes down to 0 which is when we expect a SENDME.
+   *
+   * Our current circuit package window is capped to 1000
+   * (CIRCWINDOW_START_MAX) which is also the start value. The increment is
+   * set to 100 (CIRCWINDOW_INCREMENT) which means we don't allow more than
+   * 1000/100 = 10 outstanding SENDME cells worth of data. Meaning that this
+   * list can not contain more than 10 digests of DIGEST_LEN bytes (20).
+   *
+   * At position i in the list, the digest corresponds to the
+   * (CIRCWINDOW_INCREMENT * i)-nth cell received since we expect a SENDME to
+   * be received containing that cell digest.
+   *
+   * For example, position 2 (starting at 0) means that we've received 300
+   * cells so the 300th cell digest is kept at index 2.
+   *
+   * At maximum, this list contains 200 bytes plus the smartlist overhead. */
+  smartlist_t *sendme_last_digests;
 
   /** Temporary field used during circuits_handle_oom. */
   uint32_t age_tmp;
@@ -177,6 +217,27 @@ struct circuit_t {
   /** Hashtable node: used to look up the circuit by its HS token using the HS
       circuitmap. */
   HT_ENTRY(circuit_t) hs_circuitmap_node;
+
+  /** Adaptive Padding state machines: these are immutable. The state machines
+   *  that come from the consensus are saved to a global structure, to avoid
+   *  per-circuit allocations. This merely points to the global copy in
+   *  origin_padding_machines or relay_padding_machines that should never
+   *  change or get deallocated.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES];
+
+  /** Adaptive Padding machine runtime info for above machines. This is
+   *  the per-circuit mutable information, such as the current state and
+   *  histogram token counts. Some of it is optional (aka NULL).
+   *  If a machine is being shut down, these indexes can be NULL
+   *  without the corresponding padding_machine being NULL, while we
+   *  wait for the other end to respond to our shutdown request.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  struct circpad_machine_runtime_t *padding_info[CIRCPAD_MAX_MACHINES];
 };
 
-#endif
+#endif /* !defined(CIRCUIT_ST_H) */
diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c
index 70b5d8215a..ec61b4a455 100644
--- a/src/core/or/circuitbuild.c
+++ b/src/core/or/circuitbuild.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,20 +21,19 @@
  * cells arrive, the client will invoke circuit_send_next_onion_skin() to send
  * CREATE or RELAY_EXTEND cells.
  *
- * On the server side, this module also handles the logic of responding to
- * RELAY_EXTEND requests, using circuit_extend().
+ * The server side is handled in feature/relay/circuitbuild_relay.c.
  **/
 
 #define CIRCUITBUILD_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/crypto/hs_ntor.h"
 #include "core/crypto/onion_crypto.h"
 #include "core/crypto/onion_fast.h"
 #include "core/crypto/onion_tap.h"
-#include "core/crypto/relay_crypto.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
 #include "core/or/channel.h"
@@ -42,17 +41,20 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/command.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/onion.h"
+#include "core/or/ocirc_event.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
@@ -80,15 +82,6 @@
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
-static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
-                                            uint16_t port,
-                                            const char *id_digest,
-                                            const ed25519_public_key_t *ed_id);
-static int circuit_deliver_create_cell(circuit_t *circ,
-                                       const create_cell_t *create_cell,
-                                       int relayed);
-static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
-STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 static int circuit_send_first_onion_skin(origin_circuit_t *circ);
 static int circuit_build_no_more_hops(origin_circuit_t *circ);
 static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
@@ -102,10 +95,10 @@ static const node_t *choose_good_middle_server(uint8_t purpose,
  * and then calls command_setup_channel() to give it the right
  * callbacks.
  */
-static channel_t *
-channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest,
-                            const ed25519_public_key_t *ed_id)
+MOCK_IMPL(channel_t *,
+channel_connect_for_circuit,(const tor_addr_t *addr, uint16_t port,
+                             const char *id_digest,
+                             const struct ed25519_public_key_t *ed_id))
 {
   channel_t *chan;
 
@@ -492,7 +485,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
     return NULL;
   }
 
-  control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
+  circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
 
   if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
     circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
@@ -508,6 +501,27 @@ origin_circuit_get_guard_state(origin_circuit_t *circ)
   return circ->guard_state;
 }
 
+/**
+ * Helper function to publish a channel association message
+ *
+ * circuit_handle_first_hop() calls this to notify subscribers about a
+ * channel launch event, which associates a circuit with a channel.
+ * This doesn't always correspond to an assignment of the circuit's
+ * n_chan field, because that seems to be only for fully-open
+ * channels.
+ **/
+static void
+circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = circ->global_identifier;
+  msg->chan = chan->global_identifier;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_chan_publish(msg);
+}
+
 /** Start establishing the first hop of our circuit. Figure out what
  * OR we should connect to, and if necessary start the connection to
  * it. If we're already connected, then send the 'create' cell.
@@ -522,7 +536,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   int should_launch = 0;
   const or_options_t *options = get_options();
 
-  firsthop = onion_next_hop_in_cpath(circ->cpath);
+  firsthop = cpath_get_next_non_open_hop(circ->cpath);
   tor_assert(firsthop);
   tor_assert(firsthop->extend_info);
 
@@ -545,11 +559,17 @@ circuit_handle_first_hop(origin_circuit_t *circ)
             fmt_addrport(&firsthop->extend_info->addr,
                          firsthop->extend_info->port));
 
-  n_chan = channel_get_for_extend(firsthop->extend_info->identity_digest,
-                                  &firsthop->extend_info->ed_identity,
-                                  &firsthop->extend_info->addr,
-                                  &msg,
-                                  &should_launch);
+  /* We'll cleanup this code in #33220, when we add an IPv6 address to
+   * extend_info_t. */
+  const bool addr_is_ipv4 =
+    (tor_addr_family(&firsthop->extend_info->addr) == AF_INET);
+  n_chan = channel_get_for_extend(
+                          firsthop->extend_info->identity_digest,
+                          &firsthop->extend_info->ed_identity,
+                          addr_is_ipv4 ? &firsthop->extend_info->addr : NULL,
+                          addr_is_ipv4 ? NULL : &firsthop->extend_info->addr,
+                          &msg,
+                          &should_launch);
 
   if (!n_chan) {
     /* not currently connected in a useful way. */
@@ -559,8 +579,6 @@ circuit_handle_first_hop(origin_circuit_t *circ)
     circ->base_.n_hop = extend_info_dup(firsthop->extend_info);
 
     if (should_launch) {
-      if (circ->build_state->onehop_tunnel)
-        control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
       n_chan = channel_connect_for_circuit(
           &firsthop->extend_info->addr,
           firsthop->extend_info->port,
@@ -570,6 +588,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
       }
+      circuit_chan_publish(circ, n_chan);
     }
 
     log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
@@ -581,6 +600,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   } else { /* it's already open. use it. */
     tor_assert(!circ->base_.n_hop);
     circ->base_.n_chan = n_chan;
+    circuit_chan_publish(circ, n_chan);
     log_debug(LD_CIRC,"Conn open. Delivering first onion skin.");
     if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
       log_info(LD_CIRC,"circuit_send_next_onion_skin failed.");
@@ -700,9 +720,10 @@ circuit_n_chan_done(channel_t *chan, int status, int close_origin_circuits)
  * gave us via an EXTEND cell, so we shouldn't worry if we don't understand
  * it. Return -1 if we failed to find a suitable circid, else return 0.
  */
-static int
-circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell,
-                            int relayed)
+MOCK_IMPL(int,
+circuit_deliver_create_cell,(circuit_t *circ,
+                             const struct create_cell_t *create_cell,
+                             int relayed))
 {
   cell_t cell;
   circid_t id;
@@ -762,40 +783,6 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell,
   return -1;
 }
 
-/** We've decided to start our reachability testing. If all
- * is set, log this to the user. Return 1 if we did, or 0 if
- * we chose not to log anything. */
-int
-inform_testing_reachability(void)
-{
-  char dirbuf[128];
-  char *address;
-  const routerinfo_t *me = router_get_my_routerinfo();
-  if (!me)
-    return 0;
-  address = tor_dup_ip(me->addr);
-  control_event_server_status(LOG_NOTICE,
-                              "CHECKING_REACHABILITY ORADDRESS=%s:%d",
-                              address, me->or_port);
-  if (me->dir_port) {
-    tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d",
-                 address, me->dir_port);
-    control_event_server_status(LOG_NOTICE,
-                                "CHECKING_REACHABILITY DIRADDRESS=%s:%d",
-                                address, me->dir_port);
-  }
-  log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... "
-                         "(this may take up to %d minutes -- look for log "
-                         "messages indicating success)",
-      address, me->or_port,
-      me->dir_port ? dirbuf : "",
-      me->dir_port ? "are" : "is",
-      TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60);
-
-  tor_free(address);
-  return 1;
-}
-
 /** Return true iff we should send a create_fast cell to start building a given
  * circuit */
 static inline int
@@ -941,15 +928,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
   tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
   tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
 
-  crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath);
+  crypt_path_t *hop = cpath_get_next_non_open_hop(circ->cpath);
   circuit_build_times_handle_completed_hop(circ);
 
+  circpad_machine_event_circ_added_hop(circ);
+
   if (hop) {
     /* Case two: we're on a hop after the first. */
     return circuit_send_intermediate_onion_skin(circ, hop);
   }
 
   /* Case three: the circuit is finished. Do housekeeping tasks on it. */
+  circpad_machine_event_circ_built(circ);
   return circuit_build_no_more_hops(circ);
 }
 
@@ -1192,192 +1182,6 @@ circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle)
   }
 }
 
-/** Take the 'extend' <b>cell</b>, pull out addr/port plus the onion
- * skin and identity digest for the next hop. If we're already connected,
- * pass the onion skin to the next hop using a create cell; otherwise
- * launch a new OR connection, and <b>circ</b> will notice when the
- * connection succeeds or fails.
- *
- * Return -1 if we want to warn and tear down the circuit, else return 0.
- */
-int
-circuit_extend(cell_t *cell, circuit_t *circ)
-{
-  channel_t *n_chan;
-  relay_header_t rh;
-  extend_cell_t ec;
-  const char *msg = NULL;
-  int should_launch = 0;
-
-  if (circ->n_chan) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "n_chan already set. Bug/attack. Closing.");
-    return -1;
-  }
-  if (circ->n_hop) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "conn to next hop already launched. Bug/attack. Closing.");
-    return -1;
-  }
-
-  if (!server_mode(get_options())) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Got an extend cell, but running as a client. Closing.");
-    return -1;
-  }
-
-  relay_header_unpack(&rh, cell->payload);
-
-  if (extend_cell_parse(&ec, rh.command,
-                        cell->payload+RELAY_HEADER_SIZE,
-                        rh.length) < 0) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Can't parse extend cell. Closing circuit.");
-    return -1;
-  }
-
-  if (!ec.orport_ipv4.port || tor_addr_is_null(&ec.orport_ipv4.addr)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend to zero destination port or addr.");
-    return -1;
-  }
-
-  if (tor_addr_is_internal(&ec.orport_ipv4.addr, 0) &&
-      !get_options()->ExtendAllowPrivateAddresses) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend to a private address");
-    return -1;
-  }
-
-  /* Check if they asked us for 0000..0000. We support using
-   * an empty fingerprint for the first hop (e.g. for a bridge relay),
-   * but we don't want to let clients send us extend cells for empty
-   * fingerprints -- a) because it opens the user up to a mitm attack,
-   * and b) because it lets an attacker force the relay to hold open a
-   * new TLS connection for each extend request. */
-  if (tor_digest_is_zero((const char*)ec.node_id)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend without specifying an id_digest.");
-    return -1;
-  }
-
-  /* Fill in ed_pubkey if it was not provided and we can infer it from
-   * our networkstatus */
-  if (ed25519_public_key_is_zero(&ec.ed_pubkey)) {
-    const node_t *node = node_get_by_id((const char*)ec.node_id);
-    const ed25519_public_key_t *node_ed_id = NULL;
-    if (node &&
-        node_supports_ed25519_link_authentication(node, 1) &&
-        (node_ed_id = node_get_ed25519_id(node))) {
-      ed25519_pubkey_copy(&ec.ed_pubkey, node_ed_id);
-    }
-  }
-
-  /* Next, check if we're being asked to connect to the hop that the
-   * extend cell came from. There isn't any reason for that, and it can
-   * assist circular-path attacks. */
-  if (tor_memeq(ec.node_id,
-                TO_OR_CIRCUIT(circ)->p_chan->identity_digest,
-                DIGEST_LEN)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend back to the previous hop.");
-    return -1;
-  }
-
-  /* Check the previous hop Ed25519 ID too */
-  if (! ed25519_public_key_is_zero(&ec.ed_pubkey) &&
-      ed25519_pubkey_eq(&ec.ed_pubkey,
-                        &TO_OR_CIRCUIT(circ)->p_chan->ed25519_identity)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend back to the previous hop "
-           "(by Ed25519 ID).");
-    return -1;
-  }
-
-  n_chan = channel_get_for_extend((const char*)ec.node_id,
-                                  &ec.ed_pubkey,
-                                  &ec.orport_ipv4.addr,
-                                  &msg,
-                                  &should_launch);
-
-  if (!n_chan) {
-    log_debug(LD_CIRC|LD_OR,"Next router (%s): %s",
-              fmt_addrport(&ec.orport_ipv4.addr,ec.orport_ipv4.port),
-              msg?msg:"????");
-
-    circ->n_hop = extend_info_new(NULL /*nickname*/,
-                                  (const char*)ec.node_id,
-                                  &ec.ed_pubkey,
-                                  NULL, /*onion_key*/
-                                  NULL, /*curve25519_key*/
-                                  &ec.orport_ipv4.addr,
-                                  ec.orport_ipv4.port);
-
-    circ->n_chan_create_cell = tor_memdup(&ec.create_cell,
-                                          sizeof(ec.create_cell));
-
-    circuit_set_state(circ, CIRCUIT_STATE_CHAN_WAIT);
-
-    if (should_launch) {
-      /* we should try to open a connection */
-      n_chan = channel_connect_for_circuit(&ec.orport_ipv4.addr,
-                                           ec.orport_ipv4.port,
-                                           (const char*)ec.node_id,
-                                           &ec.ed_pubkey);
-      if (!n_chan) {
-        log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
-        circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
-        return 0;
-      }
-      log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
-    }
-    /* return success. The onion/circuit/etc will be taken care of
-     * automatically (may already have been) whenever n_chan reaches
-     * OR_CONN_STATE_OPEN.
-     */
-    return 0;
-  }
-
-  tor_assert(!circ->n_hop); /* Connection is already established. */
-  circ->n_chan = n_chan;
-  log_debug(LD_CIRC,
-            "n_chan is %s",
-            channel_get_canonical_remote_descr(n_chan));
-
-  if (circuit_deliver_create_cell(circ, &ec.create_cell, 1) < 0)
-    return -1;
-
-  return 0;
-}
-
-/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
- *
- * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
- * service circuits and <b>key_data</b> must be at least
- * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
- *
- * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
- * bytes, which are used as follows:
- *   - 20 to initialize f_digest
- *   - 20 to initialize b_digest
- *   - 16 to key f_crypto
- *   - 16 to key b_crypto
- *
- * (If 'reverse' is true, then f_XX and b_XX are swapped.)
- *
- * Return 0 if init was successful, else -1 if it failed.
- */
-int
-circuit_init_cpath_crypto(crypt_path_t *cpath,
-                          const char *key_data, size_t key_data_len,
-                          int reverse, int is_hs_v3)
-{
-
-  tor_assert(cpath);
-  return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse,
-                           is_hs_v3);
-}
-
 /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>.
  * (The body of <b>reply</b> varies depending on what sort of handshake
  * this is.)
@@ -1403,7 +1207,7 @@ circuit_finish_handshake(origin_circuit_t *circ,
   if (circ->cpath->state == CPATH_STATE_AWAITING_KEYS) {
     hop = circ->cpath;
   } else {
-    hop = onion_next_hop_in_cpath(circ->cpath);
+    hop = cpath_get_next_non_open_hop(circ->cpath);
     if (!hop) { /* got an extended when we're all done? */
       log_warn(LD_PROTOCOL,"got extended when circ already built? Closing.");
       return - END_CIRC_REASON_TORPROTOCOL;
@@ -1427,14 +1231,14 @@ circuit_finish_handshake(origin_circuit_t *circ,
 
   onion_handshake_state_release(&hop->handshake_state);
 
-  if (circuit_init_cpath_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
+  if (cpath_init_circuit_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
     return -END_CIRC_REASON_TORPROTOCOL;
   }
 
   hop->state = CPATH_STATE_OPEN;
   log_info(LD_CIRC,"Finished building circuit hop:");
   circuit_log_path(LOG_INFO,LD_CIRC,circ);
-  control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0);
+  circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0);
 
   return 0;
 }
@@ -1479,7 +1283,7 @@ circuit_truncated(origin_circuit_t *circ, int reason)
     }
 
     layer->next = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
   log_info(LD_CIRC, "finished");
@@ -1487,61 +1291,6 @@ circuit_truncated(origin_circuit_t *circ, int reason)
 #endif /* 0 */
 }
 
-/** Given a response payload and keys, initialize, then send a created
- * cell back.
- */
-int
-onionskin_answer(or_circuit_t *circ,
-                 const created_cell_t *created_cell,
-                 const char *keys, size_t keys_len,
-                 const uint8_t *rend_circ_nonce)
-{
-  cell_t cell;
-
-  tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN);
-
-  if (created_cell_format(&cell, created_cell) < 0) {
-    log_warn(LD_BUG,"couldn't format created cell (type=%d, len=%d)",
-             (int)created_cell->cell_type, (int)created_cell->handshake_len);
-    return -1;
-  }
-  cell.circ_id = circ->p_circ_id;
-
-  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
-
-  log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.",
-            (unsigned int)get_uint32(keys),
-            (unsigned int)get_uint32(keys+20));
-  if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) {
-    log_warn(LD_BUG,"Circuit initialization failed");
-    return -1;
-  }
-
-  memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN);
-
-  int used_create_fast = (created_cell->cell_type == CELL_CREATED_FAST);
-
-  append_cell_to_circuit_queue(TO_CIRCUIT(circ),
-                               circ->p_chan, &cell, CELL_DIRECTION_IN, 0);
-  log_debug(LD_CIRC,"Finished sending '%s' cell.",
-            used_create_fast ? "created_fast" : "created");
-
-  /* Ignore the local bit when ExtendAllowPrivateAddresses is set:
-   * it violates the assumption that private addresses are local.
-   * Also, many test networks run on local addresses, and
-   * TestingTorNetwork sets ExtendAllowPrivateAddresses. */
-  if ((!channel_is_local(circ->p_chan)
-       || get_options()->ExtendAllowPrivateAddresses)
-      && !channel_is_outgoing(circ->p_chan)) {
-    /* record that we could process create cells from a non-local conn
-     * that we didn't initiate; presumably this means that create cells
-     * can reach us too. */
-    router_orport_found_reachable();
-  }
-
-  return 0;
-}
-
 /** Helper for new_route_len().  Choose a circuit length for purpose
  * <b>purpose</b>: DEFAULT_ROUTE_LEN (+ 1 if someone else chose the
  * exit).  If someone else chose the exit, they could be colluding
@@ -1673,24 +1422,28 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei)
  * to handle the desired path length, return -1.
  */
 STATIC int
-new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
+new_route_len(uint8_t purpose, extend_info_t *exit_ei,
+              const smartlist_t *nodes)
 {
-  int num_acceptable_routers;
   int routelen;
 
   tor_assert(nodes);
 
   routelen = route_len_for_purpose(purpose, exit_ei);
 
-  num_acceptable_routers = count_acceptable_nodes(nodes);
+  int num_acceptable_direct = count_acceptable_nodes(nodes, 1);
+  int num_acceptable_indirect = count_acceptable_nodes(nodes, 0);
 
-  log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).",
-            routelen, num_acceptable_routers, smartlist_len(nodes));
+  log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect "
+             "routers suitable).", routelen, num_acceptable_direct,
+             num_acceptable_indirect);
 
-  if (num_acceptable_routers < routelen) {
+  if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) {
     log_info(LD_CIRC,
-             "Not enough acceptable routers (%d/%d). Discarding this circuit.",
-             num_acceptable_routers, routelen);
+             "Not enough acceptable routers (%d/%d direct and %d/%d "
+             "indirect routers suitable). Discarding this circuit.",
+             num_acceptable_direct, routelen,
+             num_acceptable_indirect, routelen);
     return -1;
   }
 
@@ -2294,7 +2047,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
   state->chosen_exit = extend_info_dup(exit_ei);
 
   ++circ->build_state->desired_path_len;
-  onion_append_hop(&circ->cpath, exit_ei);
+  cpath_append_hop(&circ->cpath, exit_ei);
   return 0;
 }
 
@@ -2332,7 +2085,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
  * particular router. See bug #25885.)
  */
 MOCK_IMPL(STATIC int,
-count_acceptable_nodes, (smartlist_t *nodes))
+count_acceptable_nodes, (const smartlist_t *nodes, int direct))
 {
   int num=0;
 
@@ -2346,7 +2099,7 @@ count_acceptable_nodes, (smartlist_t *nodes))
     if (! node->is_valid)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i);
       continue;
-    if (! node_has_any_descriptor(node))
+    if (! node_has_preferred_descriptor(node, direct))
       continue;
     /* The node has a descriptor, so we can just check the ntor key directly */
     if (!node_has_curve25519_onion_key(node))
@@ -2359,47 +2112,6 @@ count_acceptable_nodes, (smartlist_t *nodes))
   return num;
 }
 
-/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
- * This function is used to extend cpath by another hop.
- */
-void
-onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop)
-{
-  if (*head_ptr) {
-    new_hop->next = (*head_ptr);
-    new_hop->prev = (*head_ptr)->prev;
-    (*head_ptr)->prev->next = new_hop;
-    (*head_ptr)->prev = new_hop;
-  } else {
-    *head_ptr = new_hop;
-    new_hop->prev = new_hop->next = new_hop;
-  }
-}
-
-#ifdef TOR_UNIT_TESTS
-
-/** Unittest helper function: Count number of hops in cpath linked list. */
-unsigned int
-cpath_get_n_hops(crypt_path_t **head_ptr)
-{
-  unsigned int n_hops = 0;
-  crypt_path_t *tmp;
-
-  if (!*head_ptr) {
-    return 0;
-  }
-
-  tmp = *head_ptr;
-  do {
-    n_hops++;
-    tmp = tmp->next;
-  } while (tmp != *head_ptr);
-
-  return n_hops;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 /**
  * Build the exclude list for vanguard circuits.
  *
@@ -2597,7 +2309,24 @@ choose_good_middle_server(uint8_t purpose,
     return choice;
   }
 
-  choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  if (options->MiddleNodes) {
+    smartlist_t *sl = smartlist_new();
+    routerset_get_all_nodes(sl, options->MiddleNodes,
+                            options->ExcludeNodes, 1);
+
+    smartlist_subtract(sl, excluded);
+
+    choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID);
+    smartlist_free(sl);
+    if (choice) {
+      log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s",
+          hex_str(choice->identity, DIGEST_LEN));
+    } else {
+      log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available");
+    }
+  } else {
+    choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  }
   smartlist_free(excluded);
   return choice;
 }
@@ -2657,20 +2386,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
   return choice;
 }
 
-/** Return the first non-open hop in cpath, or return NULL if all
- * hops are open. */
-static crypt_path_t *
-onion_next_hop_in_cpath(crypt_path_t *cpath)
-{
-  crypt_path_t *hop = cpath;
-  do {
-    if (hop->state != CPATH_STATE_OPEN)
-      return hop;
-    hop = hop->next;
-  } while (hop != cpath);
-  return NULL;
-}
-
 /** Choose a suitable next hop for the circuit <b>circ</b>.
  * Append the hop info to circ->cpath.
  *
@@ -2727,33 +2442,11 @@ onion_extend_cpath(origin_circuit_t *circ)
             extend_info_describe(info),
             cur_len+1, build_state_get_exit_nickname(state));
 
-  onion_append_hop(&circ->cpath, info);
+  cpath_append_hop(&circ->cpath, info);
   extend_info_free(info);
   return 0;
 }
 
-/** Create a new hop, annotate it with information about its
- * corresponding router <b>choice</b>, and append it to the
- * end of the cpath <b>head_ptr</b>. */
-STATIC int
-onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
-{
-  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
-
-  /* link hop into the cpath, at the end. */
-  onion_append_to_cpath(head_ptr, hop);
-
-  hop->magic = CRYPT_PATH_MAGIC;
-  hop->state = CPATH_STATE_CLOSED;
-
-  hop->extend_info = extend_info_dup(choice);
-
-  hop->package_window = circuit_initial_package_window();
-  hop->deliver_window = CIRCWINDOW_START;
-
-  return 0;
-}
-
 /** Allocate a new extend_info object based on the various arguments. */
 extend_info_t *
 extend_info_new(const char *nickname,
@@ -2895,8 +2588,8 @@ extend_info_dup(extend_info_t *info)
  * If there is no chosen exit, or if we don't know the node_t for
  * the chosen exit, return NULL.
  */
-const node_t *
-build_state_get_exit_node(cpath_build_state_t *state)
+MOCK_IMPL(const node_t *,
+build_state_get_exit_node,(cpath_build_state_t *state))
 {
   if (!state || !state->chosen_exit)
     return NULL;
@@ -2958,7 +2651,7 @@ extend_info_supports_ntor(const extend_info_t* ei)
 {
   tor_assert(ei);
   /* Valid ntor keys have at least one non-zero byte */
-  return !tor_mem_is_zero(
+  return !fast_mem_is_zero(
                           (const char*)ei->curve25519_onion_key.public_key,
                           CURVE25519_PUBKEY_LEN);
 }
diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h
index 969f5b5cc3..e62bb41de9 100644
--- a/src/core/or/circuitbuild.h
+++ b/src/core/or/circuitbuild.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,29 +29,19 @@ struct circuit_guard_state_t *origin_circuit_get_guard_state(
 int circuit_handle_first_hop(origin_circuit_t *circ);
 void circuit_n_chan_done(channel_t *chan, int status,
                          int close_origin_circuits);
-int inform_testing_reachability(void);
 int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);
 void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle);
-int circuit_extend(cell_t *cell, circuit_t *circ);
-int circuit_init_cpath_crypto(crypt_path_t *cpath,
-                              const char *key_data, size_t key_data_len,
-                              int reverse, int is_hs_v3);
 struct created_cell_t;
 int circuit_finish_handshake(origin_circuit_t *circ,
                              const struct created_cell_t *created_cell);
 int circuit_truncated(origin_circuit_t *circ, int reason);
-int onionskin_answer(or_circuit_t *circ,
-                     const struct created_cell_t *created_cell,
-                     const char *keys, size_t keys_len,
-                     const uint8_t *rend_circ_nonce);
 MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now,
                                                      int *need_uptime,
                                                      int *need_capacity));
 
 int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info);
 int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info);
-void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop);
 extend_info_t *extend_info_new(const char *nickname,
                                const char *rsa_id_digest,
                                const struct ed25519_public_key_t *ed_id,
@@ -70,7 +60,8 @@ int circuit_can_use_tap(const origin_circuit_t *circ);
 int circuit_has_usable_onion_key(const origin_circuit_t *circ);
 int extend_info_has_preferred_onion_key(const extend_info_t* ei);
 const uint8_t *build_state_get_exit_rsa_id(cpath_build_state_t *state);
-const node_t *build_state_get_exit_node(cpath_build_state_t *state);
+MOCK_DECL(const node_t *,
+          build_state_get_exit_node,(cpath_build_state_t *state));
 const char *build_state_get_exit_nickname(cpath_build_state_t *state);
 
 struct circuit_guard_state_t;
@@ -80,11 +71,26 @@ const node_t *choose_good_entry_server(uint8_t purpose,
                            struct circuit_guard_state_t **guard_state_out);
 void circuit_upgrade_circuits_from_guard_wait(void);
 
+struct ed25519_public_key_t;
+
+MOCK_DECL(channel_t *,
+channel_connect_for_circuit,(const tor_addr_t *addr,
+                             uint16_t port,
+                             const char *id_digest,
+                             const struct ed25519_public_key_t *ed_id));
+
+struct create_cell_t;
+MOCK_DECL(int,
+circuit_deliver_create_cell,(circuit_t *circ,
+                             const struct create_cell_t *create_cell,
+                             int relayed));
+
 #ifdef CIRCUITBUILD_PRIVATE
 STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
 STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
-                         smartlist_t *nodes);
-MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
+                         const smartlist_t *nodes);
+MOCK_DECL(STATIC int, count_acceptable_nodes, (const smartlist_t *nodes,
+                                               int direct));
 
 STATIC int onion_extend_cpath(origin_circuit_t *circ);
 
@@ -92,11 +98,6 @@ STATIC int
 onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
                       int is_hs_v3_rp_circuit);
 
-#if defined(TOR_UNIT_TESTS)
-unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 #endif /* defined(CIRCUITBUILD_PRIVATE) */
 
 #endif /* !defined(TOR_CIRCUITBUILD_H) */
diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c
index ccf3041bb4..384835667d 100644
--- a/src/core/or/circuitlist.c
+++ b/src/core/or/circuitlist.c
@@ -1,7 +1,7 @@
 /* Copyright 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -51,6 +51,7 @@
  * logic, which was originally circuit-focused.
  **/
 #define CIRCUITLIST_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 #include "lib/cc/torint.h"  /* TOR_PRIuSZ */
 
 #include "core/or/or.h"
@@ -61,11 +62,13 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/circuitstats.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/crypt_path.h"
 #include "core/mainloop/connection.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/crypt_ops/crypto_dh.h"
@@ -94,7 +97,9 @@
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
+
+#include "core/or/ocirc_event.h"
 
 #include "ht.h"
 
@@ -127,7 +132,6 @@ static smartlist_t *circuits_pending_other_guards = NULL;
  * circuit_mark_for_close and which are waiting for circuit_about_to_free. */
 static smartlist_t *circuits_pending_close = NULL;
 
-static void circuit_free_cpath_node(crypt_path_t *victim);
 static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
 static void circuit_about_to_free_atexit(circuit_t *circ);
 static void circuit_about_to_free(circuit_t *circ);
@@ -142,6 +146,9 @@ static int any_opened_circs_cached_val = 0;
 
 /********* END VARIABLES ************/
 
+/* Implement circuit handle helpers. */
+HANDLE_IMPL(circuit, circuit_t,)
+
 or_circuit_t *
 TO_OR_CIRCUIT(circuit_t *x)
 {
@@ -208,10 +215,10 @@ chan_circid_entry_hash_(chan_circid_circuit_map_t *a)
 static HT_HEAD(chan_circid_map, chan_circid_circuit_map_t)
      chan_circid_map = HT_INITIALIZER();
 HT_PROTOTYPE(chan_circid_map, chan_circid_circuit_map_t, node,
-             chan_circid_entry_hash_, chan_circid_entries_eq_)
+             chan_circid_entry_hash_, chan_circid_entries_eq_);
 HT_GENERATE2(chan_circid_map, chan_circid_circuit_map_t, node,
              chan_circid_entry_hash_, chan_circid_entries_eq_, 0.6,
-             tor_reallocarray_, tor_free_)
+             tor_reallocarray_, tor_free_);
 
 /** The most recently returned entry from circuit_get_by_circid_chan;
  * used to improve performance when many cells arrive in a row from the
@@ -481,6 +488,54 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
   }
 }
 
+/**
+ * Helper function to publish a message about events on an origin circuit
+ *
+ * Publishes a message to subscribers of origin circuit events, and
+ * sends the control event.
+ **/
+int
+circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                     int reason_code)
+{
+  ocirc_cevent_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  tor_assert(circ);
+
+  msg->gid = circ->global_identifier;
+  msg->evtype = tp;
+  msg->reason = reason_code;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_cevent_publish(msg);
+  return control_event_circuit_status(circ, tp, reason_code);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * circuit_set_state() calls this to notify subscribers about a change
+ * of the state of an origin circuit.  @a circ must be an origin
+ * circuit.
+ **/
+static void
+circuit_state_publish(const circuit_t *circ)
+{
+  ocirc_state_msg_t *msg = tor_malloc(sizeof(*msg));
+  const origin_circuit_t *ocirc;
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  ocirc = CONST_TO_ORIGIN_CIRCUIT(circ);
+  /* Only inbound OR circuits can be in this state, not origin circuits. */
+  tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING);
+
+  msg->gid = ocirc->global_identifier;
+  msg->state = circ->state;
+  msg->onehop = ocirc->build_state->onehop_tunnel;
+
+  ocirc_state_publish(msg);
+}
+
 /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing
  * it from lists as appropriate. */
 void
@@ -510,6 +565,8 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN)
     tor_assert(!circ->n_chan_create_cell);
   circ->state = state;
+  if (CIRCUIT_IS_ORIGIN(circ))
+    circuit_state_publish(circ);
 }
 
 /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for
@@ -767,6 +824,8 @@ circuit_purpose_to_controller_string(uint8_t purpose)
       return "PATH_BIAS_TESTING";
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "HS_VANGUARDS";
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "CIRCUIT_PADDING";
 
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
@@ -796,6 +855,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_CONTROLLER:
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
       return NULL;
 
     case CIRCUIT_PURPOSE_INTRO_POINT:
@@ -896,6 +956,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "Hidden service: Pre-built vanguard circuit";
 
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "Circuit kept open for padding";
+
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
       return buf;
@@ -931,6 +994,7 @@ init_circuit_base(circuit_t *circ)
 
   circ->package_window = circuit_initial_package_window();
   circ->deliver_window = CIRCWINDOW_START;
+  circuit_reset_sendme_randomness(circ);
   cell_queue_init(&circ->n_chan_cells);
 
   smartlist_add(circuit_get_global_list(), circ);
@@ -1072,7 +1136,7 @@ circuit_free_(circuit_t *circ)
    * circuit is closed. This is to avoid any code path that free registered
    * circuits without closing them before. This needs to be done before the
    * hs identifier is freed. */
-  hs_circ_cleanup(circ);
+  hs_circ_cleanup_on_free(circ);
 
   if (CIRCUIT_IS_ORIGIN(circ)) {
     origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
@@ -1092,7 +1156,7 @@ circuit_free_(circuit_t *circ)
 
     if (ocirc->build_state) {
         extend_info_free(ocirc->build_state->chosen_exit);
-        circuit_free_cpath_node(ocirc->build_state->pending_final_cpath);
+        cpath_free(ocirc->build_state->pending_final_cpath);
         cpath_ref_decref(ocirc->build_state->service_pending_final_cpath_ref);
     }
     tor_free(ocirc->build_state);
@@ -1171,11 +1235,23 @@ circuit_free_(circuit_t *circ)
    * "active" checks will be violated. */
   cell_queue_clear(&circ->n_chan_cells);
 
+  /* Cleanup possible SENDME state. */
+  if (circ->sendme_last_digests) {
+    SMARTLIST_FOREACH(circ->sendme_last_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(circ->sendme_last_digests);
+  }
+
   log_info(LD_CIRC, "Circuit %u (id: %" PRIu32 ") has been freed.",
            n_circ_id,
            CIRCUIT_IS_ORIGIN(circ) ?
               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
 
+  /* Free any circuit padding structures */
+  circpad_circuit_free_all_machineinfos(circ);
+
+  /* Clear all dangling handle references. */
+  circuit_handles_clear(circ);
+
   if (should_free) {
     memwipe(mem, 0xAA, memlen); /* poison memory */
     tor_free(mem);
@@ -1207,10 +1283,10 @@ circuit_clear_cpath(origin_circuit_t *circ)
   while (cpath->next && cpath->next != head) {
     victim = cpath;
     cpath = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
-  circuit_free_cpath_node(cpath);
+  cpath_free(cpath);
 
   circ->cpath = NULL;
 }
@@ -1267,29 +1343,13 @@ circuit_free_all(void)
   HT_CLEAR(chan_circid_map, &chan_circid_map);
 }
 
-/** Deallocate space associated with the cpath node <b>victim</b>. */
-static void
-circuit_free_cpath_node(crypt_path_t *victim)
-{
-  if (!victim)
-    return;
-
-  relay_crypto_clear(&victim->crypto);
-  onion_handshake_state_release(&victim->handshake_state);
-  crypto_dh_free(victim->rend_dh_handshake_state);
-  extend_info_free(victim->extend_info);
-
-  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
-  tor_free(victim);
-}
-
 /** Release a crypt_path_reference_t*, which may be NULL. */
 static void
 cpath_ref_decref(crypt_path_reference_t *cpath_ref)
 {
   if (cpath_ref != NULL) {
     if (--(cpath_ref->refcount) == 0) {
-      circuit_free_cpath_node(cpath_ref->cpath);
+      cpath_free(cpath_ref->cpath);
       tor_free(cpath_ref);
     }
   }
@@ -2140,6 +2200,11 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
   tor_assert(line);
   tor_assert(file);
 
+  /* Check whether the circuitpadding subsystem wants to block this close */
+  if (circpad_marked_circuit_for_padding(circ, reason)) {
+    return;
+  }
+
   if (circ->marked_for_close) {
     log_warn(LD_BUG,
         "Duplicate call to circuit_mark_for_close at %s:%d"
@@ -2195,7 +2260,7 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
   }
 
   /* Notify the HS subsystem that this circuit is closing. */
-  hs_circ_cleanup(circ);
+  hs_circ_cleanup_on_close(circ);
 
   if (circuits_pending_close == NULL)
     circuits_pending_close = smartlist_new();
@@ -2270,50 +2335,13 @@ circuit_about_to_free(circuit_t *circ)
     smartlist_remove(circuits_pending_other_guards, circ);
   }
   if (CIRCUIT_IS_ORIGIN(circ)) {
-    control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
+    circuit_event_status(TO_ORIGIN_CIRCUIT(circ),
      (circ->state == CIRCUIT_STATE_OPEN ||
       circ->state == CIRCUIT_STATE_GUARD_WAIT) ?
                                  CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
      orig_reason);
   }
 
-  if (circ->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    int timed_out = (reason == END_CIRC_REASON_TIMEOUT);
-    tor_assert(circ->state == CIRCUIT_STATE_OPEN);
-    tor_assert(ocirc->build_state->chosen_exit);
-    if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
-        ocirc->rend_data) {
-      /* treat this like getting a nack from it */
-      log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s",
-          safe_str_client(rend_data_get_address(ocirc->rend_data)),
-          safe_str_client(build_state_get_exit_nickname(ocirc->build_state)),
-          timed_out ? "Recording timeout." : "Removing from descriptor.");
-      rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
-                                             ocirc->rend_data,
-                                             timed_out ?
-                                             INTRO_POINT_FAILURE_TIMEOUT :
-                                             INTRO_POINT_FAILURE_GENERIC);
-    }
-  } else if (circ->purpose == CIRCUIT_PURPOSE_C_INTRODUCING &&
-             reason != END_CIRC_REASON_TIMEOUT) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    if (ocirc->build_state->chosen_exit && ocirc->rend_data) {
-      if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
-          ocirc->rend_data) {
-        log_info(LD_REND, "Failed intro circ %s to %s "
-            "(building circuit to intro point). "
-            "Marking intro point as possibly unreachable.",
-            safe_str_client(rend_data_get_address(ocirc->rend_data)),
-            safe_str_client(build_state_get_exit_nickname(
-                                              ocirc->build_state)));
-        rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
-                                              ocirc->rend_data,
-                                              INTRO_POINT_FAILURE_UNREACHABLE);
-      }
-    }
-  }
-
   if (circ->n_chan) {
     circuit_clear_cell_queue(circ, circ->n_chan);
     /* Only send destroy if the channel isn't closing anyway */
@@ -2374,13 +2402,9 @@ marked_circuit_free_cells(circuit_t *circ)
     return;
   }
   cell_queue_clear(&circ->n_chan_cells);
-  if (circ->n_mux)
-    circuitmux_clear_num_cells(circ->n_mux, circ);
   if (! CIRCUIT_IS_ORIGIN(circ)) {
     or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
     cell_queue_clear(&orcirc->p_chan_cells);
-    if (orcirc->p_mux)
-      circuitmux_clear_num_cells(orcirc->p_mux, circ);
   }
 }
 
@@ -2724,59 +2748,6 @@ circuits_handle_oom(size_t current_allocation)
              n_dirconns_killed);
 }
 
-/** Verify that cpath layer <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-void
-assert_cpath_layer_ok(const crypt_path_t *cp)
-{
-//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
-//  tor_assert(cp->port);
-  tor_assert(cp);
-  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
-  switch (cp->state)
-    {
-    case CPATH_STATE_OPEN:
-      relay_crypto_assert_ok(&cp->crypto);
-      FALLTHROUGH;
-    case CPATH_STATE_CLOSED:
-      /*XXXX Assert that there's no handshake_state either. */
-      tor_assert(!cp->rend_dh_handshake_state);
-      break;
-    case CPATH_STATE_AWAITING_KEYS:
-      /* tor_assert(cp->dh_handshake_state); */
-      break;
-    default:
-      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
-      tor_assert(0);
-    }
-  tor_assert(cp->package_window >= 0);
-  tor_assert(cp->deliver_window >= 0);
-}
-
-/** Verify that cpath <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-static void
-assert_cpath_ok(const crypt_path_t *cp)
-{
-  const crypt_path_t *start = cp;
-
-  do {
-    assert_cpath_layer_ok(cp);
-    /* layers must be in sequence of: "open* awaiting? closed*" */
-    if (cp != start) {
-      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      } else if (cp->state == CPATH_STATE_OPEN) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      }
-    }
-    cp = cp->next;
-    tor_assert(cp);
-  } while (cp != start);
-}
-
 /** Verify that circuit <b>c</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */
@@ -2838,7 +2809,7 @@ assert_circuit_ok,(const circuit_t *c))
                !smartlist_contains(circuits_pending_chans, c));
   }
   if (origin_circ && origin_circ->cpath) {
-    assert_cpath_ok(origin_circ->cpath);
+    cpath_assert_ok(origin_circ->cpath);
   }
   if (c->purpose == CIRCUIT_PURPOSE_REND_ESTABLISHED) {
     tor_assert(or_circ);
diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h
index b87c6a3667..fd7e22e4c0 100644
--- a/src/core/or/circuitlist.h
+++ b/src/core/or/circuitlist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,8 +12,10 @@
 #ifndef TOR_CIRCUITLIST_H
 #define TOR_CIRCUITLIST_H
 
+#include "lib/container/handles.h"
 #include "lib/testsupport/testsupport.h"
 #include "feature/hs/hs_ident.h"
+#include "core/or/ocirc_event.h"
 
 /** Circuit state: I'm the origin, still haven't done all my handshakes. */
 #define CIRCUIT_STATE_BUILDING 0
@@ -91,31 +93,33 @@
 #define CIRCUIT_PURPOSE_C_HS_MAX_ 13
 /** This circuit is used for build time measurement only */
 #define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 14
-#define CIRCUIT_PURPOSE_C_MAX_ 14
+/** This circuit is being held open by circuit padding */
+#define CIRCUIT_PURPOSE_C_CIRCUIT_PADDING 15
+#define CIRCUIT_PURPOSE_C_MAX_ 15
 
-#define CIRCUIT_PURPOSE_S_HS_MIN_ 15
+#define CIRCUIT_PURPOSE_S_HS_MIN_ 16
 /** Hidden-service-side circuit purpose: at the service, waiting for
  * introductions. */
-#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 15
+#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 16
 /** Hidden-service-side circuit purpose: at the service, successfully
  * established intro. */
-#define CIRCUIT_PURPOSE_S_INTRO 16
+#define CIRCUIT_PURPOSE_S_INTRO 17
 /** Hidden-service-side circuit purpose: at the service, connecting to rend
  * point. */
-#define CIRCUIT_PURPOSE_S_CONNECT_REND 17
+#define CIRCUIT_PURPOSE_S_CONNECT_REND 18
 /** Hidden-service-side circuit purpose: at the service, rendezvous
  * established. */
-#define CIRCUIT_PURPOSE_S_REND_JOINED 18
+#define CIRCUIT_PURPOSE_S_REND_JOINED 19
 /** This circuit is used for uploading hsdirs */
-#define CIRCUIT_PURPOSE_S_HSDIR_POST 19
-#define CIRCUIT_PURPOSE_S_HS_MAX_ 19
+#define CIRCUIT_PURPOSE_S_HSDIR_POST 20
+#define CIRCUIT_PURPOSE_S_HS_MAX_ 20
 
 /** A testing circuit; not meant to be used for actual traffic. */
-#define CIRCUIT_PURPOSE_TESTING 20
+#define CIRCUIT_PURPOSE_TESTING 21
 /** A controller made this circuit and Tor should not use it. */
-#define CIRCUIT_PURPOSE_CONTROLLER 21
+#define CIRCUIT_PURPOSE_CONTROLLER 22
 /** This circuit is used for path bias probing only */
-#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 22
+#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 23
 
 /** This circuit is used for vanguards/restricted paths.
  *
@@ -123,9 +127,9 @@
  *  on-demand. When an HS operation needs to take place (e.g. connect to an
  *  intro point), these circuits are then cannibalized and repurposed to the
  *  actual needed HS purpose. */
-#define CIRCUIT_PURPOSE_HS_VANGUARDS 23
+#define CIRCUIT_PURPOSE_HS_VANGUARDS 24
 
-#define CIRCUIT_PURPOSE_MAX_ 23
+#define CIRCUIT_PURPOSE_MAX_ 24
 /** A catch-all for unrecognized purposes. Currently we don't expect
  * to make or see any circuits with this purpose. */
 #define CIRCUIT_PURPOSE_UNKNOWN 255
@@ -184,6 +188,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id);
 void channel_mark_circid_usable(channel_t *chan, circid_t id);
 time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id,
                                                   channel_t *chan);
+int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                         int reason_code);
 void circuit_set_state(circuit_t *circ, uint8_t state);
 void circuit_close_all_marked(void);
 int32_t circuit_initial_package_window(void);
@@ -213,7 +219,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void);
 void circuit_synchronize_written_or_bandwidth(const circuit_t *c,
                                               circuit_channel_direction_t dir);
 MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason,
-                                          int line, const char *file));
+                                          int line, const char *cfile));
 int circuit_get_cpath_len(origin_circuit_t *circ);
 int circuit_get_cpath_opened_len(const origin_circuit_t *);
 void circuit_clear_cpath(origin_circuit_t *circ);
@@ -225,7 +231,6 @@ int circuit_count_pending_on_channel(channel_t *chan);
 #define circuit_mark_for_close(c, reason)                               \
   circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__)
 
-void assert_cpath_layer_ok(const crypt_path_t *cp);
 MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c));
 void circuit_free_all(void);
 void circuits_handle_oom(size_t current_allocation);
@@ -238,6 +243,11 @@ MOCK_DECL(void, channel_note_destroy_not_pending,
 
 smartlist_t *circuit_find_circuits_to_upgrade_from_guard_wait(void);
 
+/* Declare the handle helpers */
+HANDLE_DECL(circuit, circuit_t, )
+#define circuit_handle_free(h)    \
+    FREE_AND_NULL(circuit_handle_t, circuit_handle_free_, (h))
+
 #ifdef CIRCUITLIST_PRIVATE
 STATIC void circuit_free_(circuit_t *circ);
 #define circuit_free(circ) FREE_AND_NULL(circuit_t, circuit_free_, (circ))
diff --git a/src/core/or/circuitmux.c b/src/core/or/circuitmux.c
index e7309553c4..be54ae6ec6 100644
--- a/src/core/or/circuitmux.c
+++ b/src/core/or/circuitmux.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -69,25 +69,21 @@
  *     made to attach all existing circuits to the new policy.
  **/
 
+#define CIRCUITMUX_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitmux.h"
 #include "core/or/relay.h"
 
-#include "core/or/cell_queue_st.h"
-#include "core/or/destroy_cell_queue_st.h"
 #include "core/or/or_circuit_st.h"
 
-/*
- * Private typedefs for circuitmux.c
- */
+#include "lib/crypt_ops/crypto_util.h"
 
 /*
- * Map of muxinfos for circuitmux_t to use; struct is defined below (name
- * of struct must match HT_HEAD line).
+ * Private typedefs for circuitmux.c
  */
-typedef struct chanid_circid_muxinfo_map chanid_circid_muxinfo_map_t;
 
 /*
  * Hash table entry (yeah, calling it chanid_circid_muxinfo_s seems to
@@ -100,57 +96,14 @@ typedef struct chanid_circid_muxinfo_t chanid_circid_muxinfo_t;
  * a count of queued cells.
  */
 
-typedef struct circuit_muxinfo_s circuit_muxinfo_t;
-
-/*
- * Structures for circuitmux.c
- */
-
-struct circuitmux_s {
-  /* Keep count of attached, active circuits */
-  unsigned int n_circuits, n_active_circuits;
-
-  /* Total number of queued cells on all circuits */
-  unsigned int n_cells;
-
-  /*
-   * Map from (channel ID, circuit ID) pairs to circuit_muxinfo_t
-   */
-  chanid_circid_muxinfo_map_t *chanid_circid_map;
-
-  /** List of queued destroy cells */
-  destroy_cell_queue_t destroy_cell_queue;
-  /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit
-   * returned the destroy queue. Used to force alternation between
-   * destroy/non-destroy cells.
-   *
-   * XXXX There is no reason to think that alternating is a particularly good
-   * approach -- it's just designed to prevent destroys from starving other
-   * cells completely.
-   */
-  unsigned int last_cell_was_destroy : 1;
-  /** Destroy counter: increment this when a destroy gets queued, decrement
-   * when we unqueue it, so we can test to make sure they don't starve.
-   */
-  int64_t destroy_ctr;
-
-  /*
-   * Circuitmux policy; if this is non-NULL, it can override the built-
-   * in round-robin active circuits behavior.  This is how EWMA works in
-   * the new circuitmux_t world.
-   */
-  const circuitmux_policy_t *policy;
-
-  /* Policy-specific data */
-  circuitmux_policy_data_t *policy_data;
-};
+typedef struct circuit_muxinfo_t circuit_muxinfo_t;
 
 /*
  * This struct holds whatever we want to store per attached circuit on a
  * circuitmux_t; right now, just the count of queued cells and the direction.
  */
 
-struct circuit_muxinfo_s {
+struct circuit_muxinfo_t {
   /* Count of cells on this circuit at last update */
   unsigned int cell_count;
   /* Direction of flow */
@@ -222,15 +175,12 @@ chanid_circid_entry_hash(chanid_circid_muxinfo_t *a)
   return (unsigned) siphash24g(data, sizeof(data));
 }
 
-/* Declare the struct chanid_circid_muxinfo_map type */
-HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t);
-
 /* Emit a bunch of hash table stuff */
 HT_PROTOTYPE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node,
-             chanid_circid_entry_hash, chanid_circid_entries_eq)
+             chanid_circid_entry_hash, chanid_circid_entries_eq);
 HT_GENERATE2(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node,
              chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6,
-             tor_reallocarray_, tor_free_)
+             tor_reallocarray_, tor_free_);
 
 /*
  * Circuitmux alloc/free functions
@@ -295,9 +245,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /* Clear n_mux */
-            circ->n_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else if (circ->magic == OR_CIRCUIT_MAGIC) {
@@ -310,12 +257,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /*
-             * It has a sensible p_chan and direction == CELL_DIRECTION_IN,
-             * so clear p_mux.
-             */
-            TO_OR_CIRCUIT(circ)->p_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else {
@@ -837,18 +778,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      */
     log_info(LD_CIRC,
              "Circuit %u on channel %"PRIu64 " was already attached to "
-             "cmux %p (trying to attach to %p)",
+             "(trying to attach to %p)",
              (unsigned)circ_id, (channel_id),
-             ((direction == CELL_DIRECTION_OUT) ?
-                circ->n_mux : TO_OR_CIRCUIT(circ)->p_mux),
              cmux);
 
     /*
      * The mux pointer on this circuit and the direction in result should
      * match; otherwise assert.
      */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == cmux);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
     tor_assert(hashent->muxinfo.direction == direction);
 
     /*
@@ -873,13 +810,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
              "Attaching circuit %u on channel %"PRIu64 " to cmux %p",
               (unsigned)circ_id, (channel_id), cmux);
 
-    /*
-     * Assert that the circuit doesn't already have a mux for this
-     * direction.
-     */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == NULL);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == NULL);
-
     /* Insert it in the map */
     hashent = tor_malloc_zero(sizeof(*hashent));
     hashent->chan_id = channel_id;
@@ -903,10 +833,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     HT_INSERT(chanid_circid_muxinfo_map, cmux->chanid_circid_map,
               hashent);
 
-    /* Set the circuit's mux for this direction */
-    if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux;
-    else TO_OR_CIRCUIT(circ)->p_mux = cmux;
-
     /* Update counters */
     ++(cmux->n_circuits);
     if (cell_count > 0) {
@@ -994,14 +920,14 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
 
     /* Consistency check: the direction must match the direction searched */
     tor_assert(last_searched_direction == hashent->muxinfo.direction);
-    /* Clear the circuit's mux for this direction */
-    if (last_searched_direction == CELL_DIRECTION_OUT) circ->n_mux = NULL;
-    else TO_OR_CIRCUIT(circ)->p_mux = NULL;
 
     /* Now remove it from the map */
     HT_REMOVE(chanid_circid_muxinfo_map, cmux->chanid_circid_map, hashent);
 
-    /* Free the hash entry */
+    /* Wipe and free the hash entry */
+    // This isn't sensitive, but we want to be sure to know if we're accessing
+    // this accidentally.
+    memwipe(hashent, 0xef, sizeof(*hashent));
     tor_free(hashent);
   }
 }
diff --git a/src/core/or/circuitmux.h b/src/core/or/circuitmux.h
index 67cd9bcdd8..191ca12e30 100644
--- a/src/core/or/circuitmux.h
+++ b/src/core/or/circuitmux.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,11 +12,11 @@
 #include "core/or/or.h"
 #include "lib/testsupport/testsupport.h"
 
-typedef struct circuitmux_policy_s circuitmux_policy_t;
-typedef struct circuitmux_policy_data_s circuitmux_policy_data_t;
-typedef struct circuitmux_policy_circ_data_s circuitmux_policy_circ_data_t;
+typedef struct circuitmux_policy_t circuitmux_policy_t;
+typedef struct circuitmux_policy_data_t circuitmux_policy_data_t;
+typedef struct circuitmux_policy_circ_data_t circuitmux_policy_circ_data_t;
 
-struct circuitmux_policy_s {
+struct circuitmux_policy_t {
   /* Allocate cmux-wide policy-specific data */
   circuitmux_policy_data_t * (*alloc_cmux_data)(circuitmux_t *cmux);
   /* Free cmux-wide policy-specific data */
@@ -67,7 +67,7 @@ struct circuitmux_policy_s {
  * wide data; it just has the magic number in the base struct.
  */
 
-struct circuitmux_policy_data_s {
+struct circuitmux_policy_data_t {
   uint32_t magic;
 };
 
@@ -76,7 +76,7 @@ struct circuitmux_policy_data_s {
  * specific data; it just has the magic number in the base struct.
  */
 
-struct circuitmux_policy_circ_data_s {
+struct circuitmux_policy_circ_data_t {
   uint32_t magic;
 };
 
@@ -158,5 +158,61 @@ void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux,
 MOCK_DECL(int, circuitmux_compare_muxes,
           (circuitmux_t *cmux_1, circuitmux_t *cmux_2));
 
+#ifdef CIRCUITMUX_PRIVATE
+
+#include "core/or/destroy_cell_queue_st.h"
+
+/*
+ * Map of muxinfos for circuitmux_t to use; struct is defined below (name
+ * of struct must match HT_HEAD line).
+ */
+typedef HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t)
+  chanid_circid_muxinfo_map_t;
+
+/*
+ * Structures for circuitmux.c
+ */
+
+struct circuitmux_t {
+  /* Keep count of attached, active circuits */
+  unsigned int n_circuits, n_active_circuits;
+
+  /* Total number of queued cells on all circuits */
+  unsigned int n_cells;
+
+  /*
+   * Map from (channel ID, circuit ID) pairs to circuit_muxinfo_t
+   */
+  chanid_circid_muxinfo_map_t *chanid_circid_map;
+
+  /** List of queued destroy cells */
+  destroy_cell_queue_t destroy_cell_queue;
+  /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit
+   * returned the destroy queue. Used to force alternation between
+   * destroy/non-destroy cells.
+   *
+   * XXXX There is no reason to think that alternating is a particularly good
+   * approach -- it's just designed to prevent destroys from starving other
+   * cells completely.
+   */
+  unsigned int last_cell_was_destroy : 1;
+  /** Destroy counter: increment this when a destroy gets queued, decrement
+   * when we unqueue it, so we can test to make sure they don't starve.
+   */
+  int64_t destroy_ctr;
+
+  /*
+   * Circuitmux policy; if this is non-NULL, it can override the built-
+   * in round-robin active circuits behavior.  This is how EWMA works in
+   * the new circuitmux_t world.
+   */
+  const circuitmux_policy_t *policy;
+
+  /* Policy-specific data */
+  circuitmux_policy_data_t *policy_data;
+};
+
+#endif /* defined(CIRCUITMUX_PRIVATE) */
+
 #endif /* !defined(TOR_CIRCUITMUX_H) */
 
diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c
index 3f83c3fd5a..0dcd22e8a7 100644
--- a/src/core/or/circuitmux_ewma.c
+++ b/src/core/or/circuitmux_ewma.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,6 +38,7 @@
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
 #include "app/config/or_options_st.h"
 
@@ -58,115 +59,6 @@
 /** The natural logarithm of 0.5. */
 #define LOG_ONEHALF -0.69314718055994529
 
-/*** EWMA structures ***/
-
-typedef struct cell_ewma_s cell_ewma_t;
-typedef struct ewma_policy_data_s ewma_policy_data_t;
-typedef struct ewma_policy_circ_data_s ewma_policy_circ_data_t;
-
-/**
- * The cell_ewma_t structure keeps track of how many cells a circuit has
- * transferred recently.  It keeps an EWMA (exponentially weighted moving
- * average) of the number of cells flushed from the circuit queue onto a
- * connection in channel_flush_from_first_active_circuit().
- */
-
-struct cell_ewma_s {
-  /** The last 'tick' at which we recalibrated cell_count.
-   *
-   * A cell sent at exactly the start of this tick has weight 1.0. Cells sent
-   * since the start of this tick have weight greater than 1.0; ones sent
-   * earlier have less weight. */
-  unsigned int last_adjusted_tick;
-  /** The EWMA of the cell count. */
-  double cell_count;
-  /** True iff this is the cell count for a circuit's previous
-   * channel. */
-  unsigned int is_for_p_chan : 1;
-  /** The position of the circuit within the OR connection's priority
-   * queue. */
-  int heap_index;
-};
-
-struct ewma_policy_data_s {
-  circuitmux_policy_data_t base_;
-
-  /**
-   * Priority queue of cell_ewma_t for circuits with queued cells waiting
-   * for room to free up on the channel that owns this circuitmux.  Kept
-   * in heap order according to EWMA.  This was formerly in channel_t, and
-   * in or_connection_t before that.
-   */
-  smartlist_t *active_circuit_pqueue;
-
-  /**
-   * The tick on which the cell_ewma_ts in active_circuit_pqueue last had
-   * their ewma values rescaled.  This was formerly in channel_t, and in
-   * or_connection_t before that.
-   */
-  unsigned int active_circuit_pqueue_last_recalibrated;
-};
-
-struct ewma_policy_circ_data_s {
-  circuitmux_policy_circ_data_t base_;
-
-  /**
-   * The EWMA count for the number of cells flushed from this circuit
-   * onto this circuitmux.  Used to determine which circuit to flush
-   * from next.  This was formerly in circuit_t and or_circuit_t.
-   */
-  cell_ewma_t cell_ewma;
-
-  /**
-   * Pointer back to the circuit_t this is for; since we're separating
-   * out circuit selection policy like this, we can't attach cell_ewma_t
-   * to the circuit_t any more, so we can't use SUBTYPE_P directly to a
-   * circuit_t like before; instead get it here.
-   */
-  circuit_t *circ;
-};
-
-#define EWMA_POL_DATA_MAGIC 0x2fd8b16aU
-#define EWMA_POL_CIRC_DATA_MAGIC 0x761e7747U
-
-/*** Downcasts for the above types ***/
-
-static ewma_policy_data_t *
-TO_EWMA_POL_DATA(circuitmux_policy_data_t *);
-
-static ewma_policy_circ_data_t *
-TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *);
-
-/**
- * Downcast a circuitmux_policy_data_t to an ewma_policy_data_t and assert
- * if the cast is impossible.
- */
-
-static inline ewma_policy_data_t *
-TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol)
-{
-  if (!pol) return NULL;
-  else {
-    tor_assert(pol->magic == EWMA_POL_DATA_MAGIC);
-    return DOWNCAST(ewma_policy_data_t, pol);
-  }
-}
-
-/**
- * Downcast a circuitmux_policy_circ_data_t to an ewma_policy_circ_data_t
- * and assert if the cast is impossible.
- */
-
-static inline ewma_policy_circ_data_t *
-TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
-{
-  if (!pol) return NULL;
-  else {
-    tor_assert(pol->magic == EWMA_POL_CIRC_DATA_MAGIC);
-    return DOWNCAST(ewma_policy_circ_data_t, pol);
-  }
-}
-
 /*** Static declarations for circuitmux_ewma.c ***/
 
 static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma);
@@ -295,6 +187,7 @@ ewma_free_cmux_data(circuitmux_t *cmux,
   pol = TO_EWMA_POL_DATA(pol_data);
 
   smartlist_free(pol->active_circuit_pqueue);
+  memwipe(pol, 0xda, sizeof(ewma_policy_data_t));
   tor_free(pol);
 }
 
@@ -361,7 +254,7 @@ ewma_free_circ_data(circuitmux_t *cmux,
   if (!pol_circ_data) return;
 
   cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
-
+  memwipe(cdata, 0xdc, sizeof(ewma_policy_circ_data_t));
   tor_free(cdata);
 }
 
@@ -530,7 +423,7 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
       /* Pick whichever one has the better best circuit */
       return compare_cell_ewma_counts(ce1, ce2);
     } else {
-      if (ce1 != NULL ) {
+      if (ce1 != NULL) {
         /* We only have a circuit on cmux_1, so prefer it */
         return -1;
       } else if (ce2 != NULL) {
@@ -716,7 +609,7 @@ cmux_ewma_set_options(const or_options_t *options,
   /* convert halflife into halflife-per-tick. */
   halflife /= EWMA_TICK_LEN;
   /* compute per-tick scale factor. */
-  ewma_scale_factor = exp( LOG_ONEHALF / halflife );
+  ewma_scale_factor = exp(LOG_ONEHALF / halflife);
   log_info(LD_OR,
            "Enabled cell_ewma algorithm because of value in %s; "
            "scale factor is %f per %d seconds",
diff --git a/src/core/or/circuitmux_ewma.h b/src/core/or/circuitmux_ewma.h
index b45ce1f916..e41cf9e0f0 100644
--- a/src/core/or/circuitmux_ewma.h
+++ b/src/core/or/circuitmux_ewma.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,9 +22,117 @@ void cmux_ewma_set_options(const or_options_t *options,
 void circuitmux_ewma_free_all(void);
 
 #ifdef CIRCUITMUX_EWMA_PRIVATE
+
+/*** EWMA structures ***/
+
+typedef struct cell_ewma_t cell_ewma_t;
+typedef struct ewma_policy_data_t ewma_policy_data_t;
+typedef struct ewma_policy_circ_data_t ewma_policy_circ_data_t;
+
+/**
+ * The cell_ewma_t structure keeps track of how many cells a circuit has
+ * transferred recently.  It keeps an EWMA (exponentially weighted moving
+ * average) of the number of cells flushed from the circuit queue onto a
+ * connection in channel_flush_from_first_active_circuit().
+ */
+
+struct cell_ewma_t {
+  /** The last 'tick' at which we recalibrated cell_count.
+   *
+   * A cell sent at exactly the start of this tick has weight 1.0. Cells sent
+   * since the start of this tick have weight greater than 1.0; ones sent
+   * earlier have less weight. */
+  unsigned int last_adjusted_tick;
+  /** The EWMA of the cell count. */
+  double cell_count;
+  /** True iff this is the cell count for a circuit's previous
+   * channel. */
+  unsigned int is_for_p_chan : 1;
+  /** The position of the circuit within the OR connection's priority
+   * queue. */
+  int heap_index;
+};
+
+struct ewma_policy_data_t {
+  circuitmux_policy_data_t base_;
+
+  /**
+   * Priority queue of cell_ewma_t for circuits with queued cells waiting
+   * for room to free up on the channel that owns this circuitmux.  Kept
+   * in heap order according to EWMA.  This was formerly in channel_t, and
+   * in or_connection_t before that.
+   */
+  smartlist_t *active_circuit_pqueue;
+
+  /**
+   * The tick on which the cell_ewma_ts in active_circuit_pqueue last had
+   * their ewma values rescaled.  This was formerly in channel_t, and in
+   * or_connection_t before that.
+   */
+  unsigned int active_circuit_pqueue_last_recalibrated;
+};
+
+struct ewma_policy_circ_data_t {
+  circuitmux_policy_circ_data_t base_;
+
+  /**
+   * The EWMA count for the number of cells flushed from this circuit
+   * onto this circuitmux.  Used to determine which circuit to flush
+   * from next.  This was formerly in circuit_t and or_circuit_t.
+   */
+  cell_ewma_t cell_ewma;
+
+  /**
+   * Pointer back to the circuit_t this is for; since we're separating
+   * out circuit selection policy like this, we can't attach cell_ewma_t
+   * to the circuit_t any more, so we can't use SUBTYPE_P directly to a
+   * circuit_t like before; instead get it here.
+   */
+  circuit_t *circ;
+};
+
+#define EWMA_POL_DATA_MAGIC 0x2fd8b16aU
+#define EWMA_POL_CIRC_DATA_MAGIC 0x761e7747U
+
+/*** Downcasts for the above types ***/
+
+/**
+ * Downcast a circuitmux_policy_data_t to an ewma_policy_data_t and assert
+ * if the cast is impossible.
+ */
+
+static inline ewma_policy_data_t *
+TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol)
+{
+  if (!pol) return NULL;
+  else {
+    tor_assertf(pol->magic == EWMA_POL_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_DATA_MAGIC);
+    return DOWNCAST(ewma_policy_data_t, pol);
+  }
+}
+
+/**
+ * Downcast a circuitmux_policy_circ_data_t to an ewma_policy_circ_data_t
+ * and assert if the cast is impossible.
+ */
+
+static inline ewma_policy_circ_data_t *
+TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
+{
+  if (!pol) return NULL;
+  else {
+    tor_assertf(pol->magic == EWMA_POL_CIRC_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_CIRC_DATA_MAGIC);
+    return DOWNCAST(ewma_policy_circ_data_t, pol);
+  }
+}
+
 STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out);
 STATIC void cell_ewma_initialize_ticks(void);
-#endif
 
-#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */
+#endif /* defined(CIRCUITMUX_EWMA_PRIVATE) */
 
+#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */
diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c
new file mode 100644
index 0000000000..43f4a31624
--- /dev/null
+++ b/src/core/or/circuitpadding.c
@@ -0,0 +1,3101 @@
+/* Copyright (c) 2017 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.c
+ * \brief Circuit-level padding implementation
+ *
+ * \details
+ *
+ * This file implements Tor proposal 254 "Padding Negotiation" which is heavily
+ * inspired by the paper "Toward an Efficient Website Fingerprinting Defense"
+ * by M. Juarez, M. Imani, M. Perry, C. Diaz, M. Wright.
+ *
+ * In particular the code in this file describes mechanisms for clients to
+ * negotiate various types of circuit-level padding from relays.
+ *
+ * Each padding type is described by a state machine (circpad_machine_spec_t),
+ * which is also referred as a "padding machine" in this file.  Currently,
+ * these state machines are hardcoded in the source code (e.g. see
+ * circpad_machines_init()), but in the future we will be able to
+ * serialize them in the torrc or the consensus.
+ *
+ * As specified by prop#254, clients can negotiate padding with relays by using
+ * PADDING_NEGOTIATE cells. After successful padding negotiation, padding
+ * machines are assigned to the circuit in their mutable form as a
+ * circpad_machine_runtime_t.
+ *
+ * Each state of a padding state machine can be either:
+ * - A histogram that specifies inter-arrival padding delays.
+ * - Or a parametrized probability distribution that specifies inter-arrival
+ *   delays (see circpad_distribution_type_t).
+ *
+ * Padding machines start from the START state and finish with the END
+ * state. They can transition between states using the events in
+ * circpad_event_t.
+ *
+ * When a padding machine reaches the END state, it gets wiped from the circuit
+ * so that other padding machines can take over if needed (see
+ * circpad_machine_spec_transitioned_to_end()).
+ *
+ ****************************
+ * General notes:
+ *
+ * All used machines should be heap allocated and placed into
+ * origin_padding_machines/relay_padding_machines so that they get correctly
+ * cleaned up by the circpad_free_all() function.
+ **/
+
+#define CIRCUITPADDING_PRIVATE
+
+#include <math.h>
+#include "lib/math/fp.h"
+#include "lib/math/prob_distr.h"
+#include "core/or/or.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/relay.h"
+#include "feature/stats/rephist.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/channel.h"
+
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/crypto/relay_crypto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "app/config/config.h"
+
+static inline circpad_circuit_state_t circpad_circuit_state(
+                                        origin_circuit_t *circ);
+static void circpad_setup_machine_on_circ(circuit_t *on_circ,
+                                        const circpad_machine_spec_t *machine);
+static double circpad_distribution_sample(circpad_distribution_t dist);
+
+static inline void circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi);
+
+/** Cached consensus params */
+static uint8_t circpad_padding_disabled;
+static uint8_t circpad_padding_reduced;
+static uint8_t circpad_global_max_padding_percent;
+static uint16_t circpad_global_allowed_cells;
+static uint16_t circpad_max_circ_queued_cells;
+
+/** Global cell counts, for rate limiting */
+static uint64_t circpad_global_padding_sent;
+static uint64_t circpad_global_nonpadding_sent;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 1 (ie: are for client side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *origin_padding_machines = NULL;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 0 (ie: are for relay side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *relay_padding_machines = NULL;
+
+#ifndef COCCI
+/** Loop over the current padding state machines using <b>loop_var</b> as the
+ *  loop variable. */
+#define FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                         \
+  STMT_BEGIN                                                             \
+  for (int loop_var = 0; loop_var < CIRCPAD_MAX_MACHINES; loop_var++) {
+#define FOR_EACH_CIRCUIT_MACHINE_END } STMT_END ;
+
+/** Loop over the current active padding state machines using <b>loop_var</b>
+ *  as the loop variable. If a machine is not active, skip it. */
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(loop_var, circ)            \
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                               \
+  if (!(circ)->padding_info[loop_var])                           \
+    continue;
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ;
+#endif /* !defined(COCCI) */
+
+/**
+ * Free the machineinfo at an index
+ */
+static void
+circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx)
+{
+  if (circ->padding_info[idx]) {
+    log_fn(LOG_INFO,LD_CIRC, "Freeing padding info idx %d on circuit %u (%d)",
+           idx, CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+           circ->purpose);
+
+    tor_free(circ->padding_info[idx]->histogram);
+    timer_free(circ->padding_info[idx]->padding_timer);
+    tor_free(circ->padding_info[idx]);
+  }
+}
+
+/**
+ * Return true if circpad has decided to hold the circuit open for additional
+ * padding. This function is used to take and retain ownership of certain
+ * types of circuits that have padding machines on them, that have been passed
+ * to circuit_mark_for_close().
+ *
+ * circuit_mark_for_close() calls this function to ask circpad if any padding
+ * machines want to keep the circuit open longer to pad.
+ *
+ * Any non-measurement circuit that was closed for a normal, non-error reason
+ * code may be held open for up to CIRCPAD_DELAY_INFINITE microseconds between
+ * network-driven cell events.
+ *
+ * After CIRCPAD_DELAY_INFINITE microseconds of silence on a circuit, this
+ * function will no longer hold it open (it will return 0 regardless of
+ * what the machines ask for, and thus circuit_expire_old_circuits_clientside()
+ * will close the circuit after roughly 1.25hr of idle time, maximum,
+ * regardless of the padding machine state.
+ */
+int
+circpad_marked_circuit_for_padding(circuit_t *circ, int reason)
+{
+  /* If the circuit purpose is measurement or path bias, don't
+   * hold it open */
+  if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING ||
+      circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+    return 0;
+  }
+
+  /* If the circuit is closed for any reason other than these three valid,
+   * client-side close reasons, do not try to keep it open. It is probably
+   * damaged or unusable. Note this is OK with vanguards because
+   * controller-closed circuits have REASON=REQUESTED, so vanguards-closed
+   * circuits will not be held open (we want them to close ASAP). */
+  if (!(reason == END_CIRC_REASON_NONE ||
+        reason == END_CIRC_REASON_FINISHED ||
+        reason == END_CIRC_REASON_IP_NOW_REDUNDANT)) {
+    return 0;
+  }
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    circpad_machine_runtime_t *mi = circ->padding_info[i];
+    if (!mi) {
+      continue; // No padding runtime info; check next machine
+    }
+
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+
+    /* If we're in END state (NULL here), then check next machine */
+    if (!state) {
+      continue; // check next machine
+    }
+
+    /* If the machine does not want to control the circuit close itself, then
+     * check the next machine */
+    if (!circ->padding_machine[i]->manage_circ_lifetime) {
+      continue; // check next machine
+    }
+
+    /* If the machine has reached the END state, we can close. Check next
+     * machine. */
+    if (mi->current_state == CIRCPAD_STATE_END) {
+      continue; // check next machine
+    }
+
+    log_info(LD_CIRC, "Circuit %d is not marked for close because of a "
+             "pending padding machine in index %d.",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0, i);
+
+    /* If the machine has had no network events at all within the
+     * last circpad_delay_t timespan, it's in some deadlock state.
+     * Tell circuit_mark_for_close() that we don't own it anymore.
+     * This will allow circuit_expire_old_circuits_clientside() to
+     * close it.
+     */
+    if (circ->padding_info[i]->last_cell_time_sec +
+        (time_t)CIRCPAD_DELAY_MAX_SECS < approx_time()) {
+      log_notice(LD_BUG, "Circuit %d was not marked for close because of a "
+               "pending padding machine in index %d for over an hour. "
+               "Circuit is a %s",
+               CIRCUIT_IS_ORIGIN(circ) ?
+               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+               i, circuit_purpose_to_string(circ->purpose));
+
+      return 0; // abort timer reached; mark the circuit for close now
+    }
+
+    /* If we weren't marked dirty yet, let's pretend we're dirty now.
+     * ("Dirty" means that a circuit has been used for application traffic
+     * by Tor.. Dirty circuits have different expiry times, and are not
+     * considered in counts of built circuits, etc. By claiming that we're
+     * dirty, the rest of Tor will make decisions as if we were actually
+     * used by application data.
+     *
+     * This is most important for circuit_expire_old_circuits_clientside(),
+     * where we want that function to expire us after the padding machine
+     * has shut down, but using the MaxCircuitDirtiness timer instead of
+     * the idle circuit timer (again, we want this because we're not
+     * supposed to look idle to Guard nodes that can see our lifespan). */
+    if (!circ->timestamp_dirty)
+      circ->timestamp_dirty = approx_time();
+
+    /* Take ownership of the circuit */
+    circuit_change_purpose(circ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+    return 1;
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+
+  return 0; // No machine wanted to keep the circuit open; mark for close
+}
+
+/**
+ * Free all the machineinfos in <b>circ</b> that match <b>machine_num</b>.
+ *
+ * Returns true if any machineinfos with that number were freed.
+ * False otherwise. */
+static int
+free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num)
+{
+  int found = 0;
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    if (circ->padding_machine[i] &&
+        circ->padding_machine[i]->machine_num == machine_num) {
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      circ->padding_machine[i] = NULL;
+      found = 1;
+    }
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return found;
+}
+
+/**
+ * Free all padding machines and mutable info associated with circuit
+ */
+void
+circpad_circuit_free_all_machineinfos(circuit_t *circ)
+{
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    circpad_circuit_machineinfo_free_idx(circ, i);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Allocate a new mutable machineinfo structure.
+ */
+STATIC circpad_machine_runtime_t *
+circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index)
+{
+  circpad_machine_runtime_t *mi =
+    tor_malloc_zero(sizeof(circpad_machine_runtime_t));
+  mi->machine_index = machine_index;
+  mi->on_circ = on_circ;
+  mi->last_cell_time_sec = approx_time();
+
+  return mi;
+}
+
+/**
+ * Return the circpad_state_t for the current state based on the
+ * mutable info.
+ *
+ * This function returns NULL when the machine is in the end state or in an
+ * invalid state.
+ */
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    return NULL;
+  } else if (BUG(mi->current_state >= machine->num_states)) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Invalid circuit padding state %d",
+           mi->current_state);
+
+    return NULL;
+  }
+
+  return &machine->states[mi->current_state];
+}
+
+/**
+ * Get the lower bound of a histogram bin.
+ *
+ * You can obtain the upper bound using histogram_get_bin_upper_bound().
+ *
+ * This function can also be called with 'bin' set to a value equal or greater
+ * than histogram_len in which case the infinity bin is chosen and
+ * CIRCPAD_DELAY_INFINITE is returned.
+ */
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* The infinity bin has an upper bound of infinity, so make sure we return
+   * that if they ask for it. */
+  if (bin > CIRCPAD_INFINITY_BIN(state)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  return state->histogram_edges[bin] + rtt_add_usec;
+}
+
+/**
+ * Like circpad_histogram_bin_to_usec() but return the upper bound of bin.
+ * (The upper bound is included in the bin.)
+ */
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  return circpad_histogram_bin_to_usec(mi, bin+1) - 1;
+}
+
+/** Return the midpoint of the histogram bin <b>bin_index</b>. */
+static circpad_delay_t
+circpad_get_histogram_bin_midpoint(const circpad_machine_runtime_t *mi,
+                           int bin_index)
+{
+  circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index);
+  circpad_delay_t right_bound = histogram_get_bin_upper_bound(mi, bin_index);
+
+  return left_bound + (right_bound - left_bound)/2;
+}
+
+/**
+ * Return the bin that contains the usec argument.
+ * "Contains" is defined as us in [lower, upper).
+ *
+ * This function will never return the infinity bin (histogram_len-1), in order
+ * to simplify the rest of the code, so if a usec is provided that falls above
+ * the highest non-infinity bin, that bin index will be returned.
+ */
+STATIC circpad_hist_index_t
+circpad_histogram_usec_to_bin(const circpad_machine_runtime_t *mi,
+                              circpad_delay_t usec)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+  circpad_hist_index_t bin;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return 0;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  /* Walk through the bins and check the upper bound of each bin, if 'usec' is
+   * less-or-equal to that, return that bin. If rtt_estimate is enabled then
+   * add that to the upper bound of each bin.
+   *
+   * We don't want to return the infinity bin here, so don't go there. */
+  for (bin = 0 ; bin < CIRCPAD_INFINITY_BIN(state) ; bin++) {
+    if (usec <= histogram_get_bin_upper_bound(mi, bin) + rtt_add_usec) {
+      return bin;
+    }
+  }
+
+  /* We don't want to return the infinity bin here, so if we still didn't find
+   * the right bin, return the highest non-infinity bin */
+  return CIRCPAD_INFINITY_BIN(state)-1;
+}
+
+/**
+ * Return true if the machine supports token removal.
+ *
+ * Token removal is equivalent to having a mutable histogram in the
+ * circpad_machine_runtime_t mutable info. So while we're at it,
+ * let's assert that everything is consistent between the mutable
+ * runtime and the readonly machine spec.
+ */
+static inline int
+circpad_is_token_removal_supported(circpad_machine_runtime_t *mi)
+{
+  /* No runtime histogram == no token removal */
+  if (mi->histogram == NULL) {
+    /* Machines that don't want token removal are trying to avoid
+     * potentially expensive mallocs, extra memory accesses, and/or
+     * potentially expensive monotime calls. Let's minimize checks
+     * and keep this path fast. */
+    tor_assert_nonfatal(mi->histogram_len == 0);
+    return 0;
+  } else {
+    /* Machines that do want token removal are less sensitive to performance.
+     * Let's spend some time to check that our state is consistent and sane */
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return 1;
+    }
+    tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE);
+    tor_assert_nonfatal(state->histogram_len == mi->histogram_len);
+    tor_assert_nonfatal(mi->histogram_len != 0);
+    return 1;
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * This function frees any token bins allocated from a previous state
+ *
+ * Called after a state transition, or if the bins are empty.
+ */
+STATIC void
+circpad_machine_setup_tokens(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+
+  /* If this state doesn't exist, or doesn't have token removal,
+   * free any previous state's runtime histogram, and bail.
+   *
+   * If we don't have a token removal strategy, we also don't need a runtime
+   * histogram and we rely on the immutable one in machine_spec_t. */
+  if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) {
+    if (mi->histogram) {
+      tor_free(mi->histogram);
+      mi->histogram = NULL;
+      mi->histogram_len = 0;
+    }
+    return;
+  }
+
+  /* Try to avoid re-mallocing if we don't really need to */
+  if (!mi->histogram || (mi->histogram
+          && mi->histogram_len != state->histogram_len)) {
+    tor_free(mi->histogram); // null ok
+    mi->histogram = tor_malloc_zero(sizeof(circpad_hist_token_t)
+                                    *state->histogram_len);
+  }
+  mi->histogram_len = state->histogram_len;
+
+  memcpy(mi->histogram, state->histogram,
+         sizeof(circpad_hist_token_t)*state->histogram_len);
+}
+
+/**
+ * Choose a length for this state (in cells), if specified.
+ */
+static void
+circpad_choose_state_length(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  double length;
+
+  if (!state || state->length_dist.type == CIRCPAD_DIST_NONE) {
+    mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+    return;
+  }
+
+  length = circpad_distribution_sample(state->length_dist);
+  length = MAX(0, length);
+  length += state->start_length;
+
+  if (state->max_length) {
+    length = MIN(length, state->max_length);
+  }
+
+  mi->state_length = clamp_double_to_int64(length);
+
+  log_info(LD_CIRC, "State length sampled to %"PRIu64" for circuit %u",
+      mi->state_length, CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+}
+
+/**
+ * Sample a value from our iat_dist, and clamp it safely
+ * to circpad_delay_t.
+ *
+ * Before returning, add <b>delay_shift</b> (can be zero) to the sampled value.
+ */
+static circpad_delay_t
+circpad_distribution_sample_iat_delay(const circpad_state_t *state,
+                                      circpad_delay_t delay_shift)
+{
+  double val = circpad_distribution_sample(state->iat_dist);
+  /* These comparisons are safe, because the output is in the range
+   * [0, 2**32), and double has a precision of 53 bits. */
+  /* We want a positive sample value */
+  val = MAX(0, val);
+  /* Respect the maximum sample setting */
+  val = MIN(val, state->dist_max_sample_usec);
+
+  /* Now apply the shift:
+   * This addition is exact: val is at most 2**32-1, delay_shift is at most
+   * 2**32-1, and doubles have a precision of 53 bits. */
+  val += delay_shift;
+
+  /* Clamp the distribution at infinite delay val */
+  return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE);
+}
+
+/**
+ * Sample an expected time-until-next-packet delay from the histogram or
+ * probability distribution.
+ *
+ * A bin of the histogram is chosen with probability proportional to the number
+ * of tokens in each bin, and then a time value is chosen uniformly from that
+ * bin's [start,end) time range.
+ */
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  const circpad_hist_token_t *histogram = NULL;
+  circpad_hist_index_t curr_bin = 0;
+  circpad_delay_t bin_start, bin_end;
+  /* These three must all be larger than circpad_hist_token_t, because
+   * we sum several circpad_hist_token_t values across the histogram */
+  uint64_t curr_weight = 0;
+  uint64_t histogram_total_tokens = 0;
+  uint64_t bin_choice;
+
+  tor_assert(state);
+
+  if (state->iat_dist.type != CIRCPAD_DIST_NONE) {
+    /* Sample from a fixed IAT distribution and return */
+    circpad_delay_t iat_delay_shift = state->use_rtt_estimate ?
+      mi->rtt_estimate_usec + state->dist_added_shift_usec :
+      state->dist_added_shift_usec;
+    return circpad_distribution_sample_iat_delay(state, iat_delay_shift);
+  } else if (circpad_is_token_removal_supported(mi)) {
+    histogram = mi->histogram;
+    for (circpad_hist_index_t b = 0; b < state->histogram_len; b++)
+      histogram_total_tokens += histogram[b];
+  } else {
+    /* We have a histogram, but it's immutable */
+    histogram = state->histogram;
+    histogram_total_tokens = state->histogram_total_tokens;
+  }
+
+  /* If we are out of tokens, don't schedule padding. */
+  if (!histogram_total_tokens) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  bin_choice = crypto_fast_rng_get_uint64(get_thread_fast_rng(),
+                                          histogram_total_tokens);
+
+  /* Skip all the initial zero bins */
+  while (!histogram[curr_bin]) {
+    curr_bin++;
+  }
+  curr_weight = histogram[curr_bin];
+
+  // TODO: This is not constant-time. Pretty sure we don't
+  // really need it to be, though.
+  while (curr_weight < bin_choice) {
+    curr_bin++;
+    /* It should be impossible to run past the end of the histogram */
+    if (BUG(curr_bin >= state->histogram_len)) {
+      return CIRCPAD_DELAY_INFINITE;
+    }
+    curr_weight += histogram[curr_bin];
+  }
+
+  /* Do some basic checking of the current bin we are in */
+  if (BUG(curr_bin >= state->histogram_len) ||
+      BUG(histogram[curr_bin] == 0)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  // Store this index to remove the token upon callback.
+  if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) {
+    mi->chosen_bin = curr_bin;
+  }
+
+  if (curr_bin >= CIRCPAD_INFINITY_BIN(state)) {
+    if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE &&
+        mi->histogram[curr_bin] > 0) {
+      mi->histogram[curr_bin]--;
+    }
+
+    // Infinity: Don't send a padding packet. Wait for a real packet
+    // and then see if our bins are empty or what else we should do.
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state));
+
+  bin_start = circpad_histogram_bin_to_usec(mi, curr_bin);
+  /* We don't need to reduct 1 from the upper bound because the random range
+   * function below samples from [bin_start, bin_end) */
+  bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1);
+
+  /* Bin edges are monotonically increasing so this is a bug. Handle it. */
+  if (BUG(bin_start >= bin_end)) {
+    return bin_start;
+  }
+
+  return (circpad_delay_t)crypto_fast_rng_uint64_range(get_thread_fast_rng(),
+                                                       bin_start, bin_end);
+}
+
+/**
+ * Sample a value from the specified probability distribution.
+ *
+ * Uses functions from src/lib/math/prob_distr.c .
+ */
+static double
+circpad_distribution_sample(circpad_distribution_t dist)
+{
+  log_fn(LOG_DEBUG,LD_CIRC, "Sampling delay with distribution %d",
+         dist.type);
+
+  switch (dist.type) {
+    case CIRCPAD_DIST_NONE:
+      {
+        /* We should not get in here like this */
+        tor_assert_nonfatal_unreached();
+        return 0;
+      }
+    case CIRCPAD_DIST_UNIFORM:
+      {
+        // param2 is upper bound, param1 is lower
+        const struct uniform_t my_uniform = {
+          .base = UNIFORM(my_uniform),
+          .a = dist.param1,
+          .b = dist.param2,
+        };
+        return dist_sample(&my_uniform.base);
+      }
+    case CIRCPAD_DIST_LOGISTIC:
+      {
+      /* param1 is Mu, param2 is sigma. */
+        const struct logistic_t my_logistic = {
+          .base = LOGISTIC(my_logistic),
+          .mu = dist.param1,
+          .sigma = dist.param2,
+        };
+        return dist_sample(&my_logistic.base);
+      }
+    case CIRCPAD_DIST_LOG_LOGISTIC:
+      {
+        /* param1 is Alpha, param2 is 1.0/Beta */
+        const struct log_logistic_t my_log_logistic = {
+          .base = LOG_LOGISTIC(my_log_logistic),
+          .alpha = dist.param1,
+          .beta = dist.param2,
+        };
+        return dist_sample(&my_log_logistic.base);
+      }
+    case CIRCPAD_DIST_GEOMETRIC:
+      {
+        /* param1 is 'p' (success probability) */
+        const struct geometric_t my_geometric = {
+          .base = GEOMETRIC(my_geometric),
+          .p = dist.param1,
+        };
+        return dist_sample(&my_geometric.base);
+      }
+    case CIRCPAD_DIST_WEIBULL:
+      {
+        /* param1 is k, param2 is Lambda */
+        const struct weibull_t my_weibull = {
+          .base = WEIBULL(my_weibull),
+          .k = dist.param1,
+          .lambda = dist.param2,
+        };
+        return dist_sample(&my_weibull.base);
+      }
+    case CIRCPAD_DIST_PARETO:
+      {
+        /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */
+        const struct genpareto_t my_genpareto = {
+          .base = GENPARETO(my_genpareto),
+          .mu = 0,
+          .sigma = dist.param1,
+          .xi = dist.param2,
+        };
+        return dist_sample(&my_genpareto.base);
+      }
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * Find the index of the first bin whose upper bound is
+ * greater than the target, and that has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_higher_index(const circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  /* Don't remove from the infinity bin */
+  for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) {
+    if (mi->histogram[bin] &&
+        histogram_get_bin_upper_bound(mi, bin) >= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return mi->histogram_len;
+}
+
+/**
+ * Find the index of the first bin whose lower bound is lower or equal to
+ * <b>target_bin_usec</b>, and that still has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_lower_index(const circpad_machine_runtime_t *mi,
+                                  circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  for (; bin >= 0; bin--) {
+    if (mi->histogram[bin] &&
+        circpad_histogram_bin_to_usec(mi, bin) <= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return -1;
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * greater than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                    circpad_delay_t target_bin_usec)
+{
+  /* We need to remove the token from the first bin
+   * whose upper bound is greater than the target, and that
+   * has tokens remaining. */
+  circpad_hist_index_t bin = circpad_machine_first_higher_index(mi,
+                                                     target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * lower than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_machine_first_lower_index(mi,
+          target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/* Helper macro: Ensure that the bin has tokens available, and BUG out of the
+ * function if it's not the case. */
+#define ENSURE_BIN_CAPACITY(bin_index) \
+  if (BUG(mi->histogram[bin_index] == 0)) {                   \
+    return;                                                   \
+  }
+
+/**
+ * Remove a token from the closest non-empty bin to the target.
+ *
+ * If use_usec is true, measure "closest" in terms of the next closest bin
+ * midpoint.
+ *
+ * If it is false, use bin index distance only.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                     circpad_delay_t target_bin_usec,
+                                     bool use_usec)
+{
+  circpad_hist_index_t lower, higher, current;
+  circpad_hist_index_t bin_to_remove = -1;
+
+  lower = circpad_machine_first_lower_index(mi, target_bin_usec);
+  higher = circpad_machine_first_higher_index(mi, target_bin_usec);
+  current = circpad_histogram_usec_to_bin(mi, target_bin_usec);
+
+  /* Sanity check the results */
+  if (BUG(lower > current) || BUG(higher < current)) {
+    return;
+  }
+
+  /* Take care of edge cases first */
+  if (higher == mi->histogram_len && lower == -1) {
+    /* All bins are empty */
+    return;
+  } else if (higher == mi->histogram_len) {
+    /* All higher bins are empty */
+    ENSURE_BIN_CAPACITY(lower);
+    mi->histogram[lower]--;
+    return;
+  } else if (lower == -1) {
+    /* All lower bins are empty */
+    ENSURE_BIN_CAPACITY(higher);
+    mi->histogram[higher]--;
+    return;
+  }
+
+  /* Now handle the intermediate cases */
+  if (use_usec) {
+    /* Find the closest bin midpoint to the target */
+    circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower);
+    circpad_delay_t higher_usec =
+      circpad_get_histogram_bin_midpoint(mi, higher);
+
+    if (target_bin_usec < lower_usec) {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    } else if (target_bin_usec > higher_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else if (target_bin_usec-lower_usec > higher_usec-target_bin_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    }
+    mi->histogram[bin_to_remove]--;
+    log_debug(LD_CIRC, "Removing token from bin %d", bin_to_remove);
+    return;
+  } else {
+    if (current - lower > higher - current) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      mi->histogram[higher]--;
+      return;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      mi->histogram[lower]--;
+      return;
+    }
+  }
+}
+
+#undef ENSURE_BIN_CAPACITY
+
+/**
+ * Remove a token from the exact bin corresponding to the target.
+ *
+ * If it is empty, do nothing.
+ *
+ * Used for histograms with token removal.
+ */
+static void
+circpad_machine_remove_exact(circpad_machine_runtime_t *mi,
+                             circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+          target_bin_usec);
+
+  if (mi->histogram[bin] > 0)
+    mi->histogram[bin]--;
+}
+
+/**
+ * Check our state's cell limit count and tokens.
+ *
+ * Returns 1 if either limits are hit and we decide to change states,
+ * otherwise returns 0.
+ */
+static circpad_decision_t
+check_machine_token_supply(circpad_machine_runtime_t *mi)
+{
+  uint32_t histogram_total_tokens = 0;
+
+  /* Check if bins empty. This requires summing up the current mutable
+   * machineinfo histogram token total and checking if it is zero.
+   * Machineinfo does not keep a running token count. We're assuming the
+   * extra space is not worth this short loop iteration.
+   *
+   * We also do not count infinity bin in histogram totals.
+   */
+  if (circpad_is_token_removal_supported(mi)) {
+    for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++)
+      histogram_total_tokens += mi->histogram[b];
+
+    /* If we change state, we're done */
+    if (histogram_total_tokens == 0) {
+      if (circpad_internal_event_bins_empty(mi) == CIRCPAD_STATE_CHANGED)
+        return CIRCPAD_STATE_CHANGED;
+    }
+  }
+
+  if (mi->state_length == 0) {
+    return circpad_internal_event_state_length_up(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Count that a padding packet was sent.
+ *
+ * This updates our state length count, our machine rate limit counts,
+ * and if token removal is used, decrements the histogram.
+ */
+static inline void
+circpad_machine_count_padding_sent(circpad_machine_runtime_t *mi)
+{
+  /* If we have a valid state length bound, consider it */
+  if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE &&
+      !BUG(mi->state_length <= 0)) {
+    mi->state_length--;
+  }
+
+  /*
+   * Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct.
+   */
+  mi->padding_sent++;
+  if (mi->padding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  circpad_global_padding_sent++;
+
+  /* If we have a mutable histogram, reduce the token count from
+   * the chosen padding bin (this assumes we always send padding
+   * when we intended to). */
+  if (circpad_is_token_removal_supported(mi)) {
+    /* Check array bounds and token count before removing */
+    if (!BUG(mi->chosen_bin >= mi->histogram_len) &&
+        !BUG(mi->histogram[mi->chosen_bin] == 0)) {
+      mi->histogram[mi->chosen_bin]--;
+    }
+  }
+}
+
+/**
+ * Count a nonpadding packet as being sent.
+ *
+ * This function updates our overhead accounting variables, as well
+ * as decrements the state limit packet counter, if the latter was
+ * flagged as applying to non-padding as well.
+ */
+static inline void
+circpad_machine_count_nonpadding_sent(circpad_machine_runtime_t *mi)
+{
+  /* Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct. */
+  mi->nonpadding_sent++;
+  if (mi->nonpadding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  /* Update any state packet length limits that apply */
+  circpad_machine_update_state_length_for_nonpadding(mi);
+
+  /* Remove a token from the histogram, if applicable */
+  circpad_machine_remove_token(mi);
+}
+
+/**
+ * Decrement the state length counter for a non-padding packet.
+ *
+ * Only updates the state length if we're using that feature, we
+ * have a state, and the machine wants to count non-padding packets
+ * towards the state length.
+ */
+static inline void
+circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+
+  if (mi->state_length == CIRCPAD_STATE_LENGTH_INFINITE)
+    return;
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+
+  /* If we're enforcing a state length on non-padding packets,
+   * decrement it */
+  if (state->length_includes_nonpadding &&
+      mi->state_length > 0) {
+    mi->state_length--;
+  }
+}
+
+/**
+ * When a non-padding packet arrives, remove a token from the bin
+ * corresponding to the delta since last sent packet. If that bin
+ * is empty, choose a token based on the specified removal strategy
+ * in the state machine.
+ */
+STATIC void
+circpad_machine_remove_token(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+  circpad_time_t current_time;
+  circpad_delay_t target_bin_usec;
+
+  /* Dont remove any tokens if there was no padding scheduled */
+  if (!mi->padding_scheduled_at_usec) {
+    return;
+  }
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+  /* Don't remove any tokens if we're not doing token removal */
+  if (state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE)
+    return;
+
+  current_time = monotime_absolute_usec();
+
+  /* If we have scheduled padding some time in the future, we want to see what
+     bin we are in at the current time */
+  target_bin_usec = (circpad_delay_t)
+                  MIN((current_time - mi->padding_scheduled_at_usec),
+                      CIRCPAD_DELAY_INFINITE-1);
+
+  /* We are treating this non-padding cell as a padding cell, so we cancel
+     padding timer, if present. */
+  mi->padding_scheduled_at_usec = 0;
+  if (mi->is_padding_timer_scheduled) {
+    mi->is_padding_timer_scheduled = 0;
+    timer_disable(mi->padding_timer);
+  }
+
+  /* Perform the specified token removal strategy */
+  switch (state->token_removal) {
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 1);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 0);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_LOWER:
+      circpad_machine_remove_lower_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_HIGHER:
+      circpad_machine_remove_higher_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_EXACT:
+      circpad_machine_remove_exact(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_NONE:
+    default:
+      tor_assert_nonfatal_unreached();
+      log_warn(LD_BUG, "Circpad: Unknown token removal strategy %d",
+               state->token_removal);
+      break;
+  }
+}
+
+/**
+ * Send a relay command with a relay cell payload on a circuit to
+ * the particular hopnum.
+ *
+ * Hopnum starts at 1 (1=guard, 2=middle, 3=exit, etc).
+ *
+ * Payload may be null.
+ *
+ * Returns negative on error, 0 on success.
+ */
+MOCK_IMPL(STATIC signed_error_t,
+circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len))
+{
+  crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum);
+  signed_error_t ret;
+
+  /* Check that the cpath has the target hop */
+  if (!target_hop) {
+    log_fn(LOG_WARN, LD_BUG, "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier, circuit_get_cpath_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Check that the target hop is opened */
+  if (target_hop->state != CPATH_STATE_OPEN) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier,
+           circuit_get_cpath_opened_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Send the drop command to the second hop */
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ), relay_command,
+                                     (const char*)payload, payload_len,
+                                     target_hop);
+  return ret;
+}
+
+/**
+ * Callback helper to send a padding cell.
+ *
+ * This helper is called after our histogram-sampled delay period passes
+ * without another packet being sent first. If a packet is sent before this
+ * callback happens, it is canceled. So when we're called here, send padding
+ * right away.
+ *
+ * If sending this padding cell forced us to transition states return
+ * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED.
+ */
+circpad_decision_t
+circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi)
+{
+  circuit_t *circ = mi->on_circ;
+  int machine_idx = mi->machine_index;
+  mi->padding_scheduled_at_usec = 0;
+  circpad_statenum_t state = mi->current_state;
+
+  /* Make sure circuit didn't close on us */
+  if (mi->on_circ->marked_for_close) {
+    log_fn(LOG_INFO,LD_CIRC,
+           "Padding callback on circuit marked for close (%u). Ignoring.",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+         TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_CHANGED;
+  }
+
+  circpad_machine_count_padding_sent(mi);
+
+  if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+    circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ),
+                                CIRCPAD_GET_MACHINE(mi)->target_hopnum,
+                                RELAY_COMMAND_DROP, NULL, 0);
+    log_info(LD_CIRC, "Callback: Sending padding to origin circuit %u"
+             " (%d) [length: %"PRIu64"]",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier,
+             mi->on_circ->purpose, mi->state_length);
+  } else {
+    // If we're a non-origin circ, we can just send from here as if we're the
+    // edge.
+    if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) {
+      log_info(LD_CIRC, "Callback: Sending padding to circuit (%d)"
+               " [length: %"PRIu64"]", mi->on_circ->purpose, mi->state_length);
+      relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL,
+                                   0, NULL);
+      rep_hist_padding_count_write(PADDING_TYPE_DROP);
+    } else {
+      static ratelim_t cell_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC,
+                     "Too many cells (%d) in circ queue to send padding.",
+                      TO_OR_CIRCUIT(circ)->p_chan_cells.n);
+    }
+  }
+
+  /* This is a padding cell sent from the client or from the middle node,
+   * (because it's invoked from circuitpadding.c) */
+  circpad_cell_event_padding_sent(circ);
+
+  /* The circpad_cell_event_padding_sent() could cause us to transition.
+   * Check that we still have a padding machineinfo, and then check our token
+   * supply. */
+  if (circ->padding_info[machine_idx] != NULL) {
+    if (state != circ->padding_info[machine_idx]->current_state)
+      return CIRCPAD_STATE_CHANGED;
+    else
+      return check_machine_token_supply(circ->padding_info[machine_idx]);
+  } else {
+    return CIRCPAD_STATE_CHANGED;
+  }
+}
+
+/**
+ * Tor-timer compatible callback that tells us to send a padding cell.
+ *
+ * Timers are associated with circpad_machine_runtime_t's. When the machineinfo
+ * is freed on a circuit, the timers are cancelled. Since the lifetime
+ * of machineinfo is always longer than the timers, handles are not
+ * needed.
+ */
+static void
+circpad_send_padding_callback(tor_timer_t *timer, void *args,
+                              const struct monotime_t *time)
+{
+  circpad_machine_runtime_t *mi = ((circpad_machine_runtime_t*)args);
+  (void)timer; (void)time;
+
+  if (mi && mi->on_circ) {
+    assert_circuit_ok(mi->on_circ);
+    circpad_send_padding_cell_for_callback(mi);
+  } else {
+    // This shouldn't happen (represents a timer leak)
+    log_fn(LOG_WARN,LD_CIRC,
+            "Circuit closed while waiting for padding timer.");
+    tor_fragile_assert();
+  }
+
+  // TODO-MP-AP: Unify this counter with channelpadding for rephist stats
+  //total_timers_pending--;
+}
+
+/**
+ * Cache our consensus parameters upon consensus update.
+ */
+void
+circpad_new_consensus_params(const networkstatus_t *ns)
+{
+  circpad_padding_disabled =
+      networkstatus_get_param(ns, "circpad_padding_disabled",
+         0, 0, 1);
+
+  circpad_padding_reduced =
+      networkstatus_get_param(ns, "circpad_padding_reduced",
+         0, 0, 1);
+
+  circpad_global_allowed_cells =
+      networkstatus_get_param(ns, "circpad_global_allowed_cells",
+         0, 0, UINT16_MAX-1);
+
+  circpad_global_max_padding_percent =
+      networkstatus_get_param(ns, "circpad_global_max_padding_pct",
+         0, 0, 100);
+
+  circpad_max_circ_queued_cells =
+      networkstatus_get_param(ns, "circpad_max_circ_queued_cells",
+         CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX);
+}
+
+/**
+ * Return true if padding is allowed by torrc and consensus.
+ */
+static bool
+circpad_is_padding_allowed(void)
+{
+  /* If padding has been disabled in the consensus, don't send any more
+   * padding. Technically the machine should be shut down when the next
+   * machine condition check happens, but machine checks only happen on
+   * certain circuit events, and if padding is disabled due to some
+   * network overload or DoS condition, we really want to stop ASAP. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * Check this machine against its padding limits, as well as global
+ * consensus limits.
+ *
+ * We have two limits: a percent and a cell count. The cell count
+ * limit must be reached before the percent is enforced (this is to
+ * optionally allow very light padding of things like circuit setup
+ * while there is no other traffic on the circuit).
+ *
+ * TODO: Don't apply limits to machines form torrc.
+ *
+ * Returns 1 if limits are set and we've hit them. Otherwise returns 0.
+ */
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  /* If machine_padding_pct is non-zero, and we've sent more
+   * than the allowed count of padding cells, then check our
+   * percent limits for this machine. */
+  if (machine->max_padding_percent &&
+      mi->padding_sent >= machine->allowed_padding_count) {
+    uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*(uint32_t)mi->padding_sent) / total_cells >
+        machine->max_padding_percent) {
+      return 1; // limit is reached. Stop.
+    }
+  }
+
+  /* If circpad_max_global_padding_pct is non-zero, and we've
+   * sent more than the global padding cell limit, then check our
+   * global tor process percentage limit on padding. */
+  if (circpad_global_max_padding_percent &&
+      circpad_global_padding_sent >= circpad_global_allowed_cells) {
+    uint64_t total_cells = circpad_global_padding_sent +
+              circpad_global_nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*circpad_global_padding_sent) / total_cells >
+        circpad_global_max_padding_percent) {
+      return 1; // global limit reached. Stop.
+    }
+  }
+
+  return 0; // All good!
+}
+
+/**
+ * Schedule the next padding time according to the machineinfo on a
+ * circuit.
+ *
+ * The histograms represent inter-packet-delay. Whenever you get an packet
+ * event you should be scheduling your next timer (after cancelling any old
+ * ones and updating tokens accordingly).
+ *
+ * Returns 1 if we decide to transition states (due to infinity bin),
+ * 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi))
+{
+  circpad_delay_t in_usec = 0;
+  struct timeval timeout;
+  tor_assert(mi);
+
+  /* Don't schedule padding if it is disabled */
+  if (!circpad_is_padding_allowed()) {
+    static ratelim_t padding_lim = RATELIM_INIT(600);
+    log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+         "Padding has been disabled, but machine still on circuit %"PRIu64
+         ", %d",
+         mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+         mi->on_circ->n_circ_id);
+
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Don't schedule padding if we are currently in dormant mode. */
+  if (!is_participating_on_network()) {
+    log_info(LD_CIRC, "Not scheduling padding because we are dormant.");
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  // Don't pad in end (but  also don't cancel any previously
+  // scheduled padding either).
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    log_fn(LOG_INFO, LD_CIRC, "Padding end state on circuit %u",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check our padding limits */
+  if (circpad_machine_reached_padding_limit(mi)) {
+   if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Padding machine has reached padding limit on circuit %u",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier);
+    } else {
+      static ratelim_t padding_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+           "Padding machine has reached padding limit on circuit %"PRIu64
+           ", %d",
+           mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+           mi->on_circ->n_circ_id);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (mi->is_padding_timer_scheduled) {
+    /* Cancel current timer (if any) */
+    timer_disable(mi->padding_timer);
+    mi->is_padding_timer_scheduled = 0;
+  }
+
+  /* in_usec = in microseconds */
+  in_usec = circpad_machine_sample_delay(mi);
+  /* If we're using token removal, we need to know when the padding
+   * was scheduled at, so we can remove the appropriate token if
+   * a non-padding cell is sent before the padding timer expires.
+   *
+   * However, since monotime is unpredictably expensive, let's avoid
+   * using it for machines that don't need token removal. */
+  if (circpad_is_token_removal_supported(mi)) {
+    mi->padding_scheduled_at_usec = monotime_absolute_usec();
+  } else {
+    mi->padding_scheduled_at_usec = 1;
+  }
+  log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec on circuit %u", in_usec,
+       CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+
+  // Don't schedule if we have infinite delay.
+  if (in_usec == CIRCPAD_DELAY_INFINITE) {
+    return circpad_internal_event_infinity(mi);
+  }
+
+  if (mi->state_length == 0) {
+    /* If we're at length 0, that means we hit 0 after sending
+     * a cell earlier, and emitted an event for it, but
+     * for whatever reason we did not decide to change states then.
+     * So maybe the machine is waiting for bins empty, or for an
+     * infinity event later? That would be a strange machine,
+     * but there's no reason to make it impossible. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (in_usec <= 0) {
+    return circpad_send_padding_cell_for_callback(mi);
+  }
+
+  timeout.tv_sec = in_usec/TOR_USEC_PER_SEC;
+  timeout.tv_usec = (in_usec%TOR_USEC_PER_SEC);
+
+  log_fn(LOG_INFO, LD_CIRC, "\tPadding circuit %u in %u sec, %u usec",
+     CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+          (unsigned)timeout.tv_sec, (unsigned)timeout.tv_usec);
+
+  if (mi->padding_timer) {
+    timer_set_cb(mi->padding_timer, circpad_send_padding_callback, mi);
+  } else {
+    mi->padding_timer =
+        timer_new(circpad_send_padding_callback, mi);
+  }
+  timer_schedule(mi->padding_timer, &timeout);
+  mi->is_padding_timer_scheduled = 1;
+
+  // TODO-MP-AP: Unify with channelpadding counter
+  //rep_hist_padding_count_timers(++total_timers_pending);
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * If the machine transitioned to the END state, we need
+ * to check to see if it wants us to shut it down immediately.
+ * If it does, then we need to send the appropiate negotiation commands
+ * depending on which side it is.
+ *
+ * After this function is called, mi may point to freed memory. Do
+ * not access it.
+ */
+static void
+circpad_machine_spec_transitioned_to_end(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+  circuit_t *on_circ = mi->on_circ;
+
+  log_fn(LOG_INFO,LD_CIRC, "Padding machine in end state on circuit %u (%d)",
+         CIRCUIT_IS_ORIGIN(on_circ) ?
+         TO_ORIGIN_CIRCUIT(on_circ)->global_identifier : 0,
+         on_circ->purpose);
+
+  /*
+   * We allow machines to shut down and delete themselves as opposed
+   * to just going back to START or waiting forever in END so that
+   * we can handle the case where this machine started while it was
+   * the only machine that matched conditions, but *since* then more
+   * "higher ranking" machines now match the conditions, and would
+   * be given a chance to take precedence over this one in
+   * circpad_add_matching_machines().
+   *
+   * Returning to START or waiting forever in END would not give those
+   * other machines a chance to be launched, where as shutting down
+   * here does.
+   */
+  if (machine->should_negotiate_end) {
+    if (machine->is_origin_side) {
+      /* We free the machine info here so that we can be replaced
+       * by a different machine. But we must leave the padding_machine
+       * in place to wait for the negotiated response */
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(on_circ),
+                                machine->machine_num,
+                                machine->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    } else {
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_padding_negotiated(on_circ,
+                                machine->machine_num,
+                                CIRCPAD_COMMAND_STOP,
+                                CIRCPAD_RESPONSE_OK);
+      on_circ->padding_machine[machine->machine_index] = NULL;
+    }
+  }
+}
+
+/**
+ * Generic state transition function for padding state machines.
+ *
+ * Given an event and our mutable machine info, decide if/how to
+ * transition to a different state, and perform actions accordingly.
+ *
+ * Returns 1 if we transition states, 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_spec_transition,(circpad_machine_runtime_t *mi,
+                            circpad_event_t event))
+{
+  const circpad_state_t *state =
+      circpad_machine_current_state(mi);
+
+  /* If state is null we are in the end state. */
+  if (!state) {
+    /* If we in end state we don't pad no matter what. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check if this event is ignored or causes a cancel */
+  if (state->next_state[event] == CIRCPAD_STATE_IGNORE) {
+    return CIRCPAD_STATE_UNCHANGED;
+  } else if (state->next_state[event] == CIRCPAD_STATE_CANCEL) {
+    /* Check cancel events and cancel any pending padding */
+    mi->padding_scheduled_at_usec = 0;
+    if (mi->is_padding_timer_scheduled) {
+      mi->is_padding_timer_scheduled = 0;
+      /* Cancel current timer (if any) */
+      timer_disable(mi->padding_timer);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  } else {
+    circpad_statenum_t s = state->next_state[event];
+    /* See if we need to transition to any other states based on this event.
+     * Whenever a transition happens, even to our own state, we schedule
+     * padding.
+     *
+     * So if a state only wants to schedule padding for an event, it specifies
+     * a transition to itself. All non-specified events are ignored.
+     */
+    log_fn(LOG_INFO, LD_CIRC,
+           "Circuit %u circpad machine %d transitioning from %u to %u",
+             CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+           mi->machine_index, mi->current_state, s);
+
+    /* If this is not the same state, switch and init tokens,
+     * otherwise just reschedule padding. */
+    if (mi->current_state != s) {
+      mi->current_state = s;
+      circpad_machine_setup_tokens(mi);
+      circpad_choose_state_length(mi);
+
+      /* If we transition to the end state, check to see
+       * if this machine wants to be shut down at end */
+      if (s == CIRCPAD_STATE_END) {
+        circpad_machine_spec_transitioned_to_end(mi);
+        /* We transitioned but we don't pad in end. Also, mi
+         * may be freed. Returning STATE_CHANGED prevents us
+         * from accessing it in any callers of this function. */
+        return CIRCPAD_STATE_CHANGED;
+      }
+
+      /* We transitioned to a new state, schedule padding */
+      circpad_machine_schedule_padding(mi);
+      return CIRCPAD_STATE_CHANGED;
+    }
+
+    /* We transitioned back to the same state. Schedule padding,
+     * and inform if that causes a state transition. */
+    return circpad_machine_schedule_padding(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Estimate the circuit RTT from the current middle hop out to the
+ * end of the circuit.
+ *
+ * We estimate RTT by calculating the time between "receive" and
+ * "send" at a middle hop. This is because we "receive" a cell
+ * from the origin, and then relay it towards the exit before a
+ * response comes back. It is that response time from the exit side
+ * that we want to measure, so that we can make use of it for synthetic
+ * response delays.
+ */
+static void
+circpad_estimate_circ_rtt_on_received(circuit_t *circ,
+                                      circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update)
+    return;
+
+  /* If we already have a last received packet time, that means we
+   * did not get a response before this packet. The RTT estimate
+   * only makes sense if we do not have multiple packets on the
+   * wire, so stop estimating if this is the second packet
+   * back to back. However, for the first set of back-to-back
+   * packets, we can wait until the very first response comes back
+   * to us, to measure that RTT (for the response to optimistic
+   * data, for example). Hence stop_rtt_update is only checked
+   * in this received side function, and not in send side below.
+   */
+  if (mi->last_received_time_usec) {
+    /* We also allow multiple back-to-back packets if the circuit is not
+     * opened, to handle var cells.
+     * XXX: Will this work with out var cell plans? Maybe not,
+     * since we're opened at the middle hop as soon as we process
+     * one var extend2 :/ */
+    if (circ->state == CIRCUIT_STATE_OPEN) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Stopping padding RTT estimation on circuit (%"PRIu64
+           ", %d) after two back to back packets. Current RTT: %d",
+           circ->n_chan ?  circ->n_chan->global_identifier : 0,
+           circ->n_circ_id, mi->rtt_estimate_usec);
+      mi->stop_rtt_update = 1;
+
+      if (!mi->rtt_estimate_usec) {
+        static ratelim_t rtt_lim = RATELIM_INIT(600);
+        log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+          "Circuit got two cells back to back before estimating RTT.");
+      }
+    }
+  } else {
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return;
+    }
+
+    /* Since monotime is unpredictably expensive, only update this field
+     * if rtt estimates are needed. Otherwise, stop the rtt update. */
+    if (state->use_rtt_estimate) {
+      mi->last_received_time_usec = monotime_absolute_usec();
+    } else {
+      /* Let's fast-path future decisions not to update rtt if the
+       * feature is not in use. */
+      mi->stop_rtt_update = 1;
+    }
+  }
+}
+
+/**
+ * Handles the "send" side of RTT calculation at middle nodes.
+ *
+ * This function calculates the RTT from the middle to the end
+ * of the circuit by subtracting the last received cell timestamp
+ * from the current time. It allows back-to-back cells until
+ * the circuit is opened, to allow for var cell handshakes.
+ * XXX: Check our var cell plans to make sure this will work.
+ */
+static void
+circpad_estimate_circ_rtt_on_send(circuit_t *circ,
+                                  circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ))
+    return;
+
+  /* If last_received_time_usec is non-zero, we are waiting for a response
+   * from the exit side. Calculate the time delta and use it as RTT. */
+  if (mi->last_received_time_usec) {
+    circpad_time_t rtt_time = monotime_absolute_usec() -
+        mi->last_received_time_usec;
+
+    /* Reset the last RTT packet time, so we can tell if two cells
+     * arrive back to back */
+    mi->last_received_time_usec = 0;
+
+    /* Use INT32_MAX to ensure the addition doesn't overflow */
+    if (rtt_time >= INT32_MAX) {
+      log_fn(LOG_WARN,LD_CIRC,
+             "Circuit padding RTT estimate overflowed: %"PRIu64
+             " vs %"PRIu64, monotime_absolute_usec(),
+               mi->last_received_time_usec);
+      return;
+    }
+
+    /* If the old RTT estimate is lower than this one, use this one, because
+     * the circuit is getting longer. If this estimate is somehow
+     * faster than the previous, then maybe that was network jitter, or a
+     * bad monotonic clock source (so our ratchet returned a zero delta).
+     * In that case, average them. */
+    if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) {
+      mi->rtt_estimate_usec = (circpad_delay_t)rtt_time;
+    } else {
+      mi->rtt_estimate_usec += (circpad_delay_t)rtt_time;
+      mi->rtt_estimate_usec /= 2;
+    }
+  } else if (circ->state == CIRCUIT_STATE_OPEN) {
+    /* If last_received_time_usec is zero, then we have gotten two cells back
+     * to back. Stop estimating RTT in this case. Note that we only
+     * stop RTT update if the circuit is opened, to allow for RTT estimates
+     * of var cells during circ setup. */
+    if (!mi->rtt_estimate_usec && !mi->stop_rtt_update) {
+      static ratelim_t rtt_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+        "Circuit sent two cells back to back before estimating RTT.");
+    }
+    mi->stop_rtt_update = 1;
+  }
+}
+
+/**
+ * A "non-padding" cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_sent(circuit_t *on_circ)
+{
+  /* Update global cell count */
+  circpad_global_nonpadding_sent++;
+
+  /* If there are no machines then this loop should not iterate */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]);
+
+    /* Then, do accounting */
+    circpad_machine_count_nonpadding_sent(on_circ->padding_info[i]);
+
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                                 CIRCPAD_EVENT_NONPADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/** Check if this cell or circuit are related to circuit padding and handle
+ *  them if so.  Return 0 if the cell was handled in this subsystem and does
+ *  not need any other consideration, otherwise return 1.
+ */
+int
+circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                            crypt_path_t *layer_hint,
+                            const relay_header_t *rh)
+{
+  /* First handle the padding commands, since we want to ignore any other
+   * commands if this circuit is padding-specific. */
+  switch (rh->command) {
+    case RELAY_COMMAND_DROP:
+      /* Already examined in circpad_deliver_recognized_relay_cell_events */
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATE:
+      circpad_handle_padding_negotiate(circ, cell);
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATED:
+      if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0)
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
+      return 0;
+  }
+
+  /* If this is a padding circuit we don't need to parse any other commands
+   * than the padding ones. Just drop them to the floor.
+   *
+   * Note: we deliberately do not call circuit_read_valid_data() here. The
+   * vanguards addon (specifically the 'bandguards' component's dropped cell
+   * detection) will thus close this circuit, as it would for any other
+   * unexpected cell. However, default tor will *not* close the circuit.
+   *
+   * This is intentional. We are not yet certain that is it optimal to keep
+   * padding circuits open in cases like these, rather than closing them.
+   * We suspect that continuing to pad is optimal against a passive classifier,
+   * but as soon as the adversary is active (even as a client adversary) this
+   * might change.
+   *
+   * So as a way forward, we log the cell command and circuit number, to
+   * help us enumerate the most common instances of this in testing with
+   * vanguards, to see which are common enough to verify and handle
+   * properly.
+   * - Mike
+   */
+  if (circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Ignored cell (%d) that arrived in padding circuit "
+                      " %u.", rh->command, CIRCUIT_IS_ORIGIN(circ) ?
+                           TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * A "non-padding" cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_received(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]);
+
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                               CIRCPAD_EVENT_NONPADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_padding_sent(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+
+      on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                             CIRCPAD_EVENT_PADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_padding_received(circuit_t *on_circ)
+{
+  /* identical to padding sent */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                              CIRCPAD_EVENT_PADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * An "infinite" delay has ben chosen from one of our histograms.
+ *
+ * "Infinite" delays mean don't send padding -- but they can also
+ * mean transition to another state depending on the state machine
+ * definitions. Check the rules and react accordingly.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY);
+}
+
+/**
+ * All of the bins of our current state's histogram's are empty.
+ *
+ * Check to see if this means transition to another state, and if
+ * not, refill the tokens.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *mi)
+{
+  if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY)
+      == CIRCPAD_STATE_CHANGED) {
+    return CIRCPAD_STATE_CHANGED;
+  } else {
+    /* If we dont transition, then we refill the tokens */
+    circpad_machine_setup_tokens(mi);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+}
+
+/**
+ * This state has used up its cell count. Emit the event and
+ * see if we transition.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_state_length_up(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT);
+}
+
+/**
+ * Returns true if the circuit matches the conditions.
+ */
+static inline bool
+circpad_machine_conditions_met(origin_circuit_t *circ,
+                               const circpad_machine_spec_t *machine)
+{
+  /* If padding is disabled, no machines should match/apply. This has
+   * the effect of shutting down all machines, and not adding any more. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding)
+    return 0;
+
+  /* If the consensus or our torrc has selected reduced connection padding,
+   * then only allow this machine if it is flagged as acceptable under
+   * reduced padding conditions */
+  if (circpad_padding_reduced || get_options()->ReducedCircuitPadding) {
+    if (!machine->conditions.reduced_padding_ok)
+      return 0;
+  }
+
+  if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose)
+      & machine->conditions.purpose_mask))
+    return 0;
+
+  if (machine->conditions.requires_vanguards) {
+    const or_options_t *options = get_options();
+
+    /* Pinned middles are effectively vanguards */
+    if (!(options->HSLayer2Nodes || options->HSLayer3Nodes))
+      return 0;
+  }
+
+  /* We check for any bits set in the circuit state mask so that machines
+   * can say any of the following through their state bitmask:
+   * "I want to apply to circuits with either streams or no streams"; OR
+   * "I only want to apply to circuits with streams"; OR
+   * "I only want to apply to circuits without streams". */
+  if (!(circpad_circuit_state(circ) & machine->conditions.state_mask))
+    return 0;
+
+  if (circuit_get_cpath_opened_len(circ) < machine->conditions.min_hops)
+    return 0;
+
+  return 1;
+}
+
+/**
+ * Returns a minimized representation of the circuit state.
+ *
+ * The padding code only cares if the circuit is building,
+ * opened, used for streams, and/or still has relay early cells.
+ * This returns a bitmask of all state properities that apply to
+ * this circuit.
+ */
+static inline
+circpad_circuit_state_t
+circpad_circuit_state(origin_circuit_t *circ)
+{
+  circpad_circuit_state_t retmask = 0;
+
+  if (circ->p_streams)
+    retmask |= CIRCPAD_CIRC_STREAMS;
+  else
+    retmask |= CIRCPAD_CIRC_NO_STREAMS;
+
+  /* We use has_opened to prevent cannibialized circs from flapping. */
+  if (circ->has_opened)
+    retmask |= CIRCPAD_CIRC_OPENED;
+  else
+    retmask |= CIRCPAD_CIRC_BUILDING;
+
+  if (circ->remaining_relay_early_cells > 0)
+    retmask |= CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  else
+    retmask |= CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+
+  return retmask;
+}
+
+/**
+ * Convert a normal circuit purpose into a bitmask that we can
+ * use for determining matching circuits.
+ */
+circpad_purpose_mask_t
+circpad_circ_purpose_to_mask(uint8_t circ_purpose)
+{
+  /* Treat OR circ purposes as ignored. They should not be passed here*/
+  if (BUG(circ_purpose <= CIRCUIT_PURPOSE_OR_MAX_)) {
+    return 0;
+  }
+
+  /* Treat new client circuit purposes as "OMG ITS EVERYTHING".
+   * This also should not happen */
+  if (BUG(circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1 > 32)) {
+    return CIRCPAD_PURPOSE_ALL;
+  }
+
+  /* Convert the purpose to a bit position */
+  return 1 << (circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1);
+}
+
+/**
+ * Shut down any machines whose conditions no longer match
+ * the current circuit.
+ */
+static void
+circpad_shutdown_old_machines(origin_circuit_t *on_circ)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    if (!circpad_machine_conditions_met(on_circ,
+                                        circ->padding_machine[i])) {
+      // Clear machineinfo (frees timers)
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      // Send padding negotiate stop
+      circpad_negotiate_padding(on_circ,
+                                circ->padding_machine[i]->machine_num,
+                                circ->padding_machine[i]->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Negotiate new machines that would apply to this circuit, given the machines
+ * inside <b>machines_sl</b>.
+ *
+ * This function checks to see if we have any free machine indexes,
+ * and for each free machine index, it initializes the most recently
+ * added origin-side padding machine that matches the target machine
+ * index and circuit conditions, and negotiates it with the appropriate
+ * middle relay.
+ */
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+#ifdef TOR_UNIT_TESTS
+  /* Tests don't have to init our padding machines */
+  if (!machines_sl)
+    return;
+#endif
+
+  /* If padding negotiation failed before, do not try again */
+  if (on_circ->padding_negotiation_failed)
+    return;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* If there is a padding machine info, this index is occupied.
+     * No need to check conditions for this index. */
+    if (circ->padding_info[i])
+      continue;
+
+    /* We have a free machine index. Check the origin padding
+     * machines in reverse order, so that more recently added
+     * machines take priority over older ones. */
+    SMARTLIST_FOREACH_REVERSE_BEGIN(machines_sl,
+                                    circpad_machine_spec_t *,
+                                    machine) {
+      /* Machine definitions have a specific target machine index.
+       * This is so event ordering is deterministic with respect
+       * to which machine gets events first when there are two
+       * machines installed on a circuit. Make sure we only
+       * add this machine if its target machine index is free. */
+      if (machine->machine_index == i &&
+          circpad_machine_conditions_met(on_circ, machine)) {
+
+        // We can only replace this machine if the target hopnum
+        // is the same, otherwise we'll get invalid data
+        if (circ->padding_machine[i]) {
+          if (circ->padding_machine[i]->target_hopnum !=
+              machine->target_hopnum)
+            continue;
+          /* Replace it. (Don't free - is global). */
+          circ->padding_machine[i] = NULL;
+        }
+
+        /* Set up the machine immediately so that the slot is occupied.
+         * We will tear it down on error return, or if there is an error
+         * response from the relay. */
+        circpad_setup_machine_on_circ(circ, machine);
+        if (circpad_negotiate_padding(on_circ, machine->machine_num,
+                                  machine->target_hopnum,
+                                  CIRCPAD_COMMAND_START) < 0) {
+          log_info(LD_CIRC,
+                   "Padding not negotiated. Cleaning machine from circuit %u",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+          circpad_circuit_machineinfo_free_idx(circ, i);
+          circ->padding_machine[i] = NULL;
+          on_circ->padding_negotiation_failed = 1;
+        } else {
+          /* Success. Don't try any more machines on this index */
+          break;
+        }
+      }
+    } SMARTLIST_FOREACH_END(machine);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Event that tells us we added a hop to an origin circuit.
+ *
+ * This event is used to decide if we should create a padding machine
+ * on a circuit.
+ */
+void
+circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ)
+{
+  /* Since our padding conditions do not specify a max_hops,
+   * all we can do is add machines here */
+  circpad_add_matching_machines(on_circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is now built.
+ *
+ * Shut down any machines that only applied to un-built circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_built(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Circpad purpose changed event.
+ *
+ * Shut down any machines that don't apply to our circ purpose.
+ * Activate any new ones that do.
+ */
+void
+circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is out of RELAY_EARLY
+ * cells.
+ *
+ * Shut down any machines that only applied to RELAY_EARLY circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams attached event.
+ *
+ * Called from link_apconn_to_circ() and handle_hs_exit_conn()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams detached event.
+ *
+ * Called from circuit_detach_stream()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Verify that padding is coming from the expected hop.
+ *
+ * Returns true if from_hop matches the target hop from
+ * one of our padding machines.
+ *
+ * Returns false if we're not an origin circuit, or if from_hop
+ * does not match one of the padding machines.
+ */
+bool
+circpad_padding_is_from_expected_hop(circuit_t *circ,
+                                     crypt_path_t *from_hop)
+{
+  crypt_path_t *target_hop = NULL;
+  if (!CIRCUIT_IS_ORIGIN(circ))
+    return 0;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* We have to check padding_machine and not padding_info/active
+     * machines here because padding may arrive after we shut down a
+     * machine. The info is gone, but the padding_machine waits
+     * for the padding_negotiated response to come back. */
+    if (!circ->padding_machine[i])
+      continue;
+
+    target_hop = circuit_get_cpath_hop(TO_ORIGIN_CIRCUIT(circ),
+                    circ->padding_machine[i]->target_hopnum);
+
+    if (target_hop == from_hop)
+      return 1;
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return 0;
+}
+
+/**
+ * Deliver circpad events for an "unrecognized cell".
+ *
+ * Unrecognized cells are sent to relays and are forwarded
+ * onto the next hop of their circuits. Unrecognized cells
+ * are by definition not padding. We need to tell relay-side
+ * state machines that a non-padding cell was sent or received,
+ * depending on the direction, so they can update their histograms
+ * and decide to pad or not.
+ */
+void
+circpad_deliver_unrecognized_cell_events(circuit_t *circ,
+                                         cell_direction_t dir)
+{
+  // We should never see unrecognized cells at origin.
+  // Our caller emits a warn when this happens.
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    return;
+  }
+
+  if (dir == CELL_DIRECTION_OUT) {
+    /* When direction is out (away from origin), then we received non-padding
+       cell coming from the origin to us. */
+    circpad_cell_event_nonpadding_received(circ);
+  } else if (dir == CELL_DIRECTION_IN) {
+    /* It's in and not origin, so the cell is going away from us.
+     * So we are relaying a non-padding cell towards the origin. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for "recognized" relay cells.
+ *
+ * Recognized cells are destined for this hop, either client or middle.
+ * Check if this is a padding cell or not, and send the appropiate
+ * received event.
+ */
+void
+circpad_deliver_recognized_relay_cell_events(circuit_t *circ,
+                                             uint8_t relay_command,
+                                             crypt_path_t *layer_hint)
+{
+  if (relay_command == RELAY_COMMAND_DROP) {
+    rep_hist_padding_count_read(PADDING_TYPE_DROP);
+
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      if (circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), 0);
+      } else {
+        /* This is unexpected padding. Ignore it for now. */
+        return;
+      }
+    }
+
+    /* The cell should be recognized by now, which means that we are on the
+       destination, which means that we received a padding cell. We might be
+       the client or the Middle node, still, because leaky-pipe. */
+    circpad_cell_event_padding_received(circ);
+    log_fn(LOG_INFO, LD_CIRC, "Got padding cell on %s circuit %u.",
+           CIRCUIT_IS_ORIGIN(circ) ? "origin" : "non-origin",
+           CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+  } else {
+    /* We received a non-padding cell on the edge */
+    circpad_cell_event_nonpadding_received(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for relay cells sent from us.
+ *
+ * If this is a padding cell, update our padding stats
+ * and deliver the event. Otherwise just deliver the event.
+ */
+void
+circpad_deliver_sent_relay_cell_events(circuit_t *circ,
+                                       uint8_t relay_command)
+{
+  /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in
+   * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed
+   * or processed here).
+   *
+   * We do generate events for PADDING_NEGOTIATE and PADDING_NEGOTIATED cells.
+   */
+  if (relay_command == RELAY_COMMAND_DROP) {
+    /* Optimization: The event for RELAY_COMMAND_DROP is sent directly
+     * from circpad_send_padding_cell_for_callback(). This is to avoid
+     * putting a cell_t and a relay_header_t on the stack repeatedly
+     * if we decide to send a long train of padding cells back-to-back
+     * with 0 delay. So we do nothing here. */
+    return;
+  } else {
+    /* This is a non-padding cell sent from the client or from
+     * this node. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Initialize the states array for a circpad machine.
+ */
+void
+circpad_machine_states_init(circpad_machine_spec_t *machine,
+                            circpad_statenum_t num_states)
+{
+  if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) {
+    num_states = CIRCPAD_MAX_MACHINE_STATES;
+  }
+
+  machine->num_states = num_states;
+  machine->states = tor_malloc_zero(sizeof(circpad_state_t)*num_states);
+
+  /* Initialize the default next state for all events to
+   * "ignore" -- if events aren't specified, they are ignored. */
+  for (circpad_statenum_t s = 0; s < num_states; s++) {
+    for (int e = 0; e < CIRCPAD_NUM_EVENTS; e++) {
+      machine->states[s].next_state[e] = CIRCPAD_STATE_IGNORE;
+    }
+  }
+}
+
+static void
+circpad_setup_machine_on_circ(circuit_t *on_circ,
+                              const circpad_machine_spec_t *machine)
+{
+  if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up non-origin machine on origin circuit!");
+    return;
+  }
+
+  if (!CIRCUIT_IS_ORIGIN(on_circ) && machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up origin machine on non-origin circuit!");
+    return;
+  }
+
+  IF_BUG_ONCE(on_circ->padding_machine[machine->machine_index] != NULL) {
+    return;
+  }
+  IF_BUG_ONCE(on_circ->padding_info[machine->machine_index] != NULL) {
+    return;
+  }
+
+  /* Log message */
+  if (CIRCUIT_IS_ORIGIN(on_circ)) {
+    log_info(LD_CIRC, "Registering machine %s to origin circ %u (%d)",
+             machine->name,
+             TO_ORIGIN_CIRCUIT(on_circ)->global_identifier, on_circ->purpose);
+  } else {
+    log_info(LD_CIRC, "Registering machine %s to non-origin circ (%d)",
+             machine->name, on_circ->purpose);
+  }
+
+  on_circ->padding_info[machine->machine_index] =
+      circpad_circuit_machineinfo_new(on_circ, machine->machine_index);
+  on_circ->padding_machine[machine->machine_index] = machine;
+}
+
+/** Validate a single state of a padding machine */
+static bool
+padding_machine_state_is_valid(const circpad_state_t *state)
+{
+  int b;
+  uint32_t tokens_count = 0;
+  circpad_delay_t prev_bin_edge = 0;
+
+  /* We only validate histograms */
+  if (!state->histogram_len) {
+    return true;
+  }
+
+  /* We need at least two bins in a histogram */
+  if (state->histogram_len < 2) {
+    log_warn(LD_CIRC, "You can't have a histogram with less than 2 bins");
+    return false;
+  }
+
+  /* For each machine state, if it's a histogram, make sure all the
+   * histogram edges are well defined (i.e. are strictly monotonic). */
+  for (b = 0 ; b < state->histogram_len ; b++) {
+    /* Check that histogram edges are strictly increasing. Ignore the first
+     * edge since it can be zero. */
+    if (prev_bin_edge >= state->histogram_edges[b] && b > 0) {
+      log_warn(LD_CIRC, "Histogram edges are not increasing [%u/%u]",
+               prev_bin_edge, state->histogram_edges[b]);
+      return false;
+    }
+
+    prev_bin_edge = state->histogram_edges[b];
+
+    /* Also count the number of tokens as we go through the histogram states */
+    tokens_count += state->histogram[b];
+  }
+  /* Verify that the total number of tokens is correct */
+  if (tokens_count != state->histogram_total_tokens) {
+    log_warn(LD_CIRC, "Histogram token count is wrong [%u/%u]",
+             tokens_count, state->histogram_total_tokens);
+    return false;
+  }
+
+  return true;
+}
+
+/** Basic validation of padding machine */
+static bool
+padding_machine_is_valid(const circpad_machine_spec_t *machine)
+{
+  int i;
+
+  /* Validate the histograms of the padding machine */
+  for (i = 0 ; i < machine->num_states ; i++) {
+    if (!padding_machine_state_is_valid(&machine->states[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/* Validate and register <b>machine</b> into <b>machine_list</b>. If
+ * <b>machine_list</b> is NULL, then just validate. */
+void
+circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                 smartlist_t *machine_list)
+{
+  if (!padding_machine_is_valid(machine)) {
+    log_warn(LD_CIRC, "Machine #%u is invalid. Ignoring.",
+             machine->machine_num);
+    return;
+  }
+
+  if (machine_list) {
+    smartlist_add(machine_list, machine);
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+/* These padding machines are only used for tests pending #28634. */
+static void
+circpad_circ_client_machine_init(void)
+{
+  circpad_machine_spec_t *circ_client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  circ_client_machine->conditions.min_hops = 2;
+  circ_client_machine->conditions.state_mask =
+      CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circ_client_machine->conditions.reduced_padding_ok = 1;
+
+  circ_client_machine->target_hopnum = 2;
+  circ_client_machine->is_origin_side = 1;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_client_machine, 3);
+
+  circ_client_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* If we are in burst state, and we send a non-padding cell, then we cancel
+     the timer for the next padding cell:
+     We dont want to send fake extends when actual extends are going on */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1]= 1000000;
+
+  /* We have 5 tokens in the histogram, which means that all circuits will look
+   * like they have 7 hops (since we start this machine after the second hop,
+   * and tokens are decremented for any valid hops, and fake extends are
+   * used after that -- 2+5==7). */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5;
+
+  circ_client_machine->machine_num = smartlist_len(origin_padding_machines);
+  circpad_register_padding_machine(circ_client_machine,
+                                   origin_padding_machines);
+}
+
+static void
+circpad_circ_responder_machine_init(void)
+{
+  circpad_machine_spec_t *circ_responder_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Shut down the machine after we've sent enough packets */
+  circ_responder_machine->should_negotiate_end = 1;
+
+  /* The relay-side doesn't care what hopnum it is, but for consistency,
+   * let's match the client */
+  circ_responder_machine->target_hopnum = 2;
+  circ_responder_machine->is_origin_side = 0;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_responder_machine, 3);
+
+  /* This is the settings of the state machine. In the future we are gonna
+     serialize this into the consensus or the torrc */
+
+  /* We transition to the burst state on padding receive and on non-padding
+   * recieve */
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we _stay_ in the burst state when a non-padding
+   * is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we transition to the gap state when we receive a
+   * padding cell */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* These describe the padding charasteristics when in burst state */
+
+  /* use_rtt_estimate tries to estimate how long padding cells take to go from
+     C->M, and uses that as what as the base of the histogram */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+  /* The histogram is 2 bins: an empty one, and infinity */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1] =
+    1000000;
+  /* During burst state we wait forever for padding to arrive.
+
+     We are waiting for a padding cell from the client to come in, so that we
+     respond, and we immitate how extend looks like */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  // Only infinity bin:
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      histogram_total_tokens = 1;
+
+  /* From the gap state, we _stay_ in the gap state, when we receive padding
+   * or non padding */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* And from the gap state, we go to the end, when the bins are empty or a
+   * non-padding cell is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_END;
+
+  // FIXME: Tune this histogram
+
+  /* The gap state is the delay you wait after you receive a padding cell
+     before you send a padding response */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6;
+  /* Specify histogram bins */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[1]= 1000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[2]= 2000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[3]= 4000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[4]= 8000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[5]= 16000;
+  /* Specify histogram tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[3] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1;
+  /* Total number of tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6;
+
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC;
+
+  circ_responder_machine->machine_num = smartlist_len(relay_padding_machines);
+  circpad_register_padding_machine(circ_responder_machine,
+                                   relay_padding_machines);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize all of our padding machines.
+ *
+ * This is called at startup. It sets up some global machines, and then
+ * loads some from torrc, and from the tor consensus.
+ */
+void
+circpad_machines_init(void)
+{
+  tor_assert_nonfatal(origin_padding_machines == NULL);
+  tor_assert_nonfatal(relay_padding_machines == NULL);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  /* Register machines for hiding client-side intro circuits */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+
+  /* Register machines for hiding client-side rendezvous circuits */
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  // TODO: Parse machines from consensus and torrc
+#ifdef TOR_UNIT_TESTS
+  circpad_circ_client_machine_init();
+  circpad_circ_responder_machine_init();
+#endif
+}
+
+/**
+ * Free our padding machines
+ */
+void
+circpad_machines_free(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH(origin_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(origin_padding_machines);
+  }
+
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH(relay_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/**
+ * Check the Protover info to see if a node supports padding.
+ */
+static bool
+circpad_node_supports_padding(const node_t *node)
+{
+  if (node->rs) {
+    log_fn(LOG_INFO, LD_CIRC, "Checking padding: %s",
+           node->rs->pv.supports_hs_setup_padding ?
+              "supported" : "unsupported");
+    return node->rs->pv.supports_hs_setup_padding;
+  }
+
+  log_fn(LOG_INFO, LD_CIRC, "Empty routerstatus in padding check");
+  return 0;
+}
+
+/**
+ * Get a node_t for the nth hop in our circuit, starting from 1.
+ *
+ * Returns node_t from the consensus for that hop, if it is opened.
+ * Otherwise returns NULL.
+ */
+MOCK_IMPL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop))
+{
+  crypt_path_t *iter = circuit_get_cpath_hop(circ, hop);
+
+  if (!iter || iter->state != CPATH_STATE_OPEN)
+    return NULL;
+
+  return node_get_by_id(iter->extend_info->identity_digest);
+}
+
+/**
+ * Return true if a particular circuit supports padding
+ * at the desired hop.
+ */
+static bool
+circpad_circuit_supports_padding(origin_circuit_t *circ,
+                                 int target_hopnum)
+{
+  const node_t *hop;
+
+  if (!(hop = circuit_get_nth_node(circ, target_hopnum))) {
+    return 0;
+  }
+
+  return circpad_node_supports_padding(hop);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_negotiate_padding(origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command)
+{
+  circpad_negotiate_t type;
+  cell_t cell;
+  ssize_t len;
+
+  /* Check that the target hop lists support for padding in
+   * its ProtoVer fields */
+  if (!circpad_circuit_supports_padding(circ, target_hopnum)) {
+    return -1;
+  }
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiate_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiate_set_command(&type, command);
+  circpad_negotiate_set_version(&type, 0);
+  circpad_negotiate_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return -1;
+
+  log_fn(LOG_INFO,LD_CIRC,
+         "Negotiating padding on circuit %u (%d), command %d",
+         circ->global_identifier, TO_CIRCUIT(circ)->purpose, command);
+
+  return circpad_send_command_to_hop(circ, target_hopnum,
+                                     RELAY_COMMAND_PADDING_NEGOTIATE,
+                                     cell.payload, len);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns 1 if successful (or already set up), 0 otherwise.
+ */
+bool
+circpad_padding_negotiated(circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response)
+{
+  circpad_negotiated_t type;
+  cell_t cell;
+  ssize_t len;
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiated_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiated_set_command(&type, command);
+  circpad_negotiated_set_response(&type, response);
+  circpad_negotiated_set_version(&type, 0);
+  circpad_negotiated_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiated_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return 0;
+
+  /* Use relay_send because we're from the middle to the origin. We don't
+   * need to specify a target hop or layer_hint. */
+  return relay_send_command_from_edge(0, circ,
+                                      RELAY_COMMAND_PADDING_NEGOTIATED,
+                                      (void*)cell.payload,
+                                      (size_t)len, NULL) == 0;
+}
+
+/**
+ * Parse and react to a padding_negotiate cell.
+ *
+ * This is called at the middle node upon receipt of the client's choice of
+ * state machine, so that it can use the requested state machine index, if
+ * it is available.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell)
+{
+  int retval = 0;
+  circpad_negotiate_t *negotiate;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiate cell unsupported at origin (circuit %u)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATE cell; dropping.");
+    return -1;
+  }
+
+  if (negotiate->command == CIRCPAD_COMMAND_STOP) {
+    /* Free the machine corresponding to this machine type */
+    if (free_circ_machineinfos_with_machine_num(circ,
+                negotiate->machine_type)) {
+      log_info(LD_CIRC, "Received STOP command for machine %u",
+               negotiate->machine_type);
+      goto done;
+    }
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received circuit padding stop command for unknown machine.");
+    goto err;
+  } else if (negotiate->command == CIRCPAD_COMMAND_START) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            const circpad_machine_spec_t *, m) {
+      if (m->machine_num == negotiate->machine_type) {
+        circpad_setup_machine_on_circ(circ, m);
+        circpad_cell_event_nonpadding_received(circ);
+        goto done;
+      }
+    } SMARTLIST_FOREACH_END(m);
+  }
+
+  err:
+    retval = -1;
+
+  done:
+    circpad_padding_negotiated(circ, negotiate->machine_type,
+                   negotiate->command,
+                   (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR);
+    circpad_negotiate_free(negotiate);
+
+    return retval;
+}
+
+/**
+ * Parse and react to a padding_negotiated cell.
+ *
+ * This is called at the origin upon receipt of the middle's response
+ * to our choice of state machine.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell,
+                                  crypt_path_t *layer_hint)
+{
+  circpad_negotiated_t *negotiated;
+
+  if (!CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell unsupported at non-origin.");
+    return -1;
+  }
+
+  /* Verify this came from the expected hop */
+  if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell from wrong hop on circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATED cell on circuit %u; "
+          "dropping.", TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (negotiated->command == CIRCPAD_COMMAND_STOP) {
+    log_info(LD_CIRC,
+             "Received STOP command on PADDING_NEGOTIATED for circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    /* There may not be a padding_info here if we shut down the
+     * machine in circpad_shutdown_old_machines(). Or, if
+     * circpad_add_matching_matchines() added a new machine,
+     * there may be a padding_machine for a different machine num
+     * than this response. */
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+  } else if (negotiated->command == CIRCPAD_COMMAND_START &&
+             negotiated->response == CIRCPAD_RESPONSE_ERR) {
+    // This can happen due to consensus drift.. free the machines
+    // and be sad
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+    TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1;
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Middle node did not accept our padding request on circuit %u (%d)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier,
+           circ->purpose);
+  }
+
+  circpad_negotiated_free(negotiated);
+  return 0;
+}
+
+/** Free memory allocated by this machine spec. */
+STATIC void
+machine_spec_free_(circpad_machine_spec_t *m)
+{
+  if (!m) return;
+
+  tor_free(m->states);
+  tor_free(m);
+}
+
+/** Free all memory allocated by the circuitpadding subsystem. */
+void
+circpad_free_all(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(origin_padding_machines);
+  }
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/* Serialization */
+// TODO: Should we use keyword=value here? Are there helpers for that?
+#if 0
+static void
+circpad_state_serialize(const circpad_state_t *state,
+                        smartlist_t *chunks)
+{
+  smartlist_add_asprintf(chunks, " %u", state->histogram[0]);
+  for (int i = 1; i < state->histogram_len; i++) {
+    smartlist_add_asprintf(chunks, ",%u",
+                           state->histogram[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " 0x%x",
+                         state->transition_cancel_events);
+
+  for (int i = 0; i < CIRCPAD_NUM_STATES; i++) {
+    smartlist_add_asprintf(chunks, ",0x%x",
+                           state->transition_events[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " %u %u",
+                         state->use_rtt_estimate,
+                         state->token_removal);
+}
+
+char *
+circpad_machine_spec_to_string(const circpad_machine_spec_t *machine)
+{
+  smartlist_t *chunks = smartlist_new();
+  char *out;
+  (void)machine;
+
+  circpad_state_serialize(&machine->start, chunks);
+  circpad_state_serialize(&machine->gap, chunks);
+  circpad_state_serialize(&machine->burst, chunks);
+
+  out = smartlist_join_strings(chunks, "", 0, NULL);
+
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+  return out;
+}
+
+// XXX: Writeme
+const circpad_machine_spec_t *
+circpad_string_to_machine(const char *str)
+{
+  (void)str;
+  return NULL;
+}
+
+#endif /* 0 */
diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h
new file mode 100644
index 0000000000..74b69a1c7a
--- /dev/null
+++ b/src/core/or/circuitpadding.h
@@ -0,0 +1,813 @@
+/*
+ * Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.h
+ * \brief Header file for circuitpadding.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_H
+#define TOR_CIRCUITPADDING_H
+
+#include "trunnel/circpad_negotiation.h"
+#include "lib/evloop/timers.h"
+
+struct circuit_t;
+struct origin_circuit_t;
+struct cell_t;
+
+/**
+ * Signed error return with the specific property that negative
+ * values mean error codes of various semantics, 0 means success,
+ * and positive values are unused.
+ *
+ * XXX: Tor uses this concept a lot but just calls it int. Should we move
+ * this somewhere centralized? Where?
+ */
+typedef int signed_error_t;
+
+/**
+ * These constants specify the types of events that can cause
+ * transitions between state machine states.
+ *
+ * Note that SENT and RECV are relative to this endpoint. For
+ * relays, SENT means packets destined towards the client and
+ * RECV means packets destined towards the relay. On the client,
+ * SENT means packets destined towards the relay, where as RECV
+ * means packets destined towards the client.
+ */
+typedef enum {
+  /* A non-padding cell was received. */
+  CIRCPAD_EVENT_NONPADDING_RECV = 0,
+  /* A non-padding cell was sent. */
+  CIRCPAD_EVENT_NONPADDING_SENT = 1,
+  /* A padding cell (RELAY_COMMAND_DROP) was sent. */
+  CIRCPAD_EVENT_PADDING_SENT = 2,
+  /* A padding cell was received. */
+  CIRCPAD_EVENT_PADDING_RECV = 3,
+  /* We tried to schedule padding but we ended up picking the infinity bin
+   * which means that padding was delayed infinitely */
+  CIRCPAD_EVENT_INFINITY = 4,
+  /* All histogram bins are empty (we are out of tokens) */
+  CIRCPAD_EVENT_BINS_EMPTY = 5,
+  /* This state has used up its cell count */
+  CIRCPAD_EVENT_LENGTH_COUNT = 6
+} circpad_event_t;
+#define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1)
+
+/** Boolean type that says if we decided to transition states or not */
+typedef enum {
+  CIRCPAD_STATE_UNCHANGED = 0,
+  CIRCPAD_STATE_CHANGED = 1
+} circpad_decision_t;
+
+/** The type for the things in histogram bins (aka tokens) */
+typedef uint32_t circpad_hist_token_t;
+
+/** The type for histogram indexes (needs to be negative for errors) */
+typedef int8_t circpad_hist_index_t;
+
+/** The type for absolute time, from monotime_absolute_usec() */
+typedef uint64_t circpad_time_t;
+
+/** The type for timer delays, in microseconds */
+typedef uint32_t circpad_delay_t;
+#define CIRCPAD_DELAY_UNITS_PER_SECOND  (1000*1000)
+
+/**
+ * An infinite padding cell delay means don't schedule any padding --
+ * simply wait until a different event triggers a transition.
+ *
+ * This means that the maximum delay we can schedule is UINT32_MAX-1
+ * microseconds, or about 4300 seconds (1.25 hours).
+ * XXX: Is this enough if we want to simulate light, intermittent
+ * activity on an onion service?
+ */
+#define CIRCPAD_DELAY_INFINITE  (UINT32_MAX)
+
+/**
+ * This is the maximum delay that the circuit padding system can have, in
+ * seconds.
+ */
+#define CIRCPAD_DELAY_MAX_SECS   \
+    ((CIRCPAD_DELAY_INFINITE/CIRCPAD_DELAY_UNITS_PER_SECOND)+1)
+
+/**
+ * Macro to clarify when we're checking the infinity bin.
+ *
+ * Works with either circpad_state_t or circpad_machine_runtime_t
+ */
+#define CIRCPAD_INFINITY_BIN(mi)  ((mi)->histogram_len-1)
+
+/**
+ * These constants form a bitfield that specifies when a state machine
+ * should be applied to a circuit.
+ *
+ * If any of these elements is set, then the circuit will be tested against
+ * that specific condition. If an element is unset, then we don't test it.
+ * (E.g., if neither NO_STREAMS or STREAMS are set, then we will not care
+ * whether a circuit has streams attached when we apply a state machine.)
+ *
+ * The helper function circpad_circuit_state() converts circuit state
+ * flags into this more compact representation.
+ */
+typedef enum {
+  /* Only apply machine if the circuit is still building */
+  CIRCPAD_CIRC_BUILDING = 1<<0,
+  /* Only apply machine if the circuit is open */
+  CIRCPAD_CIRC_OPENED = 1<<1,
+  /* Only apply machine if the circuit has no attached streams */
+  CIRCPAD_CIRC_NO_STREAMS = 1<<2,
+  /* Only apply machine if the circuit has attached streams */
+  CIRCPAD_CIRC_STREAMS = 1<<3,
+  /* Only apply machine if the circuit still allows RELAY_EARLY cells */
+  CIRCPAD_CIRC_HAS_RELAY_EARLY = 1<<4,
+  /* Only apply machine if the circuit has depleted its RELAY_EARLY cells
+   * allowance. */
+  CIRCPAD_CIRC_HAS_NO_RELAY_EARLY = 1<<5
+} circpad_circuit_state_t;
+
+/** Bitmask that says "apply this machine to all states" */
+#define CIRCPAD_STATE_ALL   \
+    (CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED| \
+     CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_NO_STREAMS| \
+     CIRCPAD_CIRC_HAS_RELAY_EARLY|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY)
+
+/**
+ * A compact circuit purpose bitfield mask that allows us to compactly
+ * specify which circuit purposes a machine should apply to.
+ *
+ * The helper function circpad_circ_purpose_to_mask() converts circuit
+ * purposes into bit positions in this bitmask.
+ */
+typedef uint32_t circpad_purpose_mask_t;
+
+/** Bitmask that says "apply this machine to all purposes". */
+#define CIRCPAD_PURPOSE_ALL (0xFFFFFFFF)
+
+/**
+ * This type specifies all of the conditions that must be met before
+ * a client decides to initiate padding on a circuit.
+ *
+ * A circuit must satisfy every sub-field in this type in order
+ * to be considered to match the conditions.
+ */
+typedef struct circpad_machine_conditions_t {
+  /** Only apply the machine *if* the circuit has at least this many hops */
+  unsigned min_hops : 3;
+
+  /** Only apply the machine *if* vanguards are enabled */
+  unsigned requires_vanguards : 1;
+
+  /**
+   * This machine is ok to use if reduced padding is set in consensus
+   * or torrc. This machine will still be applied even if reduced padding
+   * is not set; this flag only acts to exclude machines that don't have
+   * it set when reduced padding is requested. Therefore, reduced padding
+   * machines should appear at the lowest priority in the padding machine
+   * lists (aka first in the list), so that non-reduced padding machines
+   * for the same purpose are given a chance to apply when reduced padding
+   * is not requested. */
+  unsigned reduced_padding_ok : 1;
+
+  /** Only apply the machine *if* the circuit's state matches any of
+   *  the bits set in this bitmask. */
+  circpad_circuit_state_t state_mask;
+
+  /** Only apply a machine *if* the circuit's purpose matches one
+   *  of the bits set in this bitmask */
+  circpad_purpose_mask_t purpose_mask;
+
+} circpad_machine_conditions_t;
+
+/**
+ * Token removal strategy options.
+ *
+ * The WTF-PAD histograms are meant to specify a target distribution to shape
+ * traffic towards. This is accomplished by removing tokens from the histogram
+ * when either padding or non-padding cells are sent.
+ *
+ * When we see a non-padding cell at a particular time since the last cell, you
+ * remove a token from the corresponding delay bin. These flags specify
+ * which bin to choose if that bin is already empty.
+ */
+typedef enum {
+  /** Don't remove any tokens */
+  CIRCPAD_TOKEN_REMOVAL_NONE = 0,
+  /**
+   * Remove from the first non-zero higher bin index when current is zero.
+   * This is the recommended strategy from the Adaptive Padding paper. */
+  CIRCPAD_TOKEN_REMOVAL_HIGHER = 1,
+  /** Remove from the first non-zero lower bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_LOWER = 2,
+  /** Remove from the closest non-zero bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST = 3,
+  /** Remove from the closest bin by time value (since bins are
+   *  exponentially spaced). */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC = 4,
+  /** Only remove from the exact bin corresponding to this delay. If
+   *  the bin is 0, simply do nothing. Don't pick another bin. */
+  CIRCPAD_TOKEN_REMOVAL_EXACT = 5
+} circpad_removal_t;
+
+/**
+ * Distribution types supported by circpad_distribution_sample().
+ *
+ * These can be used instead of histograms for the inter-packet
+ * timing distribution, or to specify a distribution on the number
+ * of cells that can be sent while in a specific state of the state
+ * machine.
+ *
+ * Each distribution takes up to two parameters which are described below. */
+typedef enum {
+  /* No probability distribution is used */
+  CIRCPAD_DIST_NONE = 0,
+  /* Uniform distribution: param1 is lower bound and param2 is upper bound */
+  CIRCPAD_DIST_UNIFORM = 1,
+  /* Logistic distribution: param1 is Mu, param2 is sigma. */
+  CIRCPAD_DIST_LOGISTIC = 2,
+  /* Log-logistic distribution: param1 is Alpha, param2 is 1.0/Beta */
+  CIRCPAD_DIST_LOG_LOGISTIC = 3,
+  /* Geometric distribution: param1 is 'p' (success probability) */
+  CIRCPAD_DIST_GEOMETRIC = 4,
+  /* Weibull distribution: param1 is k, param2 is Lambda */
+  CIRCPAD_DIST_WEIBULL = 5,
+  /* Generalized Pareto distribution: param1 is sigma, param2 is xi */
+  CIRCPAD_DIST_PARETO = 6
+} circpad_distribution_type_t;
+
+/**
+ * Distribution information.
+ *
+ * This type specifies a specific distribution above, as well as
+ * up to two parameters for that distribution. The specific
+ * per-distribution meaning of these parameters is specified
+ * in circpad_distribution_sample().
+ */
+typedef struct circpad_distribution_t {
+  circpad_distribution_type_t type;
+  double param1;
+  double param2;
+} circpad_distribution_t;
+
+/** State number type. Represents current state of state machine. */
+typedef uint16_t circpad_statenum_t;
+#define  CIRCPAD_STATENUM_MAX   (UINT16_MAX)
+
+/** A histogram can be used to sample padding delays given a machine state.
+ * This constant defines the maximum histogram width (i.e. the max number of
+ * bins).
+ *
+ * The current limit is arbitrary and could be raised if there is a need,
+ * however too many bins will be hard to serialize in the future.
+ *
+ * Memory concerns are not so great here since the corresponding histogram and
+ * histogram_edges arrays are global and not per-circuit.
+ *
+ * If we ever upgrade this to a value that can't be represented by 8-bits we
+ * also need to upgrade circpad_hist_index_t.
+ */
+#define CIRCPAD_MAX_HISTOGRAM_LEN (100)
+
+/**
+ * A state of a padding state machine. The information here are immutable and
+ * represent the initial form of the state; it does not get updated as things
+ * happen. The mutable information that gets updated in runtime are carried in
+ * a circpad_machine_runtime_t.
+ *
+ * This struct describes the histograms and/or probability distributions, as
+ * well as parameters of a single state in the adaptive padding machine.
+ * Instances of this struct exist in global circpad machine definitions that
+ * come from torrc or the consensus.
+ */
+typedef struct circpad_state_t {
+  /**
+   * If a histogram is used for this state, this specifies the number of bins
+   * of this histogram. Histograms must have at least 2 bins.
+   *
+   * In particular, the following histogram:
+   *
+   * Tokens
+   *         +
+   *      10 |    +----+
+   *       9 |    |    |           +---------+
+   *       8 |    |    |           |         |
+   *       7 |    |    |     +-----+         |
+   *       6 +----+ Bin+-----+     |         +---------------+
+   *       5 |    | #1 |     |     |         |               |
+   *         | Bin|    | Bin | Bin |  Bin #4 |    Bin #5     |
+   *         | #0 |    | #2  | #3  |         | (infinity bin)|
+   *         |    |    |     |     |         |               |
+   *         |    |    |     |     |         |               |
+   *       0 +----+----+-----+-----+---------+---------------+
+   *         0   100  200   350   500      1000             inf  microseconds
+   *
+   * would be specified the following way:
+   *    histogram_len = 6;
+   *    histogram[] =        {   6,  10,   6,  7,    9,     6 }
+   *    histogram_edges[] =  { 0, 100, 200, 350, 500, 1000 }
+   *
+   * The final bin is called the "infinity bin" and if it's chosen we don't
+   * schedule any padding. The infinity bin is strange because its lower edge
+   * is the max value of possible non-infinite delay allowed by this histogram,
+   * and its upper edge is CIRCPAD_DELAY_INFINITE. You can tell if the infinity
+   * bin is chosen by inspecting its bin index or inspecting its upper edge.
+   *
+   * If a delay probability distribution is used for this state, this is set
+   * to 0. */
+  circpad_hist_index_t histogram_len;
+  /** The histogram itself: an array of uint16s of tokens, whose
+   *  widths are exponentially spaced, in microseconds.
+   *
+   *  This array must have histogram_len elements that are strictly
+   *  monotonically increasing. */
+  circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN];
+  /* The histogram bin edges in usec.
+   *
+   * Each element of this array specifies the left edge of the corresponding
+   * bin. The rightmost edge is always infinity and is not specified in this
+   * array.
+   *
+   * This array must have histogram_len elements. */
+  circpad_delay_t histogram_edges[CIRCPAD_MAX_HISTOGRAM_LEN+1];
+  /** Total number of tokens in this histogram. This is a constant and is *not*
+   *  decremented every time we spend a token. It's used for initializing and
+   *  refilling the histogram. */
+  uint32_t histogram_total_tokens;
+
+  /**
+   * Represents a delay probability distribution (aka IAT distribution). It's a
+   * parametrized way of encoding inter-packet delay information in
+   * microseconds. It can be used instead of histograms.
+   *
+   * If it is used, token_removal below must be set to
+   * CIRCPAD_TOKEN_REMOVAL_NONE.
+   *
+   * Start_usec, range_sec, and rtt_estimates are still applied to the
+   * results of sampling from this distribution (range_sec is used as a max).
+   */
+  circpad_distribution_t iat_dist;
+  /*  If a delay probability distribution is used, this is used as the max
+   *  value we can sample from the distribution. However, RTT measurements and
+   *  dist_added_shift gets applied on top of this value to derive the final
+   *  padding delay. */
+  circpad_delay_t dist_max_sample_usec;
+  /*  If a delay probability distribution is used and this is set, we will add
+   *  this value on top of the value sampled from the IAT distribution to
+   *  derive the final padding delay (We also add the RTT measurement if it's
+   *  enabled.). */
+  circpad_delay_t dist_added_shift_usec;
+
+  /**
+   * The length dist is a parameterized way of encoding how long this
+   * state machine runs in terms of sent padding cells or all
+   * sent cells. Values are sampled from this distribution, clamped
+   * to max_len, and then start_len is added to that value.
+   *
+   * It may be specified instead of or in addition to
+   * the infinity bins and bins empty conditions. */
+  circpad_distribution_t length_dist;
+  /** A minimum length value, added to the output of length_dist */
+  uint16_t start_length;
+  /** A cap on the length value that can be sampled from the length_dist */
+  uint64_t max_length;
+
+  /** Should we decrement length when we see a nonpadding packet?
+   * XXX: Are there any machines that actually want to set this to 0? There may
+   * not be. OTOH, it's only a bit.. */
+  unsigned length_includes_nonpadding : 1;
+
+  /**
+   * This is an array that specifies the next state to transition to upon
+   * receipt an event matching the indicated array index.
+   *
+   * This aborts our scheduled packet and switches to the state
+   * corresponding to the index of the array. Tokens are filled upon
+   * this transition.
+   *
+   * States are allowed to transition to themselves, which means re-schedule
+   * a new padding timer. They are also allowed to temporarily "transition"
+   * to the "IGNORE" and "CANCEL" pseudo-states. See defines below
+   * for details on state behavior and meaning.
+   */
+  circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS];
+
+  /**
+   * If true, estimate the RTT from this relay to the exit/website and add that
+   * to start_usec for use as the histogram bin 0 start delay.
+   *
+   * Right now this is only supported for relay-side state machines.
+   */
+  unsigned use_rtt_estimate : 1;
+
+  /** This specifies the token removal strategy to use upon padding and
+   *  non-padding activity. */
+  circpad_removal_t token_removal;
+} circpad_state_t;
+
+/**
+ * The start state for this machine.
+ *
+ * In the original WTF-PAD, this is only used for transition to/from
+ * the burst state. All other fields are not used. But to simplify the
+ * code we've made it a first-class state. This has no performance
+ * consequences, but may make naive serialization of the state machine
+ * large, if we're not careful about how we represent empty fields.
+ */
+#define  CIRCPAD_STATE_START       0
+
+/**
+ * The burst state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD
+ * (https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf and
+ * https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/), the burst
+ * state serves to detect bursts in traffic. This is done by using longer
+ * delays in its histogram, which represent the expected delays between
+ * bursts of packets in the target stream. If this delay expires without a
+ * real packet being sent, the burst state sends a padding packet and then
+ * immediately transitions to the gap state, which is used to generate
+ * a synthetic padding packet train. In this implementation, this transition
+ * needs to be explicitly specified in the burst state's transition events.
+ *
+ * Because of this flexibility, other padding mechanisms can transition
+ * between these two states arbitrarily, to encode other dynamics of
+ * target traffic.
+ */
+#define  CIRCPAD_STATE_BURST       1
+
+/**
+ * The gap state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD, the gap
+ * state serves to simulate an artificial packet train composed of padding
+ * packets. It does this by specifying much lower inter-packet delays than
+ * the burst state, and transitioning back to itself after padding is sent
+ * if these timers expire before real traffic is sent. If real traffic is
+ * sent, it transitions back to the burst state.
+ *
+ * Again, in this implementation, these transitions must be specified
+ * explicitly, and other transitions are also permitted.
+ */
+#define  CIRCPAD_STATE_GAP         2
+
+/**
+ * End is a pseudo-state that causes the machine to go completely
+ * idle, and optionally get torn down (depending on the
+ * value of circpad_machine_spec_t.should_negotiate_end)
+ *
+ * End MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_END         CIRCPAD_STATENUM_MAX
+
+/**
+ * "Ignore" is a pseudo-state that means "do not react to this
+ * event".
+ *
+ * "Ignore" MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_IGNORE         (CIRCPAD_STATENUM_MAX-1)
+
+/**
+ * "Cancel" is a pseudo-state that means "cancel pending timers,
+ * but remain in your current state".
+ *
+ * Cancel MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_CANCEL         (CIRCPAD_STATENUM_MAX-2)
+
+/**
+ * Since we have 3 pseudo-states, the max state array length is
+ * up to one less than cancel's statenum.
+ */
+#define CIRCPAD_MAX_MACHINE_STATES  (CIRCPAD_STATE_CANCEL-1)
+
+/**
+ * Mutable padding machine info.
+ *
+ * This structure contains mutable information about a padding
+ * machine. The mutable information must be kept separate because
+ * it exists per-circuit, where as the machines themselves are global.
+ * This separation is done to conserve space in the circuit structure.
+ *
+ * This is the per-circuit state that changes regarding the global state
+ * machine. Some parts of it are optional (ie NULL).
+ *
+ * XXX: Play with layout to minimize space on x64 Linux (most common relay).
+ */
+typedef struct circpad_machine_runtime_t {
+  /** The callback pointer for the padding callbacks.
+   *
+   *  These timers stick around the machineinfo until the machineinfo's circuit
+   *  is closed, at which point the timer is cancelled. For this reason it's
+   *  safe to assume that the machineinfo exists if this timer gets
+   *  triggered. */
+  tor_timer_t *padding_timer;
+
+  /** The circuit for this machine */
+  struct circuit_t *on_circ;
+
+  /** A mutable copy of the histogram for the current state.
+   *  NULL if remove_tokens is false for that state */
+  circpad_hist_token_t *histogram;
+  /** Length of the above histogram.
+   * XXX: This field *could* be removed at the expense of added
+   * complexity+overhead for reaching back into the immutable machine
+   * state every time we need to inspect the histogram. It's only a byte,
+   * though, so it seemed worth it.
+   */
+  circpad_hist_index_t histogram_len;
+  /** Remove token from this index upon sending padding */
+  circpad_hist_index_t chosen_bin;
+
+  /** Stop padding/transition if this many cells sent */
+  uint64_t state_length;
+#define CIRCPAD_STATE_LENGTH_INFINITE UINT64_MAX
+
+  /** A scaled count of padding packets sent, used to limit padding overhead.
+   * When this reaches UINT16_MAX, we cut it and nonpadding_sent in half. */
+  uint16_t padding_sent;
+  /** A scaled count of non-padding packets sent, used to limit padding
+   *  overhead. When this reaches UINT16_MAX, we cut it and padding_sent in
+   *  half. */
+  uint16_t nonpadding_sent;
+
+  /**
+   * Timestamp of the most recent cell event (sent, received, padding,
+   * non-padding), in seconds from approx_time().
+   *
+   * Used as an emergency break to stop holding padding circuits open.
+   */
+  time_t last_cell_time_sec;
+
+  /**
+   * EWMA estimate of the RTT of the circuit from this hop
+   * to the exit end, in microseconds. */
+  circpad_delay_t rtt_estimate_usec;
+
+  /**
+   * The last time we got an event relevant to estimating
+   * the RTT. Monotonic time in microseconds since system
+   * start.
+   */
+  circpad_time_t last_received_time_usec;
+
+  /**
+   * The time at which we scheduled a non-padding packet,
+   * or selected an infinite delay.
+   *
+   * Monotonic time in microseconds since system start.
+   * This is 0 if we haven't chosen a padding delay.
+   */
+  circpad_time_t padding_scheduled_at_usec;
+
+  /** What state is this machine in? */
+  circpad_statenum_t current_state;
+
+  /**
+   * True if we have scheduled a timer for padding.
+   *
+   * This is 1 if a timer is pending. It is 0 if
+   * no timer is scheduled. (It can be 0 even when
+   * padding_was_scheduled_at_usec is non-zero).
+   */
+  unsigned is_padding_timer_scheduled : 1;
+
+  /**
+   * If this is true, we have seen full duplex behavior.
+   * Stop updating the RTT.
+   */
+  unsigned stop_rtt_update : 1;
+
+/** Max number of padding machines on each circuit. If changed,
+ * also ensure the machine_index bitwith supports the new size. */
+#define CIRCPAD_MAX_MACHINES    (2)
+  /** Which padding machine index was this for.
+   * (make sure changes to the bitwidth can support the
+   * CIRCPAD_MAX_MACHINES define). */
+  unsigned machine_index : 1;
+
+} circpad_machine_runtime_t;
+
+/** Helper macro to get an actual state machine from a machineinfo */
+#define CIRCPAD_GET_MACHINE(machineinfo) \
+    ((machineinfo)->on_circ->padding_machine[(machineinfo)->machine_index])
+
+/**
+ * This specifies a particular padding machine to use after negotiation.
+ *
+ * The constants for machine_num_t are in trunnel.
+ * We want to be able to define extra numbers in the consensus/torrc, though.
+ */
+typedef uint8_t circpad_machine_num_t;
+
+/** Global state machine structure from the consensus */
+typedef struct circpad_machine_spec_t {
+  /* Just a user-friendly machine name for logs */
+  const char *name;
+
+  /** Global machine number */
+  circpad_machine_num_t machine_num;
+
+  /** Which machine index slot should this machine go into in
+   *  the array on the circuit_t */
+  unsigned machine_index : 1;
+
+  /** Send a padding negotiate to shut down machine at end state? */
+  unsigned should_negotiate_end : 1;
+
+  // These next three fields are origin machine-only...
+  /** Origin side or relay side */
+  unsigned is_origin_side : 1;
+
+  /** Which hop in the circuit should we send padding to/from?
+   *  1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */
+  unsigned target_hopnum : 3;
+
+  /** If this flag is enabled, don't close circuits that use this machine even
+   *  if another part of Tor wants to close this circuit.
+   *
+   *  If this flag is set, the circuitpadding subsystem will close circuits the
+   *  moment the machine transitions to the END state, and only if the circuit
+   *  has already been asked to be closed by another part of Tor.
+   *
+   *  Circuits that should have been closed but were kept open by a padding
+   *  machine are re-purposed to CIRCUIT_PURPOSE_C_CIRCUIT_PADDING, hence
+   *  machines should take that purpose into account if they are filtering
+   *  circuits by purpose. */
+  unsigned manage_circ_lifetime : 1;
+
+  /** This machine only kills fascists if the following conditions are met. */
+  circpad_machine_conditions_t conditions;
+
+  /** How many padding cells can be sent before we apply overhead limits?
+   * XXX: Note that we can only allow up to 64k of padding cells on an
+   * otherwise quiet circuit. Is this enough? It's 33MB. */
+  uint16_t allowed_padding_count;
+
+  /** Padding percent cap: Stop padding if we exceed this percent overhead.
+   * 0 means no limit. Overhead is defined as percent of total traffic, so
+   * that we can use 0..100 here. This is the same definition as used in
+   * Prop#265. */
+  uint8_t max_padding_percent;
+
+  /** State array: indexed by circpad_statenum_t */
+  circpad_state_t *states;
+
+  /**
+   * Number of states this machine has (ie: length of the states array).
+   * XXX: This field is not needed other than for safety. */
+  circpad_statenum_t num_states;
+} circpad_machine_spec_t;
+
+void circpad_new_consensus_params(const networkstatus_t *ns);
+
+int circpad_marked_circuit_for_padding(circuit_t *circ, int reason);
+
+/**
+ * The following are event call-in points that are of interest to
+ * the state machines. They are called during cell processing. */
+void circpad_deliver_unrecognized_cell_events(struct circuit_t *circ,
+                                              cell_direction_t dir);
+void circpad_deliver_sent_relay_cell_events(struct circuit_t *circ,
+                                            uint8_t relay_command);
+void circpad_deliver_recognized_relay_cell_events(struct circuit_t *circ,
+                                                  uint8_t relay_command,
+                                                  crypt_path_t *layer_hint);
+
+/** Cell events are delivered by the above delivery functions */
+void circpad_cell_event_nonpadding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_nonpadding_received(struct circuit_t *on_circ);
+void circpad_cell_event_padding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_padding_received(struct circuit_t *on_circ);
+
+/** Internal events are events the machines send to themselves */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi);
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *);
+circpad_decision_t circpad_internal_event_state_length_up(
+                                  circpad_machine_runtime_t *);
+
+/** Machine creation events are events that cause us to set up or
+ *  tear down padding state machines. */
+void circpad_machine_event_circ_added_hop(struct origin_circuit_t *on_circ);
+void circpad_machine_event_circ_built(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_purpose_changed(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_streams(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_no_streams(struct origin_circuit_t *circ);
+void
+circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ);
+
+void circpad_machines_init(void);
+void circpad_machines_free(void);
+void circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                      smartlist_t *machine_list);
+
+void circpad_machine_states_init(circpad_machine_spec_t *machine,
+                                 circpad_statenum_t num_states);
+
+void circpad_circuit_free_all_machineinfos(struct circuit_t *circ);
+
+bool circpad_padding_is_from_expected_hop(struct circuit_t *circ,
+                                         crypt_path_t *from_hop);
+
+/** Serializaton functions for writing to/from torrc and consensus */
+char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine);
+const circpad_machine_spec_t *circpad_string_to_machine(const char *str);
+
+/* Padding negotiation between client and middle */
+signed_error_t circpad_handle_padding_negotiate(struct circuit_t *circ,
+                                      struct cell_t *cell);
+signed_error_t circpad_handle_padding_negotiated(struct circuit_t *circ,
+                                      struct cell_t *cell,
+                                      crypt_path_t *layer_hint);
+signed_error_t circpad_negotiate_padding(struct origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command);
+bool circpad_padding_negotiated(struct circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response);
+
+circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose);
+
+int circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                                crypt_path_t *layer_hint,
+                                const relay_header_t *rh);
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *));
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_spec_transition, (circpad_machine_runtime_t *mi,
+                             circpad_event_t event));
+
+circpad_decision_t circpad_send_padding_cell_for_callback(
+                                 circpad_machine_runtime_t *mi);
+
+void circpad_free_all(void);
+
+#ifdef CIRCUITPADDING_PRIVATE
+STATIC void  machine_spec_free_(circpad_machine_spec_t *m);
+#define machine_spec_free(chan) \
+  FREE_AND_NULL(circpad_machine_spec_t,machine_spec_free_, (m))
+
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi);
+
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi);
+
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi);
+
+STATIC void circpad_machine_remove_token(circpad_machine_runtime_t *mi);
+
+STATIC circpad_hist_index_t circpad_histogram_usec_to_bin(
+                                       const circpad_machine_runtime_t *mi,
+                                       circpad_delay_t us);
+
+STATIC circpad_machine_runtime_t *circpad_circuit_machineinfo_new(
+                                               struct circuit_t *on_circ,
+                                               int machine_index);
+STATIC void circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us,
+                                         bool use_usec);
+STATIC void circpad_machine_setup_tokens(circpad_machine_runtime_t *mi);
+
+MOCK_DECL(STATIC signed_error_t,
+circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len));
+
+MOCK_DECL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop));
+
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl);
+
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *origin_padding_machines;
+extern smartlist_t *relay_padding_machines;
+
+#endif
+
+#endif /* defined(CIRCUITPADDING_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_H) */
diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c
new file mode 100644
index 0000000000..98767f9e8f
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.c
@@ -0,0 +1,454 @@
+/* Copyright (c) 2019 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.c
+ * \brief Circuit padding state machines
+ *
+ * Introduce circuit padding machines that will be used by Tor circuits, as
+ * specified by proposal 302 "Hiding onion service clients using padding".
+ *
+ * Right now this file introduces two machines that aim to hide the client-side
+ * of onion service circuits against naive classifiers like the ones from the
+ * "Circuit Fingerprinting Attacks: Passive Deanonymization of Tor Hidden
+ * Services" paper from USENIX. By naive classifiers we mean classifiers that
+ * use basic features like "circuit construction circuits" and "incoming and
+ * outgoing cell counts" and "duration of activity".
+ *
+ * In particular, these machines aim to be lightweight and protect against
+ * these basic classifiers. They don't aim to protect against more advanced
+ * attacks that use deep learning or even correlate various circuit
+ * construction events together. Machines that fool such advanced classifiers
+ * are also possible, but they can't be so lightweight and might require more
+ * WTF-PAD features. So for now we opt for the following two machines:
+ *
+ * Client-side introduction circuit hiding machine:
+ *
+ *    This machine hides client-side introduction circuits by making their
+ *    circuit consruction sequence look like normal general circuits that
+ *    download directory information. Furthermore, the circuits are kept open
+ *    until all the padding has been sent, since intro circuits are usually
+ *    very short lived and this act as a distinguisher. For more info see
+ *    circpad_machine_client_hide_intro_circuits() and the sec.
+ *
+ * Client-side rendezvous circuit hiding machine:
+ *
+ *    This machine hides client-side rendezvous circuits by making their
+ *    circuit construction sequence look like normal general circuits. For more
+ *    details see circpad_machine_client_hide_rend_circuits() and the spec.
+ *
+ * TODO: These are simple machines that carefully manipulate the cells of the
+ *   initial circuit setup procedure to make them look like general
+ *   circuits. In the future, more states can be baked into their state machine
+ *   to do more advanced obfuscation.
+ **/
+
+#define CIRCUITPADDING_MACHINES_PRIVATE
+
+#include "core/or/or.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/circuitlist.h"
+
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitpadding.h"
+
+/** Create a client-side padding machine that aims to hide IP circuits. In
+ *  particular, it keeps intro circuits alive until a bunch of fake traffic has
+ *  been pushed through.
+ */
+void
+circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_ip_circ";
+
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad introduction circuits, and we want to start padding
+   * only after the INTRODUCE1 cell has been sent, so set the purposes
+   * appropriately.
+   *
+   * In particular we want introduction circuits to blend as much as possible
+   * with general circuits. Most general circuits have the following initial
+   * relay cell sequence (outgoing cells marked in [brackets]):
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * Whereas normal introduction circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2
+   *   -> [INTRO1] -> INTRODUCE_ACK
+   *
+   * This means that up to the sixth cell (first line of each sequence above),
+   * both general and intro circuits have identical cell sequences. After that
+   * we want to mimic the second line sequence of
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * We achieve this by starting padding INTRODUCE1 has been sent. With padding
+   * negotiation cells, in the common case of the second line looks like:
+   *   -> [INTRO1] -> [PADDING_NEGOTIATE] -> PADDING_NEGOTIATED -> INTRO_ACK
+   *
+   * Then, the middle node will send between INTRO_MACHINE_MINIMUM_PADDING and
+   * INTRO_MACHINE_MAXIMUM_PADDING cells, to match the "...(inbound data cells
+   * continue)" portion of the trace (aka the rest of an HTTPS response body).
+   */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Keep the circuit alive even after the introduction has been finished,
+   * otherwise the short-term lifetime of the circuit will blow our cover */
+  client_machine->manage_circ_lifetime = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* For the origin-side machine, we transition to OBFUSCATE_CIRC_SETUP after
+   * sending PADDING_NEGOTIATE, and we stay there (without sending any padding)
+   * until we receive a STOP from the other side. */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* origin-side machine has no event reactions while in
+   * CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP, so no more state transitions here. */
+
+  /* The client side should never send padding, so it does not need
+   * to specify token removal, or a histogram definition or state lengths.
+   * That is all controlled by the middle node. */
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client intro point hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits. See
+ *  comments on the function above for more details on the workings of the
+ *  machine. */
+void
+circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_ip_circ";
+
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* We want to negotiate END from this side after all our padding is done, so
+   * that the origin-side machine goes into END state, and eventually closes
+   * the circuit. */
+  relay_machine->should_negotiate_end = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* For the relay-side machine, we want to transition
+   * START -> OBFUSCATE_CIRC_SETUP upon first non-padding
+   * cell sent (PADDING_NEGOTIATED in this case).  */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* For the relay-side, we want to transition from OBFUSCATE_CIRC_SETUP to END
+   * state when the length finishes. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Now let's define the OBF -> OBF transitions that maintain our padding
+   * flow:
+   *
+   * For the relay-side machine, we want to keep on sending padding bytes even
+   * when nothing else happens on this circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_PADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+  /* For the relay-side machine, we need this transition so that we re-enter
+     the state, after PADDING_NEGOTIATED is sent. Otherwise, the remove token
+     function will disable the timer, and nothing will restart it since there
+     is no other motion on an intro circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* Token removal strategy for OBFUSCATE_CIRC_SETUP state: Don't
+   * remove any tokens.
+   *
+   * We rely on the state length sampling and not token removal, to avoid
+   * the mallocs required to copy the histograms for token removal,
+   * and to avoid monotime calls needed to determine histogram
+   * bins for token removal. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Figure out the length of the OBFUSCATE_CIRC_SETUP state so that it's
+   * randomized. The relay side will send between INTRO_MACHINE_MINIMUM_PADDING
+   * and INTRO_MACHINE_MAXIMUM_PADDING padding cells towards the client. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = INTRO_MACHINE_MINIMUM_PADDING;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = INTRO_MACHINE_MAXIMUM_PADDING;
+
+  /* Configure histogram */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+     histogram_len = 2;
+
+  /* For the relay-side machine we want to batch padding instantly to pretend
+   * its an incoming directory download. So set the histogram edges tight:
+   * (1, 10ms, infinity). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 1000;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 10000;
+
+  /* We put all our tokens in bin 0, which means we want 100% probability
+   * for choosing a inter-packet delay of between 1000 and 10000 microseconds
+   * (1 to 10ms). Since we only have 1 bin, it doesn't matter how many tokens
+   * there are, 1000 out of 1000 is 100% */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1000;
+
+  /* just one bin, so setup the total tokens */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens =
+      relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].histogram[0];
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay intro circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
+
+/************************** Rendezvous-circuit machine ***********************/
+
+/** Create a client-side padding machine that aims to hide rendezvous
+ *  circuits.*/
+void
+circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad rendezvous circuits, and we want to start padding only
+   * after the rendezvous circuit has been established.
+   *
+   * Following a similar argument as for intro circuits, we are aiming for
+   * padded rendezvous circuits to blend in with the initial cell sequence of
+   * general circuits which usually look like this:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *    -> [DATA] -> [DATA] -> DATA -> DATA...(incoming cells continue)
+   *
+   * Whereas normal rendezvous circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> REND2 -> [BEGIN]
+   *
+   * This means that up to the sixth cell (in the first line), both general and
+   * rend circuits have identical cell sequences.
+   *
+   * After that we want to mimic a [DATA] -> [DATA] -> DATA -> DATA sequence.
+   *
+   * With padding negotiation right after the REND_ESTABLISHED, the sequence
+   * becomes:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP...
+   *
+   * After which normal application DATA cells continue on the circuit.
+   *
+   * Hence this way we make rendezvous circuits look like general circuits up
+   * till the end of the circuit setup. */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_JOINED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = 1;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATE) */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that we send our outgoing [DROP] before the PADDING_NEGOTIATED comes
+   * back from the relay side. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client rendezvous circuit hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits.
+ *
+ *  This is meant to follow the client-side machine.
+ */
+void
+circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+    = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  relay_machine->conditions.min_hops = 2;
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = 1;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATED) */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_END;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that the outgoing DROP cell is sent immediately after the
+   * PADDING_NEGOTIATED. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay rendezvous circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
diff --git a/src/core/or/circuitpadding_machines.h b/src/core/or/circuitpadding_machines.h
new file mode 100644
index 0000000000..3c9798d42d
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.h
+ * \brief Header file for circuitpadding_machines.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_MACHINES_H
+#define TOR_CIRCUITPADDING_MACHINES_H
+
+void circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl);
+
+#ifdef CIRCUITPADDING_MACHINES_PRIVATE
+
+/** State of the padding machines that actually sends padding */
+#define CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP CIRCPAD_STATE_BURST
+
+/** Constants defining the amount of padding that a machine will send to hide
+ *  HS circuits. The actual value is sampled uniformly random between the
+ *  min/max values.
+ */
+
+/** Minimum number of relay-side padding cells to be sent by this machine */
+#define INTRO_MACHINE_MINIMUM_PADDING 7
+/** Maximum number of relay-side padding cells to be sent by this machine.
+ *  The actual value will be sampled between the min and max.*/
+#define INTRO_MACHINE_MAXIMUM_PADDING 10
+
+#endif /* defined(CIRCUITPADDING_MACHINES_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_MACHINES_H) */
diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c
index 2cde21fa1f..5875627b93 100644
--- a/src/core/or/circuitstats.c
+++ b/src/core/or/circuitstats.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,8 +29,8 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
-#include "feature/control/control.h"
+#include "lib/confmgt/confmgt.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/networkstatus.h"
@@ -44,6 +44,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/relay/relay_periodic.h"
 
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -639,9 +640,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n)
 void
 circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ,
-                               CIRC_EVENT_FAILED,
-                               END_CIRC_REASON_TIMEOUT);
+  circuit_event_status(circ,
+                       CIRC_EVENT_FAILED,
+                       END_CIRC_REASON_TIMEOUT);
   circuit_change_purpose(TO_CIRCUIT(circ),
                          CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
   /* Record this event to check for too many timeouts
@@ -971,7 +972,7 @@ circuit_build_times_update_state(const circuit_build_times_t *cbt,
 /**
  * Shuffle the build times array.
  *
- * Adapted from http://en.wikipedia.org/wiki/Fisher-Yates_shuffle
+ * Adapted from https://en.wikipedia.org/wiki/Fisher-Yates_shuffle
  */
 static void
 circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt,
@@ -1182,7 +1183,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
 
 /**
  * Estimates the Xm and Alpha parameters using
- * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation
+ * https://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation
  *
  * The notable difference is that we use mode instead of min to estimate Xm.
  * This is because our distribution is frechet-like. We claim this is
@@ -1197,7 +1198,7 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt)
   int n=0,i=0,abandoned_count=0;
   build_time_t max_time=0;
 
-  /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */
+  /* https://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */
   /* We sort of cheat here and make our samples slightly more pareto-like
    * and less frechet-like. */
   cbt->Xm = circuit_build_times_get_xm(cbt);
@@ -1269,9 +1270,9 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt)
  * We use it to calculate the timeout and also to generate synthetic
  * values of time for circuits that timeout before completion.
  *
- * See http://en.wikipedia.org/wiki/Quantile_function,
- * http://en.wikipedia.org/wiki/Inverse_transform_sampling and
- * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_
+ * See https://en.wikipedia.org/wiki/Quantile_function,
+ * https://en.wikipedia.org/wiki/Inverse_transform_sampling and
+ * https://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_
  *     random_sample_from_Pareto_distribution
  * That's right. I'll cite wikipedia all day long.
  *
@@ -1420,6 +1421,7 @@ void
 circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 {
   time_t now = approx_time();
+  // XXXX this should use pubsub
   if (cbt->liveness.nonlive_timeouts > 0) {
     time_t time_since_live = now - cbt->liveness.network_last_live;
     log_notice(LD_CIRC,
diff --git a/src/core/or/circuitstats.h b/src/core/or/circuitstats.h
index 845d7b6722..52c9100f53 100644
--- a/src/core/or/circuitstats.h
+++ b/src/core/or/circuitstats.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -175,7 +175,7 @@ typedef struct {
 } network_liveness_t;
 
 /** Structure for circuit build times history */
-struct circuit_build_times_s {
+struct circuit_build_times_t {
   /** The circular array of recorded build times in milliseconds */
   build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE];
   /** Current index in the circuit_build_times circular array */
diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c
index efd69fb4a3..a88ccf9dd1 100644
--- a/src/core/or/circuituse.c
+++ b/src/core/or/circuituse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,13 +35,15 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/client/proxymode.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_client.h"
@@ -69,7 +71,7 @@
 #include "core/or/origin_circuit_st.h"
 #include "core/or/socks_request_st.h"
 
-static void circuit_expire_old_circuits_clientside(void);
+STATIC void circuit_expire_old_circuits_clientside(void);
 static void circuit_increment_failure_count(void);
 
 /** Check whether the hidden service destination of the stream at
@@ -177,7 +179,6 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
       purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
       purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
     tor_addr_t addr;
-    const int family = tor_addr_parse(&addr, conn->socks_request->address);
     if (!exitnode && !build_state->onehop_tunnel) {
       log_debug(LD_CIRC,"Not considering circuit with unknown router.");
       return 0; /* this circuit is screwed and doesn't know it yet,
@@ -198,6 +199,8 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
           return 0; /* this is a circuit to somewhere else */
         if (tor_digest_is_zero(digest)) {
           /* we don't know the digest; have to compare addr:port */
+          const int family = tor_addr_parse(&addr,
+                                            conn->socks_request->address);
           if (family < 0 ||
               !tor_addr_eq(&build_state->chosen_exit->addr, &addr) ||
               build_state->chosen_exit->port != conn->socks_request->port)
@@ -210,12 +213,14 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
         return 0;
       }
     }
-    if (origin_circ->prepend_policy && family != -1) {
-      int r = compare_tor_addr_to_addr_policy(&addr,
-                                              conn->socks_request->port,
-                                              origin_circ->prepend_policy);
-      if (r == ADDR_POLICY_REJECTED)
-        return 0;
+    if (origin_circ->prepend_policy) {
+      if (tor_addr_parse(&addr, conn->socks_request->address) != -1) {
+        int r = compare_tor_addr_to_addr_policy(&addr,
+                                                conn->socks_request->port,
+                                                origin_circ->prepend_policy);
+        if (r == ADDR_POLICY_REJECTED)
+          return 0;
+      }
     }
     if (exitnode && !connection_ap_can_use_exit(conn, exitnode)) {
       /* can't exit from this router */
@@ -543,9 +548,10 @@ circuit_expire_building(void)
              MAX(get_circuit_build_close_time_ms()*2 + 1000,
                  options->SocksTimeout * 1000));
 
+  bool fixed_time = circuit_build_times_disabled(get_options());
+
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *,victim) {
     struct timeval cutoff;
-    bool fixed_time = circuit_build_times_disabled(get_options());
 
     if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */
         victim->marked_for_close)     /* don't mess with marked circs */
@@ -727,7 +733,7 @@ circuit_expire_building(void)
           circuit_build_times_enough_to_compute(get_circuit_build_times())) {
 
         log_info(LD_CIRC,
-                 "Deciding to count the timeout for circuit %"PRIu32"\n",
+                 "Deciding to count the timeout for circuit %"PRIu32,
                  TO_ORIGIN_CIRCUIT(victim)->global_identifier);
 
         /* Circuits are allowed to last longer for measurement.
@@ -770,16 +776,11 @@ circuit_expire_building(void)
     if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
       switch (victim->purpose) {
       case CIRCUIT_PURPOSE_C_REND_READY:
-        /* We only want to spare a rend circ if it has been specified in
-         * an INTRODUCE1 cell sent to a hidden service.  A circ's
-         * pending_final_cpath field is non-NULL iff it is a rend circ
-         * and we have tried to send an INTRODUCE1 cell specifying it.
-         * Thus, if the pending_final_cpath field *is* NULL, then we
-         * want to not spare it. */
-        if (TO_ORIGIN_CIRCUIT(victim)->build_state &&
-            TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath ==
-            NULL)
+        /* We only want to spare a rend circ iff it has been specified in an
+         * INTRODUCE1 cell sent to a hidden service. */
+        if (!hs_circ_is_rend_sent_in_intro1(CONST_TO_ORIGIN_CIRCUIT(victim))) {
           break;
+        }
         FALLTHROUGH;
       case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
       case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED:
@@ -1425,6 +1426,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
       if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
         hs_dec_rdv_stream_counter(origin_circ);
       }
+
+      /* If there are no more streams on this circ, tell circpad */
+      if (!origin_circ->p_streams)
+        circpad_machine_event_circ_has_no_streams(origin_circ);
+
       return;
     }
   } else {
@@ -1465,7 +1471,7 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
 /** Find each circuit that has been unused for too long, or dirty
  * for too long and has no streams on it: mark it for close.
  */
-static void
+STATIC void
 circuit_expire_old_circuits_clientside(void)
 {
   struct timeval cutoff, now;
@@ -1505,6 +1511,7 @@ circuit_expire_old_circuits_clientside(void)
                 circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT ||
                 circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
                 circ->purpose == CIRCUIT_PURPOSE_TESTING ||
+                circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING ||
                 (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
                 circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) ||
                 circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
@@ -1674,7 +1681,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop)
 void
 circuit_has_opened(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0);
+  circuit_event_status(circ, CIRC_EVENT_BUILT, 0);
 
   /* Remember that this circuit has finished building. Now if we start
    * it building again later (e.g. by extending it), we will know not
@@ -1954,23 +1961,61 @@ have_enough_path_info(int need_exit)
 int
 circuit_purpose_is_hidden_service(uint8_t purpose)
 {
-   if (purpose == CIRCUIT_PURPOSE_HS_VANGUARDS) {
-     return 1;
-   }
-
-   /* Client-side purpose */
-   if (purpose >= CIRCUIT_PURPOSE_C_HS_MIN_ &&
-       purpose <= CIRCUIT_PURPOSE_C_HS_MAX_) {
-     return 1;
-   }
-
-   /* Service-side purpose */
-   if (purpose >= CIRCUIT_PURPOSE_S_HS_MIN_ &&
-       purpose <= CIRCUIT_PURPOSE_S_HS_MAX_) {
-     return 1;
-   }
-
-   return 0;
+  /* HS Vanguard purpose. */
+  if (circuit_purpose_is_hs_vanguards(purpose)) {
+    return 1;
+  }
+
+  /* Client-side purpose */
+  if (circuit_purpose_is_hs_client(purpose)) {
+    return 1;
+  }
+
+  /* Service-side purpose */
+  if (circuit_purpose_is_hs_service(purpose)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Retrun true iff the given circuit is an HS client circuit. */
+bool
+circuit_purpose_is_hs_client(const uint8_t purpose)
+{
+  return (purpose >= CIRCUIT_PURPOSE_C_HS_MIN_ &&
+          purpose <= CIRCUIT_PURPOSE_C_HS_MAX_);
+}
+
+/** Retrun true iff the given circuit is an HS service circuit. */
+bool
+circuit_purpose_is_hs_service(const uint8_t purpose)
+{
+  return (purpose >= CIRCUIT_PURPOSE_S_HS_MIN_ &&
+          purpose <= CIRCUIT_PURPOSE_S_HS_MAX_);
+}
+
+/** Retrun true iff the given circuit is an HS Vanguards circuit. */
+bool
+circuit_purpose_is_hs_vanguards(const uint8_t purpose)
+{
+  return (purpose == CIRCUIT_PURPOSE_HS_VANGUARDS);
+}
+
+/** Retrun true iff the given circuit is an HS v2 circuit. */
+bool
+circuit_is_hs_v2(const circuit_t *circ)
+{
+  return (CIRCUIT_IS_ORIGIN(circ) &&
+          (CONST_TO_ORIGIN_CIRCUIT(circ)->rend_data != NULL));
+}
+
+/** Retrun true iff the given circuit is an HS v3 circuit. */
+bool
+circuit_is_hs_v3(const circuit_t *circ)
+{
+  return (CIRCUIT_IS_ORIGIN(circ) &&
+          (CONST_TO_ORIGIN_CIRCUIT(circ)->hs_ident != NULL));
 }
 
 /**
@@ -2523,8 +2568,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
           circ->rend_data = rend_data_dup(edge_conn->rend_data);
         } else if (edge_conn->hs_ident) {
           circ->hs_ident =
-            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk,
-                                 HS_IDENT_CIRCUIT_INTRO);
+            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk);
         }
         if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
             circ->base_.state == CIRCUIT_STATE_OPEN)
@@ -2596,6 +2640,12 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   /* add it into the linked list of streams on this circuit */
   log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.",
             (unsigned)circ->base_.n_circ_id);
+
+  /* If this is the first stream on this circuit, tell circpad
+   * that streams are attached */
+  if (!circ->p_streams)
+    circpad_machine_event_circ_has_streams(circ);
+
   /* reset it, so we can measure circ timeouts */
   ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL);
   ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams;
@@ -3070,7 +3120,7 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose)
     /* Take specific actions if we are repurposing a hidden service circuit. */
     if (circuit_purpose_is_hidden_service(circ->purpose) &&
         !circuit_purpose_is_hidden_service(new_purpose)) {
-      hs_circ_cleanup(circ);
+      hs_circ_cleanup_on_repurpose(circ);
     }
   }
 
@@ -3080,6 +3130,8 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose)
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ),
                                           old_purpose);
+
+    circpad_machine_event_circ_purpose_changed(TO_ORIGIN_CIRCUIT(circ));
   }
 }
 
@@ -3113,7 +3165,9 @@ circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
 {
   if (!circ) return;
 
-  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+  tor_assertf_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE,
+                       "Wrong relay_body_len: %d (should be at most %d)",
+                       relay_body_len, RELAY_PAYLOAD_SIZE);
 
   circ->n_delivered_written_circ_bw =
       tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len);
diff --git a/src/core/or/circuituse.h b/src/core/or/circuituse.h
index 25588dbb11..95d36d6474 100644
--- a/src/core/or/circuituse.h
+++ b/src/core/or/circuituse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -64,6 +64,15 @@ int hostname_in_track_host_exits(const or_options_t *options,
 void mark_circuit_unusable_for_new_conns(origin_circuit_t *circ);
 
 int circuit_purpose_is_hidden_service(uint8_t);
+
+/* Series of helper functions for hidden services. */
+bool circuit_purpose_is_hs_client(const uint8_t purpose);
+bool circuit_purpose_is_hs_service(const uint8_t purpose);
+bool circuit_purpose_is_hs_vanguards(const uint8_t purpose);
+
+bool circuit_is_hs_v2(const circuit_t *circ);
+bool circuit_is_hs_v3(const circuit_t *circ);
+
 int circuit_should_use_vanguards(uint8_t);
 void circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
 void circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
diff --git a/src/core/or/command.c b/src/core/or/command.c
index 5fb6640c22..8a1d2066cc 100644
--- a/src/core/or/command.c
+++ b/src/core/or/command.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,11 +49,12 @@
 #include "core/or/dos.h"
 #include "core/or/onion.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/relay/circuitbuild_relay.h"
 #include "feature/relay/routermode.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -182,7 +183,7 @@ command_process_cell(channel_t *chan, cell_t *cell)
     command_time_process_cell(cl, cn, & tp ## time ,            \
                               command_process_ ## tp ## _cell);  \
   } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
+#else /* !defined(KEEP_TIMING_STATS) */
 #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn)
 #endif /* defined(KEEP_TIMING_STATS) */
 
@@ -217,23 +218,6 @@ command_process_cell(channel_t *chan, cell_t *cell)
   }
 }
 
-/** Process an incoming var_cell from a channel; in the current protocol all
- * the var_cells are handshake-related and handled below the channel layer,
- * so this just logs a warning and drops the cell.
- */
-
-void
-command_process_var_cell(channel_t *chan, var_cell_t *var_cell)
-{
-  tor_assert(chan);
-  tor_assert(var_cell);
-
-  log_info(LD_PROTOCOL,
-           "Received unexpected var_cell above the channel layer of type %d"
-           "; dropping it.",
-           var_cell->command);
-}
-
 /** Process a 'create' <b>cell</b> that just arrived from <b>chan</b>. Make a
  * new circuit with the p_circ_id specified in cell. Put the circuit in state
  * onionskin_pending, and pass the onionskin to the cpuworker. Circ will get
@@ -685,8 +669,7 @@ command_setup_channel(channel_t *chan)
   tor_assert(chan);
 
   channel_set_cell_handlers(chan,
-                            command_process_cell,
-                            command_process_var_cell);
+                            command_process_cell);
 }
 
 /** Given a listener, install the right handler to process incoming
diff --git a/src/core/or/command.h b/src/core/or/command.h
index 8c90e1de6f..14ebb4a339 100644
--- a/src/core/or/command.h
+++ b/src/core/or/command.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,6 @@
 #include "core/or/channel.h"
 
 void command_process_cell(channel_t *chan, cell_t *cell);
-void command_process_var_cell(channel_t *chan, var_cell_t *cell);
 void command_setup_channel(channel_t *chan);
 void command_setup_listener(channel_listener_t *chan_l);
 
diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c
index 84b80313ce..1394a41c73 100644
--- a/src/core/or/connection_edge.c
+++ b/src/core/or/connection_edge.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -62,21 +62,24 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/sendme.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/dnsserv.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
@@ -97,7 +100,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_util.h"
 
 #include "core/or/cell_st.h"
@@ -300,6 +303,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial)
       }
       return 0;
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
       if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) {
         /* (We already sent an end cell if possible) */
@@ -424,6 +432,21 @@ warn_if_hs_unreachable(const edge_connection_t *conn, uint8_t reason)
   }
 }
 
+/** Given a TTL (in seconds) from a DNS response or from a relay, determine
+ * what TTL clients and relays should actually use for caching it. */
+uint32_t
+clip_dns_ttl(uint32_t ttl)
+{
+  /* This logic is a defense against "DefectTor" DNS-based traffic
+   * confirmation attacks, as in https://nymity.ch/tor-dns/tor-dns.pdf .
+   * We only give two values: a "low" value and a "high" value.
+   */
+  if (ttl < MIN_DNS_TTL)
+    return MIN_DNS_TTL;
+  else
+    return MAX_DNS_TTL;
+}
+
 /** Send a relay end cell from stream <b>conn</b> down conn's circuit, and
  * remember that we've done so.  If this is not a client connection, set the
  * relay end cell's reason for closing as <b>reason</b>.
@@ -472,7 +495,7 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason)
       memcpy(payload+1, tor_addr_to_in6_addr8(&conn->base_.addr), 16);
       addrlen = 16;
     }
-    set_uint32(payload+1+addrlen, htonl(dns_clip_ttl(conn->address_ttl)));
+    set_uint32(payload+1+addrlen, htonl(clip_dns_ttl(conn->address_ttl)));
     payload_len += 4+addrlen;
   }
 
@@ -754,8 +777,13 @@ connection_edge_flushed_some(edge_connection_t *conn)
 {
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       break;
   }
   return 0;
@@ -779,7 +807,7 @@ connection_edge_finished_flushing(edge_connection_t *conn)
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       return 0;
     case AP_CONN_STATE_SOCKS_WAIT:
     case AP_CONN_STATE_NATD_WAIT:
@@ -832,7 +860,7 @@ connected_cell_format_payload(uint8_t *payload_out,
     return -1;
   }
 
-  set_uint32(payload_out + connected_payload_len, htonl(dns_clip_ttl(ttl)));
+  set_uint32(payload_out + connected_payload_len, htonl(clip_dns_ttl(ttl)));
   connected_payload_len += 4;
 
   tor_assert(connected_payload_len <= MAX_CONNECTED_CELL_PAYLOAD_LEN);
@@ -1211,7 +1239,7 @@ connection_ap_rescan_and_attach_pending(void)
     entry_conn->marked_pending_circ_line = 0;   \
     entry_conn->marked_pending_circ_file = 0;   \
   } while (0)
-#else /* !(defined(DEBUGGING_17659)) */
+#else /* !defined(DEBUGGING_17659) */
 #define UNMARK() do { } while (0)
 #endif /* defined(DEBUGGING_17659) */
 
@@ -1540,6 +1568,107 @@ consider_plaintext_ports(entry_connection_t *conn, uint16_t port)
   return 0;
 }
 
+/** Parse the given hostname in address. Returns true if the parsing was
+ * successful and type_out contains the type of the hostname. Else, false is
+ * returned which means it was not recognized and type_out is set to
+ * BAD_HOSTNAME.
+ *
+ * The possible recognized forms are (where true is returned):
+ *
+ *  If address is of the form "y.onion" with a well-formed handle y:
+ *     Put a NUL after y, lower-case it, and return ONION_V2_HOSTNAME or
+ *     ONION_V3_HOSTNAME depending on the HS version.
+ *
+ *  If address is of the form "x.y.onion" with a well-formed handle x:
+ *     Drop "x.", put a NUL after y, lower-case it, and return
+ *     ONION_V2_HOSTNAME or ONION_V3_HOSTNAME depending on the HS version.
+ *
+ * If address is of the form "y.onion" with a badly-formed handle y:
+ *     Return BAD_HOSTNAME and log a message.
+ *
+ * If address is of the form "y.exit":
+ *     Put a NUL after y and return EXIT_HOSTNAME.
+ *
+ * Otherwise:
+ *     Return NORMAL_HOSTNAME and change nothing.
+ */
+STATIC bool
+parse_extended_hostname(char *address, hostname_type_t *type_out)
+{
+  char *s;
+  char *q;
+  char query[HS_SERVICE_ADDR_LEN_BASE32+1];
+
+  s = strrchr(address,'.');
+  if (!s) {
+    *type_out = NORMAL_HOSTNAME; /* no dot, thus normal */
+    goto success;
+  }
+  if (!strcmp(s+1,"exit")) {
+    *s = 0; /* NUL-terminate it */
+    *type_out = EXIT_HOSTNAME; /* .exit */
+    goto success;
+  }
+  if (strcmp(s+1,"onion")) {
+    *type_out = NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */
+    goto success;
+  }
+
+  /* so it is .onion */
+  *s = 0; /* NUL-terminate it */
+  /* locate a 'sub-domain' component, in order to remove it */
+  q = strrchr(address, '.');
+  if (q == address) {
+    *type_out = BAD_HOSTNAME;
+    goto failed; /* reject sub-domain, as DNS does */
+  }
+  q = (NULL == q) ? address : q + 1;
+  if (strlcpy(query, q, HS_SERVICE_ADDR_LEN_BASE32+1) >=
+      HS_SERVICE_ADDR_LEN_BASE32+1) {
+    *type_out = BAD_HOSTNAME;
+    goto failed;
+  }
+  if (q != address) {
+    memmove(address, q, strlen(q) + 1 /* also get \0 */);
+  }
+  /* v2 onion address check. */
+  if (strlen(query) == REND_SERVICE_ID_LEN_BASE32) {
+    *type_out = ONION_V2_HOSTNAME;
+    if (rend_valid_v2_service_id(query)) {
+      goto success;
+    }
+    goto failed;
+  }
+
+  /* v3 onion address check. */
+  if (strlen(query) == HS_SERVICE_ADDR_LEN_BASE32) {
+    *type_out = ONION_V3_HOSTNAME;
+    if (hs_address_is_valid(query)) {
+      goto success;
+    }
+    goto failed;
+  }
+
+  /* Reaching this point, nothing was recognized. */
+  *type_out = BAD_HOSTNAME;
+  goto failed;
+
+ success:
+  return true;
+ failed:
+  /* otherwise, return to previous state and return 0 */
+  *s = '.';
+  const bool is_onion = (*type_out == ONION_V2_HOSTNAME) ||
+    (*type_out == ONION_V3_HOSTNAME);
+  log_warn(LD_APP, "Invalid %shostname %s; rejecting",
+           is_onion ? "onion " : "",
+           safe_str_client(address));
+  if (*type_out == ONION_V3_HOSTNAME) {
+      *type_out = BAD_HOSTNAME;
+  }
+  return false;
+}
+
 /** How many times do we try connecting with an exit configured via
  * TrackHostExits before concluding that it won't work any more and trying a
  * different one? */
@@ -2007,16 +2136,15 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   const int automap = rr.automap;
   const addressmap_entry_source_t exit_source = rr.exit_source;
 
-  /* Now, we parse the address to see if it's an .onion or .exit or
-   * other special address.
-   */
-  const hostname_type_t addresstype = parse_extended_hostname(socks->address);
-
   /* Now see whether the hostname is bogus.  This could happen because of an
    * onion hostname whose format we don't recognize. */
-  if (addresstype == BAD_HOSTNAME) {
+  hostname_type_t addresstype;
+  if (!parse_extended_hostname(socks->address, &addresstype)) {
     control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                 escaped(socks->address));
+    if (addresstype == BAD_HOSTNAME) {
+      conn->socks_request->socks_extended_error_code = SOCKS5_HS_BAD_ADDRESS;
+    }
     connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
     return -1;
   }
@@ -2803,6 +2931,31 @@ connection_ap_process_natd(entry_connection_t *conn)
   return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
 }
 
+static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 405 Method Not Allowed\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>This is an HTTP CONNECT tunnel, not a full HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>This is an HTTP CONNECT tunnel, not an HTTP proxy.</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use this Tor port as\n"
+  "an HTTP proxy.\n"
+  "</p><p>\n"
+  "This is not correct: This port is configured as a CONNECT tunnel, not\n"
+  "an HTTP proxy. Please configure your client accordingly.  You can also\n"
+  "use HTTPS; then the client should automatically use HTTP CONNECT."
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
 /** Called on an HTTP CONNECT entry connection when some bytes have arrived,
  * but we have not yet received a full HTTP CONNECT request.  Try to parse an
  * HTTP CONNECT request from the connection's inbuf.  On success, set up the
@@ -2843,7 +2996,7 @@ connection_ap_process_http_connect(entry_connection_t *conn)
   tor_assert(command);
   tor_assert(addrport);
   if (strcasecmp(command, "connect")) {
-    errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n";
+    errmsg = HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG;
     goto err;
   }
 
@@ -3309,8 +3462,9 @@ tell_controller_about_resolved_result(entry_connection_t *conn,
   expires = time(NULL) + ttl;
   if (answer_type == RESOLVED_TYPE_IPV4 && answer_len >= 4) {
     char *cp = tor_dup_ip(ntohl(get_uint32(answer)));
-    control_event_address_mapped(conn->socks_request->address,
-                                 cp, expires, NULL, 0);
+    if (cp)
+      control_event_address_mapped(conn->socks_request->address,
+                                   cp, expires, NULL, 0);
     tor_free(cp);
   } else if (answer_type == RESOLVED_TYPE_HOSTNAME && answer_len < 256) {
     char *cp = tor_strndup(answer, answer_len);
@@ -3383,7 +3537,7 @@ connection_ap_handshake_socks_resolved,(entry_connection_t *conn,
       }
     } else if (answer_type == RESOLVED_TYPE_IPV6 && answer_len == 16) {
       tor_addr_t a;
-      tor_addr_from_ipv6_bytes(&a, (char*)answer);
+      tor_addr_from_ipv6_bytes(&a, answer);
       if (! tor_addr_is_null(&a)) {
         client_dns_set_addressmap(conn,
                                   conn->socks_request->address, &a,
@@ -3484,11 +3638,17 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
                                     size_t replylen, int endreason)
 {
   char buf[256];
-  socks5_reply_status_t status =
-    stream_end_reason_to_socks5_response(endreason);
+  socks5_reply_status_t status;
 
   tor_assert(conn->socks_request); /* make sure it's an AP stream */
 
+  if (conn->socks_request->socks_use_extended_errors &&
+      conn->socks_request->socks_extended_error_code != 0) {
+    status = conn->socks_request->socks_extended_error_code;
+  } else {
+    status = stream_end_reason_to_socks5_response(endreason);
+  }
+
   if (!SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command)) {
     control_event_stream_status(conn, status==SOCKS5_SUCCEEDED ?
                                 STREAM_EVENT_SUCCEEDED : STREAM_EVENT_FAILED,
@@ -3706,6 +3866,10 @@ handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn)
   /* Link the circuit and the connection crypt path. */
   conn->cpath_layer = origin_circ->cpath->prev;
 
+  /* If this is the first stream on this circuit, tell circpad */
+  if (!origin_circ->p_streams)
+    circpad_machine_event_circ_has_streams(origin_circ);
+
   /* Add it into the linked list of p_streams on this circuit */
   conn->next_stream = origin_circ->p_streams;
   origin_circ->p_streams = conn;
@@ -3796,6 +3960,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
 
   if (! bcell.is_begindir) {
     /* Steal reference */
+    tor_assert(bcell.address);
     address = bcell.address;
     port = bcell.port;
 
@@ -4300,68 +4465,6 @@ connection_ap_can_use_exit(const entry_connection_t *conn,
   return 1;
 }
 
-/** If address is of the form "y.onion" with a well-formed handle y:
- *     Put a NUL after y, lower-case it, and return ONION_V2_HOSTNAME or
- *     ONION_V3_HOSTNAME depending on the HS version.
- *
- *  If address is of the form "x.y.onion" with a well-formed handle x:
- *     Drop "x.", put a NUL after y, lower-case it, and return
- *     ONION_V2_HOSTNAME or ONION_V3_HOSTNAME depending on the HS version.
- *
- * If address is of the form "y.onion" with a badly-formed handle y:
- *     Return BAD_HOSTNAME and log a message.
- *
- * If address is of the form "y.exit":
- *     Put a NUL after y and return EXIT_HOSTNAME.
- *
- * Otherwise:
- *     Return NORMAL_HOSTNAME and change nothing.
- */
-hostname_type_t
-parse_extended_hostname(char *address)
-{
-    char *s;
-    char *q;
-    char query[HS_SERVICE_ADDR_LEN_BASE32+1];
-
-    s = strrchr(address,'.');
-    if (!s)
-      return NORMAL_HOSTNAME; /* no dot, thus normal */
-    if (!strcmp(s+1,"exit")) {
-      *s = 0; /* NUL-terminate it */
-      return EXIT_HOSTNAME; /* .exit */
-    }
-    if (strcmp(s+1,"onion"))
-      return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */
-
-    /* so it is .onion */
-    *s = 0; /* NUL-terminate it */
-    /* locate a 'sub-domain' component, in order to remove it */
-    q = strrchr(address, '.');
-    if (q == address) {
-      goto failed; /* reject sub-domain, as DNS does */
-    }
-    q = (NULL == q) ? address : q + 1;
-    if (strlcpy(query, q, HS_SERVICE_ADDR_LEN_BASE32+1) >=
-        HS_SERVICE_ADDR_LEN_BASE32+1)
-      goto failed;
-    if (q != address) {
-      memmove(address, q, strlen(q) + 1 /* also get \0 */);
-    }
-    if (rend_valid_v2_service_id(query)) {
-      return ONION_V2_HOSTNAME; /* success */
-    }
-    if (hs_address_is_valid(query)) {
-      return ONION_V3_HOSTNAME;
-    }
- failed:
-    /* otherwise, return to previous state and return 0 */
-    *s = '.';
-    log_warn(LD_APP, "Invalid onion hostname %s; rejecting",
-             safe_str_client(address));
-    return BAD_HOSTNAME;
-}
-
 /** Return true iff the (possibly NULL) <b>alen</b>-byte chunk of memory at
  * <b>a</b> is equal to the (possibly NULL) <b>blen</b>-byte chunk of memory
  * at <b>b</b>. */
@@ -4565,6 +4668,25 @@ circuit_clear_isolation(origin_circuit_t *circ)
   circ->socks_username_len = circ->socks_password_len = 0;
 }
 
+/** Send an END and mark for close the given edge connection conn using the
+ * given reason that has to be a stream reason.
+ *
+ * Note: We don't unattached the AP connection (if applicable) because we
+ * don't want to flush the remaining data. This function aims at ending
+ * everything quickly regardless of the connection state.
+ *
+ * This function can't fail and does nothing if conn is NULL. */
+void
+connection_edge_end_close(edge_connection_t *conn, uint8_t reason)
+{
+  if (!conn) {
+    return;
+  }
+
+  connection_edge_end(conn, reason);
+  connection_mark_for_close(TO_CONN(conn));
+}
+
 /** Free all storage held in module-scoped variables for connection_edge.c */
 void
 connection_edge_free_all(void)
diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h
index 68d8b19a11..8c06af5664 100644
--- a/src/core/or/connection_edge.h
+++ b/src/core/or/connection_edge.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,6 +71,15 @@ entry_connection_t *EDGE_TO_ENTRY_CONN(edge_connection_t *);
 #define connection_mark_unattached_ap(conn, endreason)                  \
   connection_mark_unattached_ap_((conn), (endreason), __LINE__, SHORT_FILE__)
 
+/** Possible return values for parse_extended_hostname. */
+typedef enum hostname_type_t {
+  BAD_HOSTNAME,
+  EXIT_HOSTNAME,
+  NORMAL_HOSTNAME,
+  ONION_V2_HOSTNAME,
+  ONION_V3_HOSTNAME,
+} hostname_type_t;
+
 MOCK_DECL(void,connection_mark_unattached_ap_,
           (entry_connection_t *conn, int endreason,
            int line, const char *file));
@@ -80,6 +89,7 @@ int connection_edge_process_inbuf(edge_connection_t *conn,
 int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn);
 int connection_edge_end(edge_connection_t *conn, uint8_t reason);
 int connection_edge_end_errno(edge_connection_t *conn);
+void connection_edge_end_close(edge_connection_t *conn, uint8_t reason);
 int connection_edge_flushed_some(edge_connection_t *conn);
 int connection_edge_finished_flushing(edge_connection_t *conn);
 int connection_edge_finished_connecting(edge_connection_t *conn);
@@ -154,13 +164,6 @@ int connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
                                                origin_circuit_t *circ,
                                                crypt_path_t *cpath);
 
-/** Possible return values for parse_extended_hostname. */
-typedef enum hostname_type_t {
-  NORMAL_HOSTNAME, ONION_V2_HOSTNAME, ONION_V3_HOSTNAME,
-  EXIT_HOSTNAME, BAD_HOSTNAME
-} hostname_type_t;
-hostname_type_t parse_extended_hostname(char *address);
-
 #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
 int get_pf_socket(void);
 #endif
@@ -179,6 +182,21 @@ void connection_ap_warn_and_unmark_if_pending_circ(
                                              entry_connection_t *entry_conn,
                                              const char *where);
 
+/** Lowest value for DNS ttl that a server should give or a client should
+ * believe. */
+#define MIN_DNS_TTL (5*60)
+/** Highest value for DNS ttl that a server should give or a client should
+ * believe. */
+#define MAX_DNS_TTL (60*60)
+/** How long do we keep DNS cache entries before purging them (regardless of
+ * their TTL)? */
+#define MAX_DNS_ENTRY_AGE (3*60*60)
+/** How long do we cache/tell clients to cache DNS records when no TTL is
+ * known? */
+#define DEFAULT_DNS_TTL (30*60)
+
+uint32_t clip_dns_ttl(uint32_t ttl);
+
 int connection_half_edge_is_valid_data(const smartlist_t *half_conns,
                                        streamid_t stream_id);
 int connection_half_edge_is_valid_sendme(const smartlist_t *half_conns,
@@ -218,6 +236,8 @@ void half_edge_free_(struct half_edge_t *he);
 
 #ifdef CONNECTION_EDGE_PRIVATE
 
+STATIC bool parse_extended_hostname(char *address, hostname_type_t *type_out);
+
 /** A parsed BEGIN or BEGIN_DIR cell */
 typedef struct begin_cell_t {
   /** The address the client has asked us to connect to, or NULL if this is
diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c
index 67157d8d0b..b88d1b6afb 100644
--- a/src/core/or/connection_or.c
+++ b/src/core/or/connection_or.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,17 +18,19 @@
  * tortls.c) which it uses as its TLS stream.  It is responsible for
  * sending and receiving cells over that TLS.
  *
- * This module also implements the client side of the v3 Tor link handshake,
+ * This module also implements the client side of the v3 (and greater) Tor
+ * link handshake.
  **/
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CONNECTION_OR_PRIVATE
+#define ORCONN_EVENT_PRIVATE
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitbuild.h"
@@ -38,14 +40,14 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
-#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/relay/relay_handshake.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/client/entrynodes.h"
 #include "lib/geoip/geoip.h"
 #include "core/mainloop/mainloop.h"
-#include "trunnel/link_handshake.h"
+#include "trunnel/netinfo.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -76,7 +78,8 @@
 #include "lib/crypt_ops/crypto_format.h"
 
 #include "lib/tls/tortls.h"
-#include "lib/tls/x509.h"
+
+#include "core/or/orconn_event.h"
 
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_launch_v3_or_handshake(or_connection_t *conn);
@@ -91,13 +94,6 @@ static unsigned int
 connection_or_is_bad_for_new_circs(or_connection_t *or_conn);
 static void connection_or_mark_bad_for_new_circs(or_connection_t *or_conn);
 
-/*
- * Call this when changing connection state, so notifications to the owning
- * channel can be handled.
- */
-
-static void connection_or_change_state(or_connection_t *conn, uint8_t state);
-
 static void connection_or_check_canonicity(or_connection_t *conn,
                                            int started_here);
 
@@ -112,10 +108,6 @@ TO_OR_CONN(connection_t *c)
   return DOWNCAST(or_connection_t, c);
 }
 
-/** Global map between Extended ORPort identifiers and OR
- *  connections. */
-static digestmap_t *orconn_ext_or_id_map = NULL;
-
 /** Clear clear conn->identity_digest and update other data
  * structures as appropriate.*/
 void
@@ -201,71 +193,6 @@ connection_or_set_identity_digest(or_connection_t *conn,
     channel_set_identity_digest(chan, rsa_digest, ed_id);
 }
 
-/** Remove the Extended ORPort identifier of <b>conn</b> from the
- *  global identifier list. Also, clear the identifier from the
- *  connection itself. */
-void
-connection_or_remove_from_ext_or_id_map(or_connection_t *conn)
-{
-  or_connection_t *tmp;
-  if (!orconn_ext_or_id_map)
-    return;
-  if (!conn->ext_or_conn_id)
-    return;
-
-  tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id);
-  if (!tor_digest_is_zero(conn->ext_or_conn_id))
-    tor_assert(tmp == conn);
-
-  memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN);
-}
-
-/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such
- * connection is found. */
-or_connection_t *
-connection_or_get_by_ext_or_id(const char *id)
-{
-  if (!orconn_ext_or_id_map)
-    return NULL;
-  return digestmap_get(orconn_ext_or_id_map, id);
-}
-
-/** Deallocate the global Extended ORPort identifier list */
-void
-connection_or_clear_ext_or_id_map(void)
-{
-  digestmap_free(orconn_ext_or_id_map, NULL);
-  orconn_ext_or_id_map = NULL;
-}
-
-/** Creates an Extended ORPort identifier for <b>conn</b> and deposits
- *  it into the global list of identifiers. */
-void
-connection_or_set_ext_or_identifier(or_connection_t *conn)
-{
-  char random_id[EXT_OR_CONN_ID_LEN];
-  or_connection_t *tmp;
-
-  if (!orconn_ext_or_id_map)
-    orconn_ext_or_id_map = digestmap_new();
-
-  /* Remove any previous identifiers: */
-  if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id))
-      connection_or_remove_from_ext_or_id_map(conn);
-
-  do {
-    crypto_rand(random_id, sizeof(random_id));
-  } while (digestmap_get(orconn_ext_or_id_map, random_id));
-
-  if (!conn->ext_or_conn_id)
-    conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN);
-
-  memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN);
-
-  tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn);
-  tor_assert(!tmp);
-}
-
 /**************************************************************/
 
 /** Map from a string describing what a non-open OR connection was doing when
@@ -400,23 +327,69 @@ connection_or_report_broken_states(int severity, int domain)
   smartlist_free(items);
 }
 
+/**
+ * Helper function to publish an OR connection status event
+ *
+ * Publishes a messages to subscribers of ORCONN messages, and sends
+ * the control event.
+ **/
+void
+connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp,
+                           int reason)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  msg->status = tp;
+  msg->reason = reason;
+  orconn_status_publish(msg);
+  control_event_or_conn_status(conn, tp, reason);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * connection_or_change_state() calls this to notify subscribers about
+ * a change of an OR connection state.
+ **/
+static void
+connection_or_state_publish(const or_connection_t *conn, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  if (conn->is_pt) {
+    /* Do extra decoding because conn->proxy_type indicates the proxy
+     * protocol that tor uses to talk with the transport plugin,
+     * instead of PROXY_PLUGGABLE. */
+    tor_assert_nonfatal(conn->proxy_type != PROXY_NONE);
+    msg->proxy_type = PROXY_PLUGGABLE;
+  } else {
+    msg->proxy_type = conn->proxy_type;
+  }
+  msg->state = state;
+  if (conn->chan) {
+    msg->chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier;
+  } else {
+    msg->chan = 0;
+  }
+  orconn_state_publish(msg);
+}
+
 /** Call this to change or_connection_t states, so the owning channel_tls_t can
  * be notified.
  */
 
-static void
-connection_or_change_state(or_connection_t *conn, uint8_t state)
+MOCK_IMPL(STATIC void,
+connection_or_change_state,(or_connection_t *conn, uint8_t state))
 {
-  uint8_t old_state;
-
   tor_assert(conn);
 
-  old_state = conn->base_.state;
   conn->base_.state = state;
 
+  connection_or_state_publish(conn, state);
   if (conn->chan)
-    channel_tls_handle_state_change_on_orconn(conn->chan, conn,
-                                              old_state, state);
+    channel_tls_handle_state_change_on_orconn(conn->chan, conn, state);
 }
 
 /** Return the number of circuits using an or_connection_t; this used to
@@ -676,6 +649,19 @@ connection_or_finished_flushing(or_connection_t *conn)
 
   switch (conn->base_.state) {
     case OR_CONN_STATE_PROXY_HANDSHAKING:
+      /* PROXY_HAPROXY gets connected by receiving an ack. */
+      if (conn->proxy_type == PROXY_HAPROXY) {
+        tor_assert(TO_CONN(conn)->proxy_state == PROXY_HAPROXY_WAIT_FOR_FLUSH);
+        TO_CONN(conn)->proxy_state = PROXY_CONNECTED;
+
+        if (connection_tls_start_handshake(conn, 0) < 0) {
+          /* TLS handshaking error of some kind. */
+          connection_or_close_for_error(conn, 0);
+          return -1;
+        }
+        break;
+      }
+      break;
     case OR_CONN_STATE_OPEN:
     case OR_CONN_STATE_OR_HANDSHAKING_V2:
     case OR_CONN_STATE_OR_HANDSHAKING_V3:
@@ -707,8 +693,6 @@ connection_or_finished_connecting(or_connection_t *or_conn)
 
   log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.",
             conn->address,conn->port);
-  control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  control_event_boot_first_orconn();
 
   if (proxy_type != PROXY_NONE) {
     /* start proxy handshake */
@@ -717,8 +701,9 @@ connection_or_finished_connecting(or_connection_t *or_conn)
       return -1;
     }
 
-    connection_start_reading(conn);
     connection_or_change_state(or_conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+    connection_start_reading(conn);
+
     return 0;
   }
 
@@ -758,21 +743,27 @@ connection_or_about_to_close(or_connection_t *or_conn)
       entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan));
       if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) {
         int reason = tls_error_to_orconn_end_reason(or_conn->tls_error);
-        control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED,
-                                     reason);
-        if (!authdir_mode_tests_reachability(options))
-          control_event_bootstrap_prob_or(
-                orconn_end_reason_to_control_string(reason),
-                reason, or_conn);
+        connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED,
+                                   reason);
+        if (!authdir_mode_tests_reachability(options)) {
+          const char *warning = NULL;
+          if (reason == END_OR_CONN_REASON_TLS_ERROR && or_conn->tls) {
+            warning = tor_tls_get_last_error_msg(or_conn->tls);
+          }
+          if (warning == NULL) {
+            warning = orconn_end_reason_to_control_string(reason);
+          }
+          control_event_bootstrap_prob_or(warning, reason, or_conn);
+        }
       }
     }
   } else if (conn->hold_open_until_flushed) {
     /* We only set hold_open_until_flushed when we're intentionally
      * closing a connection. */
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   } else if (!tor_digest_is_zero(or_conn->identity_digest)) {
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   }
 }
@@ -918,12 +909,21 @@ connection_or_check_canonicity(or_connection_t *conn, int started_here)
   }
 
   if (r) {
-    tor_addr_port_t node_ap;
-    node_get_pref_orport(r, &node_ap);
-    /* XXXX proposal 186 is making this more complex.  For now, a conn
-       is canonical when it uses the _preferred_ address. */
-    if (tor_addr_eq(&conn->base_.addr, &node_ap.addr))
+    tor_addr_port_t node_ipv4_ap;
+    tor_addr_port_t node_ipv6_ap;
+    node_get_prim_orport(r, &node_ipv4_ap);
+    node_get_pref_ipv6_orport(r, &node_ipv6_ap);
+    if (tor_addr_eq(&conn->base_.addr, &node_ipv4_ap.addr) ||
+        tor_addr_eq(&conn->base_.addr, &node_ipv6_ap.addr)) {
       connection_or_set_canonical(conn, 1);
+    }
+    /* Choose the correct canonical address and port. */
+    tor_addr_port_t *node_ap;
+    if (tor_addr_family(&conn->base_.addr) == AF_INET) {
+      node_ap = &node_ipv4_ap;
+    } else {
+      node_ap = &node_ipv6_ap;
+    }
     if (!started_here) {
       /* Override the addr/port, so our log messages will make sense.
        * This is dangerous, since if we ever try looking up a conn by
@@ -935,13 +935,14 @@ connection_or_check_canonicity(or_connection_t *conn, int started_here)
        * right IP address and port 56244, that wouldn't be as helpful. now we
        * log the "right" port too, so we know if it's moria1 or moria2.
        */
-      tor_addr_copy(&conn->base_.addr, &node_ap.addr);
-      conn->base_.port = node_ap.port;
+      /* See #33898 for a ticket that resolves this technical debt. */
+      tor_addr_copy(&conn->base_.addr, &node_ap->addr);
+      conn->base_.port = node_ap->port;
     }
     tor_free(conn->nickname);
     conn->nickname = tor_strdup(node_get_nickname(r));
     tor_free(conn->base_.address);
-    conn->base_.address = tor_addr_to_str_dup(&node_ap.addr);
+    conn->base_.address = tor_addr_to_str_dup(&node_ap->addr);
   } else {
     tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
@@ -1229,11 +1230,11 @@ or_connect_failure_ht_hash(const or_connect_failure_entry_t *entry)
 }
 
 HT_PROTOTYPE(or_connect_failure_ht, or_connect_failure_entry_t, node,
-             or_connect_failure_ht_hash, or_connect_failure_ht_eq)
+             or_connect_failure_ht_hash, or_connect_failure_ht_eq);
 
 HT_GENERATE2(or_connect_failure_ht, or_connect_failure_entry_t, node,
              or_connect_failure_ht_hash, or_connect_failure_ht_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 /* Initialize a given connect failure entry with the given identity_digest,
  * addr and port. All field are optional except ocf. */
@@ -1364,7 +1365,7 @@ void
 connection_or_connect_failed(or_connection_t *conn,
                              int reason, const char *msg)
 {
-  control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason);
+  connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason);
   if (!authdir_mode_tests_reachability(get_options()))
     control_event_bootstrap_prob_or(msg, reason, conn);
   note_or_connect_failed(conn);
@@ -1430,7 +1431,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
   int r;
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt = 0;
 
   tor_assert(_addr);
   tor_assert(id_digest);
@@ -1471,21 +1472,27 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
     return NULL;
   }
 
-  connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
-
   conn->is_outgoing = 1;
 
   /* If we are using a proxy server, find it and use it. */
-  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, TO_CONN(conn));
+  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         TO_CONN(conn));
   if (r == 0) {
     conn->proxy_type = proxy_type;
     if (proxy_type != PROXY_NONE) {
       tor_addr_copy(&addr, &proxy_addr);
       port = proxy_port;
       conn->base_.proxy_state = PROXY_INFANT;
+      conn->is_pt = is_pt;
     }
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
   } else {
+    /* This duplication of state change calls is necessary in case we
+     * run into an error condition below */
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
+
     /* get_proxy_addrport() might fail if we have a Bridge line that
        references a transport, but no ClientTransportPlugin lines
        defining its transport proxy. If this is the case, let's try to
@@ -1691,7 +1698,8 @@ connection_tls_continue_handshake(or_connection_t *conn)
 
   switch (result) {
     CASE_TOR_TLS_ERROR_ANY:
-    log_info(LD_OR,"tls error [%s]. breaking connection.",
+      conn->tls_error = result;
+      log_info(LD_OR,"tls error [%s]. breaking connection.",
              tor_tls_err_to_string(result));
       return -1;
     case TOR_TLS_DONE:
@@ -1723,6 +1731,7 @@ connection_tls_continue_handshake(or_connection_t *conn)
       log_debug(LD_OR,"wanted read");
       return 0;
     case TOR_TLS_CLOSE:
+      conn->tls_error = result;
       log_info(LD_OR,"tls closed. breaking connection.");
       return -1;
   }
@@ -1981,8 +1990,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
 
     /* Tell the new guard API about the channel failure */
     entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan));
-    control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED,
-                                 END_OR_CONN_REASON_OR_IDENTITY);
+    connection_or_event_status(conn, OR_CONN_EVENT_FAILED,
+                               END_OR_CONN_REASON_OR_IDENTITY);
     if (!authdir_mode_tests_reachability(options))
       control_event_bootstrap_prob_or(
                                 "Unexpected identity in router certificate",
@@ -2222,7 +2231,7 @@ int
 connection_or_set_state_open(or_connection_t *conn)
 {
   connection_or_change_state(conn, OR_CONN_STATE_OPEN);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0);
 
   /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection
    * that uses an earlier link protocol should not be treated as a relay. */
@@ -2252,6 +2261,8 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   cell_pack(&networkcell, cell, conn->wide_circ_ids);
 
+  /* We need to count padding cells from this non-packed code path
+   * since they are sent via chan->write_cell() (which is not packed) */
   rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   if (cell->command == CELL_PADDING)
     rep_hist_padding_count_write(PADDING_TYPE_CELL);
@@ -2262,7 +2273,7 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
   if (conn->chan) {
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
-    if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) {
+    if (TLS_CHAN_TO_BASE(conn->chan)->padding_enabled) {
       rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
       if (cell->command == CELL_PADDING)
         rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
@@ -2292,6 +2303,7 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0);
 
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   /* Touch the channel's active timestamp if there is one */
   if (conn->chan)
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
@@ -2428,6 +2440,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus)
   return 0;
 }
 
+static netinfo_addr_t *
+netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr)
+{
+  sa_family_t addr_family = tor_addr_family(tor_addr);
+
+  if (BUG(addr_family != AF_INET && addr_family != AF_INET6))
+    return NULL;
+
+  netinfo_addr_t *netinfo_addr = netinfo_addr_new();
+
+  if (addr_family == AF_INET) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4);
+    netinfo_addr_set_len(netinfo_addr, 4);
+    netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr));
+  } else if (addr_family == AF_INET6) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6);
+    netinfo_addr_set_len(netinfo_addr, 16);
+    uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr);
+    memcpy(ipv6_buf, in6_addr, 16);
+  }
+
+  return netinfo_addr;
+}
+
 /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know
  * about their address, our address, and the current time. */
 MOCK_IMPL(int,
@@ -2436,8 +2473,7 @@ connection_or_send_netinfo,(or_connection_t *conn))
   cell_t cell;
   time_t now = time(NULL);
   const routerinfo_t *me;
-  int len;
-  uint8_t *out;
+  int r = -1;
 
   tor_assert(conn->handshake_state);
 
@@ -2450,20 +2486,21 @@ connection_or_send_netinfo,(or_connection_t *conn))
   memset(&cell, 0, sizeof(cell_t));
   cell.command = CELL_NETINFO;
 
+  netinfo_cell_t *netinfo_cell = netinfo_cell_new();
+
   /* Timestamp, if we're a relay. */
   if (public_server_mode(get_options()) || ! conn->is_outgoing)
-    set_uint32(cell.payload, htonl((uint32_t)now));
+    netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now);
 
   /* Their address. */
-  out = cell.payload + 4;
+  const tor_addr_t *remote_tor_addr =
+    !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr;
   /* We use &conn->real_addr below, unless it hasn't yet been set. If it
    * hasn't yet been set, we know that base_.addr hasn't been tampered with
    * yet either. */
-  len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr)
-                                       ? &conn->real_addr : &conn->base_.addr);
-  if (len<0)
-    return -1;
-  out += len;
+  netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr);
+
+  netinfo_cell_set_other_addr(netinfo_cell, their_addr);
 
   /* My address -- only include it if I'm a public relay, or if I'm a
    * bridge and this is an incoming connection. If I'm a bridge and this
@@ -2471,556 +2508,40 @@ connection_or_send_netinfo,(or_connection_t *conn))
   if ((public_server_mode(get_options()) || !conn->is_outgoing) &&
       (me = router_get_my_routerinfo())) {
     tor_addr_t my_addr;
-    *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr);
-
     tor_addr_from_ipv4h(&my_addr, me->addr);
-    len = append_address_to_payload(out, &my_addr);
-    if (len < 0)
-      return -1;
-    out += len;
-
-    if (!tor_addr_is_null(&me->ipv6_addr)) {
-      len = append_address_to_payload(out, &me->ipv6_addr);
-      if (len < 0)
-        return -1;
-    }
-  } else {
-    *out = 0;
-  }
-
-  conn->handshake_state->digest_sent_data = 0;
-  conn->handshake_state->sent_netinfo = 1;
-  connection_or_write_cell_to_buf(&cell, conn);
-
-  return 0;
-}
-
-/** Helper used to add an encoded certs to a cert cell */
-static void
-add_certs_cell_cert_helper(certs_cell_t *certs_cell,
-                           uint8_t cert_type,
-                           const uint8_t *cert_encoded,
-                           size_t cert_len)
-{
-  tor_assert(cert_len <= UINT16_MAX);
-  certs_cell_cert_t *ccc = certs_cell_cert_new();
-  ccc->cert_type = cert_type;
-  ccc->cert_len = cert_len;
-  certs_cell_cert_setlen_body(ccc, cert_len);
-  memcpy(certs_cell_cert_getarray_body(ccc), cert_encoded, cert_len);
-
-  certs_cell_add_certs(certs_cell, ccc);
-}
-
-/** Add an encoded X509 cert (stored as <b>cert_len</b> bytes at
- * <b>cert_encoded</b>) to the trunnel certs_cell_t object that we are
- * building in <b>certs_cell</b>.  Set its type field to <b>cert_type</b>.
- * (If <b>cert</b> is NULL, take no action.) */
-static void
-add_x509_cert(certs_cell_t *certs_cell,
-              uint8_t cert_type,
-              const tor_x509_cert_t *cert)
-{
-  if (NULL == cert)
-    return;
 
-  const uint8_t *cert_encoded = NULL;
-  size_t cert_len;
-  tor_x509_cert_get_der(cert, &cert_encoded, &cert_len);
+    uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr);
+    netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs);
 
-  add_certs_cell_cert_helper(certs_cell, cert_type, cert_encoded, cert_len);
-}
+    netinfo_cell_add_my_addrs(netinfo_cell,
+                              netinfo_addr_from_tor_addr(&my_addr));
 
-/** Add an Ed25519 cert from <b>cert</b> to the trunnel certs_cell_t object
- * that we are building in <b>certs_cell</b>.  Set its type field to
- * <b>cert_type</b>. (If <b>cert</b> is NULL, take no action.) */
-static void
-add_ed25519_cert(certs_cell_t *certs_cell,
-                 uint8_t cert_type,
-                 const tor_cert_t *cert)
-{
-  if (NULL == cert)
-    return;
-
-  add_certs_cell_cert_helper(certs_cell, cert_type,
-                             cert->encoded, cert->encoded_len);
-}
-
-#ifdef TOR_UNIT_TESTS
-int certs_cell_ed25519_disabled_for_testing = 0;
-#else
-#define certs_cell_ed25519_disabled_for_testing 0
-#endif
-
-/** Send a CERTS cell on the connection <b>conn</b>.  Return 0 on success, -1
- * on failure. */
-int
-connection_or_send_certs_cell(or_connection_t *conn)
-{
-  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL;
-  tor_x509_cert_t *own_link_cert = NULL;
-  var_cell_t *cell;
-
-  certs_cell_t *certs_cell = NULL;
-
-  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
-
-  if (! conn->handshake_state)
-    return -1;
-
-  const int conn_in_server_mode = ! conn->handshake_state->started_here;
-
-  /* Get the encoded values of the X509 certificates */
-  if (tor_tls_get_my_certs(conn_in_server_mode,
-                           &global_link_cert, &id_cert) < 0)
-    return -1;
-
-  if (conn_in_server_mode) {
-    own_link_cert = tor_tls_get_own_cert(conn->tls);
-  }
-  tor_assert(id_cert);
-
-  certs_cell = certs_cell_new();
-
-  /* Start adding certs.  First the link cert or auth1024 cert. */
-  if (conn_in_server_mode) {
-    tor_assert_nonfatal(own_link_cert);
-    add_x509_cert(certs_cell,
-                  OR_CERT_TYPE_TLS_LINK, own_link_cert);
-  } else {
-    tor_assert(global_link_cert);
-    add_x509_cert(certs_cell,
-                  OR_CERT_TYPE_AUTH_1024, global_link_cert);
-  }
-
-  /* Next the RSA->RSA ID cert */
-  add_x509_cert(certs_cell,
-                OR_CERT_TYPE_ID_1024, id_cert);
-
-  /* Next the Ed25519 certs */
-  add_ed25519_cert(certs_cell,
-                   CERTTYPE_ED_ID_SIGN,
-                   get_master_signing_key_cert());
-  if (conn_in_server_mode) {
-    tor_assert_nonfatal(conn->handshake_state->own_link_cert ||
-                        certs_cell_ed25519_disabled_for_testing);
-    add_ed25519_cert(certs_cell,
-                     CERTTYPE_ED_SIGN_LINK,
-                     conn->handshake_state->own_link_cert);
-  } else {
-    add_ed25519_cert(certs_cell,
-                     CERTTYPE_ED_SIGN_AUTH,
-                     get_current_auth_key_cert());
-  }
-
-  /* And finally the crosscert. */
-  {
-    const uint8_t *crosscert=NULL;
-    size_t crosscert_len;
-    get_master_rsa_crosscert(&crosscert, &crosscert_len);
-    if (crosscert) {
-      add_certs_cell_cert_helper(certs_cell,
-                               CERTTYPE_RSA1024_ID_EDID,
-                               crosscert, crosscert_len);
+    if (!tor_addr_is_null(&me->ipv6_addr)) {
+      netinfo_cell_add_my_addrs(netinfo_cell,
+                                netinfo_addr_from_tor_addr(&me->ipv6_addr));
     }
   }
 
-  /* We've added all the certs; make the cell. */
-  certs_cell->n_certs = certs_cell_getlen_certs(certs_cell);
-
-  ssize_t alloc_len = certs_cell_encoded_len(certs_cell);
-  tor_assert(alloc_len >= 0 && alloc_len <= UINT16_MAX);
-  cell = var_cell_new(alloc_len);
-  cell->command = CELL_CERTS;
-  ssize_t enc_len = certs_cell_encode(cell->payload, alloc_len, certs_cell);
-  tor_assert(enc_len > 0 && enc_len <= alloc_len);
-  cell->payload_len = enc_len;
-
-  connection_or_write_var_cell_to_buf(cell, conn);
-  var_cell_free(cell);
-  certs_cell_free(certs_cell);
-  tor_x509_cert_free(own_link_cert);
-
-  return 0;
-}
-
-#ifdef TOR_UNIT_TESTS
-int testing__connection_or_pretend_TLSSECRET_is_supported = 0;
-#else
-#define testing__connection_or_pretend_TLSSECRET_is_supported 0
-#endif
-
-/** Return true iff <b>challenge_type</b> is an AUTHCHALLENGE type that
- * we can send and receive. */
-int
-authchallenge_type_is_supported(uint16_t challenge_type)
-{
-  switch (challenge_type) {
-     case AUTHTYPE_RSA_SHA256_TLSSECRET:
-#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
-       return 1;
-#else
-       return testing__connection_or_pretend_TLSSECRET_is_supported;
-#endif
-     case AUTHTYPE_ED25519_SHA256_RFC5705:
-       return 1;
-     case AUTHTYPE_RSA_SHA256_RFC5705:
-     default:
-       return 0;
+  const char *errmsg = NULL;
+  if ((errmsg = netinfo_cell_check(netinfo_cell))) {
+    log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s",
+                    errmsg);
+    goto cleanup;
   }
-}
-
-/** Return true iff <b>challenge_type_a</b> is one that we would rather
- * use than <b>challenge_type_b</b>. */
-int
-authchallenge_type_is_better(uint16_t challenge_type_a,
-                             uint16_t challenge_type_b)
-{
-  /* Any supported type is better than an unsupported one;
-   * all unsupported types are equally bad. */
-  if (!authchallenge_type_is_supported(challenge_type_a))
-    return 0;
-  if (!authchallenge_type_is_supported(challenge_type_b))
-    return 1;
-  /* It happens that types are superior in numerically ascending order.
-   * If that ever changes, this must change too. */
-  return (challenge_type_a > challenge_type_b);
-}
-
-/** Send an AUTH_CHALLENGE cell on the connection <b>conn</b>. Return 0
- * on success, -1 on failure. */
-int
-connection_or_send_auth_challenge_cell(or_connection_t *conn)
-{
-  var_cell_t *cell = NULL;
-  int r = -1;
-  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
-
-  if (! conn->handshake_state)
-    return -1;
-
-  auth_challenge_cell_t *ac = auth_challenge_cell_new();
 
-  tor_assert(sizeof(ac->challenge) == 32);
-  crypto_rand((char*)ac->challenge, sizeof(ac->challenge));
-
-  if (authchallenge_type_is_supported(AUTHTYPE_RSA_SHA256_TLSSECRET))
-    auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
-  /* Disabled, because everything that supports this method also supports
-   * the much-superior ED25519_SHA256_RFC5705 */
-  /* auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_RFC5705); */
-  if (authchallenge_type_is_supported(AUTHTYPE_ED25519_SHA256_RFC5705))
-    auth_challenge_cell_add_methods(ac, AUTHTYPE_ED25519_SHA256_RFC5705);
-  auth_challenge_cell_set_n_methods(ac,
-                                    auth_challenge_cell_getlen_methods(ac));
-
-  cell = var_cell_new(auth_challenge_cell_encoded_len(ac));
-  ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len,
-                                           ac);
-  if (len != cell->payload_len) {
-    /* LCOV_EXCL_START */
-    log_warn(LD_BUG, "Encoded auth challenge cell length not as expected");
-    goto done;
-    /* LCOV_EXCL_STOP */
+  if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                          netinfo_cell) < 0) {
+    log_warn(LD_OR, "Failed generating NETINFO cell");
+    goto cleanup;
   }
-  cell->command = CELL_AUTH_CHALLENGE;
 
-  connection_or_write_var_cell_to_buf(cell, conn);
-  r = 0;
+  conn->handshake_state->digest_sent_data = 0;
+  conn->handshake_state->sent_netinfo = 1;
+  connection_or_write_cell_to_buf(&cell, conn);
 
- done:
-  var_cell_free(cell);
-  auth_challenge_cell_free(ac);
+  r = 0;
+ cleanup:
+  netinfo_cell_free(netinfo_cell);
 
   return r;
 }
-
-/** Compute the main body of an AUTHENTICATE cell that a client can use
- * to authenticate itself on a v3 handshake for <b>conn</b>.  Return it
- * in a var_cell_t.
- *
- * If <b>server</b> is true, only calculate the first
- * V3_AUTH_FIXED_PART_LEN bytes -- the part of the authenticator that's
- * determined by the rest of the handshake, and which match the provided value
- * exactly.
- *
- * If <b>server</b> is false and <b>signing_key</b> is NULL, calculate the
- * first V3_AUTH_BODY_LEN bytes of the authenticator (that is, everything
- * that should be signed), but don't actually sign it.
- *
- * If <b>server</b> is false and <b>signing_key</b> is provided, calculate the
- * entire authenticator, signed with <b>signing_key</b>.
- *
- * Return the length of the cell body on success, and -1 on failure.
- */
-var_cell_t *
-connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                             const int authtype,
-                                             crypto_pk_t *signing_key,
-                                      const ed25519_keypair_t *ed_signing_key,
-                                      int server)
-{
-  auth1_t *auth = NULL;
-  auth_ctx_t *ctx = auth_ctx_new();
-  var_cell_t *result = NULL;
-  int old_tlssecrets_algorithm = 0;
-  const char *authtype_str = NULL;
-
-  int is_ed = 0;
-
-  /* assert state is reasonable XXXX */
-  switch (authtype) {
-  case AUTHTYPE_RSA_SHA256_TLSSECRET:
-    authtype_str = "AUTH0001";
-    old_tlssecrets_algorithm = 1;
-    break;
-  case AUTHTYPE_RSA_SHA256_RFC5705:
-    authtype_str = "AUTH0002";
-    break;
-  case AUTHTYPE_ED25519_SHA256_RFC5705:
-    authtype_str = "AUTH0003";
-    is_ed = 1;
-    break;
-  default:
-    tor_assert(0);
-    break;
-  }
-
-  auth = auth1_new();
-  ctx->is_ed = is_ed;
-
-  /* Type: 8 bytes. */
-  memcpy(auth1_getarray_type(auth), authtype_str, 8);
-
-  {
-    const tor_x509_cert_t *id_cert=NULL;
-    const common_digests_t *my_digests, *their_digests;
-    const uint8_t *my_id, *their_id, *client_id, *server_id;
-    if (tor_tls_get_my_certs(server, NULL, &id_cert))
-      goto err;
-    my_digests = tor_x509_cert_get_id_digests(id_cert);
-    their_digests =
-      tor_x509_cert_get_id_digests(conn->handshake_state->certs->id_cert);
-    tor_assert(my_digests);
-    tor_assert(their_digests);
-    my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
-    their_id = (uint8_t*)their_digests->d[DIGEST_SHA256];
-
-    client_id = server ? their_id : my_id;
-    server_id = server ? my_id : their_id;
-
-    /* Client ID digest: 32 octets. */
-    memcpy(auth->cid, client_id, 32);
-
-    /* Server ID digest: 32 octets. */
-    memcpy(auth->sid, server_id, 32);
-  }
-
-  if (is_ed) {
-    const ed25519_public_key_t *my_ed_id, *their_ed_id;
-    if (!conn->handshake_state->certs->ed_id_sign) {
-      log_warn(LD_OR, "Ed authenticate without Ed ID cert from peer.");
-      goto err;
-    }
-    my_ed_id = get_master_identity_key();
-    their_ed_id = &conn->handshake_state->certs->ed_id_sign->signing_key;
-
-    const uint8_t *cid_ed = (server ? their_ed_id : my_ed_id)->pubkey;
-    const uint8_t *sid_ed = (server ? my_ed_id : their_ed_id)->pubkey;
-
-    memcpy(auth->u1_cid_ed, cid_ed, ED25519_PUBKEY_LEN);
-    memcpy(auth->u1_sid_ed, sid_ed, ED25519_PUBKEY_LEN);
-  }
-
-  {
-    crypto_digest_t *server_d, *client_d;
-    if (server) {
-      server_d = conn->handshake_state->digest_sent;
-      client_d = conn->handshake_state->digest_received;
-    } else {
-      client_d = conn->handshake_state->digest_sent;
-      server_d = conn->handshake_state->digest_received;
-    }
-
-    /* Server log digest : 32 octets */
-    crypto_digest_get_digest(server_d, (char*)auth->slog, 32);
-
-    /* Client log digest : 32 octets */
-    crypto_digest_get_digest(client_d, (char*)auth->clog, 32);
-  }
-
-  {
-    /* Digest of cert used on TLS link : 32 octets. */
-    tor_x509_cert_t *cert = NULL;
-    if (server) {
-      cert = tor_tls_get_own_cert(conn->tls);
-    } else {
-      cert = tor_tls_get_peer_cert(conn->tls);
-    }
-    if (!cert) {
-      log_warn(LD_OR, "Unable to find cert when making %s data.",
-               authtype_str);
-      goto err;
-    }
-
-    memcpy(auth->scert,
-           tor_x509_cert_get_cert_digests(cert)->d[DIGEST_SHA256], 32);
-
-    tor_x509_cert_free(cert);
-  }
-
-  /* HMAC of clientrandom and serverrandom using master key : 32 octets */
-  if (old_tlssecrets_algorithm) {
-    if (tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets) < 0) {
-      log_fn(LOG_PROTOCOL_WARN, LD_OR, "Somebody asked us for an older TLS "
-         "authentication method (AUTHTYPE_RSA_SHA256_TLSSECRET) "
-         "which we don't support.");
-    }
-  } else {
-    char label[128];
-    tor_snprintf(label, sizeof(label),
-                 "EXPORTER FOR TOR TLS CLIENT BINDING %s", authtype_str);
-    int r = tor_tls_export_key_material(conn->tls, auth->tlssecrets,
-                                        auth->cid, sizeof(auth->cid),
-                                        label);
-    if (r < 0) {
-      if (r != -2)
-        log_warn(LD_BUG, "TLS key export failed for unknown reason.");
-      // If r == -2, this was openssl bug 7712.
-      goto err;
-    }
-  }
-
-  /* 8 octets were reserved for the current time, but we're trying to get out
-   * of the habit of sending time around willynilly.  Fortunately, nothing
-   * checks it.  That's followed by 16 bytes of nonce. */
-  crypto_rand((char*)auth->rand, 24);
-
-  ssize_t maxlen = auth1_encoded_len(auth, ctx);
-  if (ed_signing_key && is_ed) {
-    maxlen += ED25519_SIG_LEN;
-  } else if (signing_key && !is_ed) {
-    maxlen += crypto_pk_keysize(signing_key);
-  }
-
-  const int AUTH_CELL_HEADER_LEN = 4; /* 2 bytes of type, 2 bytes of length */
-  result = var_cell_new(AUTH_CELL_HEADER_LEN + maxlen);
-  uint8_t *const out = result->payload + AUTH_CELL_HEADER_LEN;
-  const size_t outlen = maxlen;
-  ssize_t len;
-
-  result->command = CELL_AUTHENTICATE;
-  set_uint16(result->payload, htons(authtype));
-
-  if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) {
-    /* LCOV_EXCL_START */
-    log_warn(LD_BUG, "Unable to encode signed part of AUTH1 data.");
-    goto err;
-    /* LCOV_EXCL_STOP */
-  }
-
-  if (server) {
-    auth1_t *tmp = NULL;
-    ssize_t len2 = auth1_parse(&tmp, out, len, ctx);
-    if (!tmp) {
-      /* LCOV_EXCL_START */
-      log_warn(LD_BUG, "Unable to parse signed part of AUTH1 data that "
-               "we just encoded");
-      goto err;
-      /* LCOV_EXCL_STOP */
-    }
-    result->payload_len = (tmp->end_of_signed - result->payload);
-
-    auth1_free(tmp);
-    if (len2 != len) {
-      /* LCOV_EXCL_START */
-      log_warn(LD_BUG, "Mismatched length when re-parsing AUTH1 data.");
-      goto err;
-      /* LCOV_EXCL_STOP */
-    }
-    goto done;
-  }
-
-  if (ed_signing_key && is_ed) {
-    ed25519_signature_t sig;
-    if (ed25519_sign(&sig, out, len, ed_signing_key) < 0) {
-      /* LCOV_EXCL_START */
-      log_warn(LD_BUG, "Unable to sign ed25519 authentication data");
-      goto err;
-      /* LCOV_EXCL_STOP */
-    }
-    auth1_setlen_sig(auth, ED25519_SIG_LEN);
-    memcpy(auth1_getarray_sig(auth), sig.sig, ED25519_SIG_LEN);
-
-  } else if (signing_key && !is_ed) {
-    auth1_setlen_sig(auth, crypto_pk_keysize(signing_key));
-
-    char d[32];
-    crypto_digest256(d, (char*)out, len, DIGEST_SHA256);
-    int siglen = crypto_pk_private_sign(signing_key,
-                                    (char*)auth1_getarray_sig(auth),
-                                    auth1_getlen_sig(auth),
-                                    d, 32);
-    if (siglen < 0) {
-      log_warn(LD_OR, "Unable to sign AUTH1 data.");
-      goto err;
-    }
-
-    auth1_setlen_sig(auth, siglen);
-  }
-
-  len = auth1_encode(out, outlen, auth, ctx);
-  if (len < 0) {
-    /* LCOV_EXCL_START */
-    log_warn(LD_BUG, "Unable to encode signed AUTH1 data.");
-    goto err;
-    /* LCOV_EXCL_STOP */
-  }
-  tor_assert(len + AUTH_CELL_HEADER_LEN <= result->payload_len);
-  result->payload_len = len + AUTH_CELL_HEADER_LEN;
-  set_uint16(result->payload+2, htons(len));
-
-  goto done;
-
- err:
-  var_cell_free(result);
-  result = NULL;
- done:
-  auth1_free(auth);
-  auth_ctx_free(ctx);
-  return result;
-}
-
-/** Send an AUTHENTICATE cell on the connection <b>conn</b>.  Return 0 on
- * success, -1 on failure */
-MOCK_IMPL(int,
-connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype))
-{
-  var_cell_t *cell;
-  crypto_pk_t *pk = tor_tls_get_my_client_auth_key();
-  /* XXXX make sure we're actually supposed to send this! */
-
-  if (!pk) {
-    log_warn(LD_BUG, "Can't compute authenticate cell: no client auth key");
-    return -1;
-  }
-  if (! authchallenge_type_is_supported(authtype)) {
-    log_warn(LD_BUG, "Tried to send authenticate cell with unknown "
-             "authentication type %d", authtype);
-    return -1;
-  }
-
-  cell = connection_or_compute_authenticate_cell_body(conn,
-                                                 authtype,
-                                                 pk,
-                                                 get_current_auth_keypair(),
-                                                 0 /* not server */);
-  if (! cell) {
-    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unable to compute authenticate cell!");
-    return -1;
-  }
-  connection_or_write_var_cell_to_buf(cell, conn);
-  var_cell_free(cell);
-
-  return 0;
-}
diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h
index 817bcdd317..e9ace56ab4 100644
--- a/src/core/or/connection_or.h
+++ b/src/core/or/connection_or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,40 +17,11 @@ struct ed25519_keypair_t;
 
 or_connection_t *TO_OR_CONN(connection_t *);
 
-#define OR_CONN_STATE_MIN_ 1
-/** State for a connection to an OR: waiting for connect() to finish. */
-#define OR_CONN_STATE_CONNECTING 1
-/** State for a connection to an OR: waiting for proxy handshake to complete */
-#define OR_CONN_STATE_PROXY_HANDSHAKING 2
-/** State for an OR connection client: SSL is handshaking, not done
- * yet. */
-#define OR_CONN_STATE_TLS_HANDSHAKING 3
-/** State for a connection to an OR: We're doing a second SSL handshake for
- * renegotiation purposes. (V2 handshake only.) */
-#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
-/** State for a connection at an OR: We're waiting for the client to
- * renegotiate (to indicate a v2 handshake) or send a versions cell (to
- * indicate a v3 handshake) */
-#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
-/** State for an OR connection: We're done with our SSL handshake, we've done
- * renegotiation, but we haven't yet negotiated link protocol versions and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
-/** State for an OR connection: We're done with our SSL handshake, but we
- * haven't yet negotiated link protocol versions, done a V3 handshake, and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
-/** State for an OR connection: Ready to send/receive cells. */
-#define OR_CONN_STATE_OPEN 8
-#define OR_CONN_STATE_MAX_ 8
+#include "core/or/orconn_event.h"
 
 void connection_or_clear_identity(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
 void clear_broken_connection_map(int disable);
-or_connection_t *connection_or_get_for_extend(const char *digest,
-                                              const tor_addr_t *target_addr,
-                                              const char **msg_out,
-                                              int *launch_out);
 
 void connection_or_block_renegotiation(or_connection_t *conn);
 int connection_or_reached_eof(or_connection_t *conn);
@@ -81,6 +52,9 @@ MOCK_DECL(void,connection_or_close_for_error,
 
 void connection_or_report_broken_states(int severity, int domain);
 
+void connection_or_event_status(or_connection_t *conn,
+                                or_conn_status_event_t tp, int reason);
+
 MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn,
                                               int receiving));
 int connection_tls_continue_handshake(or_connection_t *conn);
@@ -119,19 +93,6 @@ MOCK_DECL(void,connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
                                                    or_connection_t *conn));
 int connection_or_send_versions(or_connection_t *conn, int v3_plus);
 MOCK_DECL(int,connection_or_send_netinfo,(or_connection_t *conn));
-int connection_or_send_certs_cell(or_connection_t *conn);
-int connection_or_send_auth_challenge_cell(or_connection_t *conn);
-int authchallenge_type_is_supported(uint16_t challenge_type);
-int authchallenge_type_is_better(uint16_t challenge_type_a,
-                                 uint16_t challenge_type_b);
-var_cell_t *connection_or_compute_authenticate_cell_body(
-                              or_connection_t *conn,
-                              const int authtype,
-                              crypto_pk_t *signing_key,
-                              const struct ed25519_keypair_t *ed_signing_key,
-                              int server);
-MOCK_DECL(int,connection_or_send_authenticate_cell,
-          (or_connection_t *conn, int type));
 
 int is_or_protocol_version_known(uint16_t version);
 
@@ -156,10 +117,16 @@ void connection_or_group_set_badness_(smartlist_t *group, int force);
 #ifdef CONNECTION_OR_PRIVATE
 STATIC int should_connect_to_relay(const or_connection_t *or_conn);
 STATIC void note_or_connect_failed(const or_connection_t *or_conn);
-#endif
+
+/*
+ * Call this when changing connection state, so notifications to the owning
+ * channel can be handled.
+ */
+MOCK_DECL(STATIC void,connection_or_change_state,
+          (or_connection_t *conn, uint8_t state));
+#endif /* defined(CONNECTION_OR_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
-extern int certs_cell_ed25519_disabled_for_testing;
 extern int testing__connection_or_pretend_TLSSECRET_is_supported;
 #endif
 
diff --git a/src/core/or/connection_st.h b/src/core/or/connection_st.h
index c197a81340..685c9f89f4 100644
--- a/src/core/or/connection_st.h
+++ b/src/core/or/connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file connection_st.h
+ * @brief Base connection structure.
+ **/
+
 #ifndef CONNECTION_ST_H
 #define CONNECTION_ST_H
 
@@ -149,4 +154,4 @@ struct connection_t {
  * directory connection. */
 #define DIR_CONN_IS_SERVER(conn) ((conn)->purpose == DIR_PURPOSE_SERVER)
 
-#endif
+#endif /* !defined(CONNECTION_ST_H) */
diff --git a/src/core/or/core_or.md b/src/core/or/core_or.md
new file mode 100644
index 0000000000..0b4d430a00
--- /dev/null
+++ b/src/core/or/core_or.md
@@ -0,0 +1,62 @@
+@dir core/or
+@brief core/or: **Onion routing happens here!**
+
+This is the central part of Tor that handles the core tasks of onion routing:
+building circuit, handling circuits, attaching circuit to streams, moving
+data around, and so forth.
+
+Some aspects of this module should probably be refactored into others.
+
+Notable files here include:
+
+`channel.c`
+: Generic channel implementation. Channels handle sending and receiving cells
+among tor nodes.
+
+`channeltls.c`
+: Channel implementation for TLS-based OR connections. Uses `connection_or.c`.
+
+`circuitbuild.c`
+: Code for constructing circuits and choosing their paths.  (*Note*:
+this module could plausibly be split into handling the client side,
+the server side, and the path generation aspects of circuit building.)
+
+`circuitlist.c`
+: Code for maintaining and navigating the global list of circuits.
+
+`circuitmux.c`
+: Generic circuitmux implementation. A circuitmux handles deciding, for a
+particular channel, which circuit should write next.
+
+`circuitmux_ewma.c`
+: A circuitmux implementation based on the EWMA (exponentially
+weighted moving average) algorithm.
+
+`circuituse.c`
+: Code to actually send and receive data on circuits.
+
+`command.c`
+: Handles incoming cells on channels.
+
+`connection.c`
+: Generic and common connection tools, and implementation for the simpler
+connection types.
+
+`connection_edge.c`
+: Implementation for entry and exit connections.
+
+`connection_or.c`
+: Implementation for OR connections (the ones that send cells over TLS).
+
+`onion.c`
+: Generic code for generating and responding to CREATE and CREATED
+cells, and performing the appropriate onion handshakes. Also contains
+code to manage the server-side onion queue.
+
+`relay.c`
+: Handles particular types of relay cells, and provides code to receive,
+encrypt, route, and interpret relay cells.
+
+`scheduler.c`
+: Decides which channel/circuit pair is ready to receive the next cell.
+
diff --git a/src/core/or/cpath_build_state_st.h b/src/core/or/cpath_build_state_st.h
index dbe596d851..ee9a0d972c 100644
--- a/src/core/or/cpath_build_state_st.h
+++ b/src/core/or/cpath_build_state_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cpath_build_state_st.h
+ * @brief Circuit-build-stse structure
+ **/
+
 #ifndef CIRCUIT_BUILD_STATE_ST_ST_H
 #define CIRCUIT_BUILD_STATE_ST_ST_H
 
@@ -34,5 +39,4 @@ struct cpath_build_state_t {
   time_t expiry_time;
 };
 
-#endif
-
+#endif /* !defined(CIRCUIT_BUILD_STATE_ST_ST_H) */
diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c
new file mode 100644
index 0000000000..8f41540848
--- /dev/null
+++ b/src/core/or/crypt_path.c
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypt_path.c
+ *
+ * \brief Functions dealing with layered circuit encryption. This file aims to
+ *   provide an API around the crypt_path_t structure which holds crypto
+ *   information about a specific hop of a circuit.
+ *
+ * TODO: We should eventually move all functions dealing and manipulating
+ *   crypt_path_t to this file, so that eventually we encapsulate more and more
+ *   of crypt_path_t. Here are some more functions that can be moved here with
+ *   some more effort:
+ *
+ *   - circuit_list_path_impl()
+ *   - Functions dealing with cpaths in HSv2 create_rend_cpath() and
+ *     create_rend_cpath_legacy()
+ *   - The cpath related parts of rend_service_receive_introduction() and
+ *     rend_client_send_introduction().
+ **/
+
+#define CRYPT_PATH_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/crypt_path.h"
+
+#include "core/crypto/relay_crypto.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/cell_st.h"
+
+/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
+ * This function is used to extend cpath by another hop.
+ */
+void
+cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop)
+{
+  if (*head_ptr) {
+    new_hop->next = (*head_ptr);
+    new_hop->prev = (*head_ptr)->prev;
+    (*head_ptr)->prev->next = new_hop;
+    (*head_ptr)->prev = new_hop;
+  } else {
+    *head_ptr = new_hop;
+    new_hop->prev = new_hop->next = new_hop;
+  }
+}
+
+/** Create a new hop, annotate it with information about its
+ * corresponding router <b>choice</b>, and append it to the
+ * end of the cpath <b>head_ptr</b>. */
+int
+cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
+{
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+
+  /* link hop into the cpath, at the end. */
+  cpath_extend_linked_list(head_ptr, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_CLOSED;
+
+  hop->extend_info = extend_info_dup(choice);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  return 0;
+}
+
+/** Verify that cpath <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_ok(const crypt_path_t *cp)
+{
+  const crypt_path_t *start = cp;
+
+  do {
+    cpath_assert_layer_ok(cp);
+    /* layers must be in sequence of: "open* awaiting? closed*" */
+    if (cp != start) {
+      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      } else if (cp->state == CPATH_STATE_OPEN) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      }
+    }
+    cp = cp->next;
+    tor_assert(cp);
+  } while (cp != start);
+}
+
+/** Verify that cpath layer <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_layer_ok(const crypt_path_t *cp)
+{
+//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
+//  tor_assert(cp->port);
+  tor_assert(cp);
+  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
+  switch (cp->state)
+    {
+    case CPATH_STATE_OPEN:
+      relay_crypto_assert_ok(&cp->pvt_crypto);
+      FALLTHROUGH;
+    case CPATH_STATE_CLOSED:
+      /*XXXX Assert that there's no handshake_state either. */
+      tor_assert(!cp->rend_dh_handshake_state);
+      break;
+    case CPATH_STATE_AWAITING_KEYS:
+      /* tor_assert(cp->dh_handshake_state); */
+      break;
+    default:
+      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
+      tor_assert(0);
+    }
+  tor_assert(cp->package_window >= 0);
+  tor_assert(cp->deliver_window >= 0);
+}
+
+/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
+ *   - 20 to initialize f_digest
+ *   - 20 to initialize b_digest
+ *   - 16 to key f_crypto
+ *   - 16 to key b_crypto
+ *
+ * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
+ */
+int
+cpath_init_circuit_crypto(crypt_path_t *cpath,
+                          const char *key_data, size_t key_data_len,
+                          int reverse, int is_hs_v3)
+{
+
+  tor_assert(cpath);
+  return relay_crypto_init(&cpath->pvt_crypto, key_data, key_data_len,
+                           reverse, is_hs_v3);
+}
+
+/** Deallocate space associated with the cpath node <b>victim</b>. */
+void
+cpath_free(crypt_path_t *victim)
+{
+  if (!victim)
+    return;
+
+  relay_crypto_clear(&victim->pvt_crypto);
+  onion_handshake_state_release(&victim->handshake_state);
+  crypto_dh_free(victim->rend_dh_handshake_state);
+  extend_info_free(victim->extend_info);
+
+  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
+  tor_free(victim);
+}
+
+/********************** cpath crypto API *******************************/
+
+/** Encrypt or decrypt <b>payload</b> using the crypto of <b>cpath</b>. Actual
+ *  operation decided by <b>is_decrypt</b>.  */
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt)
+{
+  if (is_decrypt) {
+    relay_crypt_one_payload(cpath->pvt_crypto.b_crypto, payload);
+  } else {
+    relay_crypt_one_payload(cpath->pvt_crypto.f_crypto, payload);
+  }
+}
+
+/** Getter for the incoming digest of <b>cpath</b>. */
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath)
+{
+  return cpath->pvt_crypto.b_digest;
+}
+
+/** Set the right integrity digest on the outgoing <b>cell</b> based on the
+ *  cell payload and update the forward digest of <b>cpath</b>. */
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell)
+{
+  relay_set_digest(cpath->pvt_crypto.f_digest, cell);
+}
+
+/************ cpath sendme API ***************************/
+
+/** Return the sendme_digest of this <b>cpath</b>. */
+uint8_t *
+cpath_get_sendme_digest(crypt_path_t *cpath)
+{
+  return relay_crypto_get_sendme_digest(&cpath->pvt_crypto);
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+cpath_sendme_record_cell_digest(crypt_path_t *cpath, bool is_foward_digest)
+{
+  tor_assert(cpath);
+  relay_crypto_record_sendme_digest(&cpath->pvt_crypto, is_foward_digest);
+}
+
+/************ other cpath functions ***************************/
+
+/** Return the first non-open hop in cpath, or return NULL if all
+ * hops are open. */
+crypt_path_t *
+cpath_get_next_non_open_hop(crypt_path_t *cpath)
+{
+  crypt_path_t *hop = cpath;
+  do {
+    if (hop->state != CPATH_STATE_OPEN)
+      return hop;
+    hop = hop->next;
+  } while (hop != cpath);
+  return NULL;
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/** Unittest helper function: Count number of hops in cpath linked list. */
+unsigned int
+cpath_get_n_hops(crypt_path_t **head_ptr)
+{
+  unsigned int n_hops = 0;
+  crypt_path_t *tmp;
+
+  if (!*head_ptr) {
+    return 0;
+  }
+
+  tmp = *head_ptr;
+  do {
+    n_hops++;
+    tmp = tmp->next;
+  } while (tmp != *head_ptr);
+
+  return n_hops;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h
new file mode 100644
index 0000000000..7a95fec2b4
--- /dev/null
+++ b/src/core/or/crypt_path.h
@@ -0,0 +1,46 @@
+/**
+ * \file crypt_path.h
+ * \brief Header file for crypt_path.c.
+ **/
+
+#ifndef CRYPT_PATH_H
+#define CRYPT_PATH_H
+
+void cpath_assert_layer_ok(const crypt_path_t *cp);
+
+void cpath_assert_ok(const crypt_path_t *cp);
+
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+
+int cpath_init_circuit_crypto(crypt_path_t *cpath,
+                              const char *key_data, size_t key_data_len,
+                              int reverse, int is_hs_v3);
+
+void
+cpath_free(crypt_path_t *victim);
+
+void cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop);
+
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt);
+
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath);
+
+void cpath_sendme_record_cell_digest(crypt_path_t *cpath,
+                                     bool is_foward_digest);
+
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell);
+
+crypt_path_t *cpath_get_next_non_open_hop(crypt_path_t *cpath);
+
+void cpath_sendme_circuit_record_inbound_cell(crypt_path_t *cpath);
+
+uint8_t *cpath_get_sendme_digest(crypt_path_t *cpath);
+
+#if defined(TOR_UNIT_TESTS)
+unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(CRYPT_PATH_H) */
diff --git a/src/core/or/crypt_path_reference_st.h b/src/core/or/crypt_path_reference_st.h
index 3d79f26c1c..71f9cb8c36 100644
--- a/src/core/or/crypt_path_reference_st.h
+++ b/src/core/or/crypt_path_reference_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file crypt_path_reference_st.h
+ * @brief reference-counting structure for crypt_path_t
+ **/
+
 #ifndef CRYPT_PATH_REFERENCE_ST_H
 #define CRYPT_PATH_REFERENCE_ST_H
 
@@ -19,5 +24,4 @@ struct crypt_path_reference_t {
   crypt_path_t *cpath;
 };
 
-#endif
-
+#endif /* !defined(CRYPT_PATH_REFERENCE_ST_H) */
diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h
index 429480f8ab..2b69728a6d 100644
--- a/src/core/or/crypt_path_st.h
+++ b/src/core/or/crypt_path_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file crypt_path_st.h
+ * @brief Path structures for origin circuits.
+ **/
+
 #ifndef CRYPT_PATH_ST_H
 #define CRYPT_PATH_ST_H
 
@@ -24,15 +29,24 @@ struct onion_handshake_state_t {
   } u;
 };
 
+/** Macro to encapsulate private members of a struct.
+ *
+ *  Renames 'x' to 'x_crypt_path_private_field'.
+ */
+#define CRYPT_PATH_PRIV_FIELD(x) x ## _crypt_path_private_field
+
+#ifdef CRYPT_PATH_PRIVATE
+
+/* Helper macro to access private members of a struct. */
+#define pvt_crypto CRYPT_PATH_PRIV_FIELD(crypto)
+
+#endif /* defined(CRYPT_PATH_PRIVATE) */
+
 /** Holds accounting information for a single step in the layered encryption
  * performed by a circuit.  Used only at the client edge of a circuit. */
 struct crypt_path_t {
   uint32_t magic;
 
-  /** Cryptographic state used for encrypting and authenticating relay
-   * cells to and from this hop. */
-  relay_crypto_t crypto;
-
   /** Current state of the handshake as performed with the OR at this
    * step. */
   onion_handshake_state_t handshake_state;
@@ -65,6 +79,12 @@ struct crypt_path_t {
                        * at this step? */
   int deliver_window; /**< How many cells are we willing to deliver originating
                        * at this step? */
+
+  /*********************** Private members ****************************/
+
+  /** Private member: Cryptographic state used for encrypting and
+   * authenticating relay cells to and from this hop. */
+  relay_crypto_t CRYPT_PATH_PRIV_FIELD(crypto);
 };
 
-#endif
+#endif /* !defined(CRYPT_PATH_ST_H) */
diff --git a/src/core/or/dataflow.md b/src/core/or/dataflow.md
new file mode 100644
index 0000000000..1098701780
--- /dev/null
+++ b/src/core/or/dataflow.md
@@ -0,0 +1,236 @@
+@tableofcontents
+
+@page dataflow Data flow in the Tor process
+
+We read bytes from the network, we write bytes to the network.  For the
+most part, the bytes we write correspond roughly to bytes we have read,
+with bits of cryptography added in.
+
+The rest is a matter of details.
+
+### Connections and buffers: reading, writing, and interpreting.
+
+At a low level, Tor's networking code is based on "connections".  Each
+connection represents an object that can send or receive network-like
+events.  For the most part, each connection has a single underlying TCP
+stream (I'll discuss counterexamples below).
+
+A connection that behaves like a TCP stream has an input buffer and an
+output buffer.  Incoming data is
+written into the input buffer ("inbuf"); data to be written to the
+network is queued on an output buffer ("outbuf").
+
+Buffers are implemented in buffers.c.  Each of these buffers is
+implemented as a linked queue of memory extents, in the style of classic
+BSD mbufs, or Linux skbufs.
+
+A connection's reading and writing can be enabled or disabled.  Under
+the hood, this functionality is implemented using libevent events: one
+for reading, one for writing.  These events are turned on/off in
+main.c, in the functions connection_{start,stop}_{reading,writing}.
+
+When a read or write event is turned on, the main libevent loop polls
+the kernel, asking which sockets are ready to read or write.  (This
+polling happens in the event_base_loop() call in run_main_loop_once()
+in main.c.)  When libevent finds a socket that's ready to read or write,
+it invokes conn_{read,write}_callback(), also in main.c
+
+These callback functions delegate to connection_handle_read() and
+connection_handle_write() in connection.c, which read or write on the
+network as appropriate, possibly delegating to openssl.
+
+After data is read or written, or other event occurs, these
+connection_handle_read_write() functions call logic functions whose job is
+to respond to the information.  Some examples included:
+
+   * connection_flushed_some() -- called after a connection writes any
+     amount of data from its outbuf.
+   * connection_finished_flushing() -- called when a connection has
+     emptied its outbuf.
+   * connection_finished_connecting() -- called when an in-process connection
+     finishes making a remote connection.
+   * connection_reached_eof() -- called after receiving a FIN from the
+     remote server.
+   * connection_process_inbuf() -- called when more data arrives on
+     the inbuf.
+
+These functions then call into specific implementations depending on
+the type of the connection.  For example, if the connection is an
+edge_connection_t, connection_reached_eof() will call
+connection_edge_reached_eof().
+
+> **Note:** "Also there are bufferevents!"  We have vestigial
+> code for an alternative low-level networking
+> implementation, based on Libevent's evbuffer and bufferevent
+> code.  These two object types take on (most of) the roles of
+> buffers and connections respectively. It isn't working in today's
+> Tor, due to code rot and possible lingering libevent bugs.  More
+> work is needed; it would be good to get this working efficiently
+> again, to have IOCP support on Windows.
+
+
+#### Controlling connections ####
+
+A connection can have reading or writing enabled or disabled for a
+wide variety of reasons, including:
+
+   * Writing is disabled when there is no more data to write
+   * For some connection types, reading is disabled when the inbuf is
+     too full.
+   * Reading/writing is temporarily disabled on connections that have
+     recently read/written enough data up to their bandwidth
+   * Reading is disabled on connections when reading more data from them
+     would require that data to be buffered somewhere else that is
+     already full.
+
+Currently, these conditions are checked in a diffuse set of
+increasingly complex conditional expressions.  In the future, it could
+be helpful to transition to a unified model for handling temporary
+read/write suspensions.
+
+#### Kinds of connections ####
+
+Today Tor has the following connection and pseudoconnection types.
+For the most part, each type of channel has an associated C module
+that implements its underlying logic.
+
+**Edge connections** receive data from and deliver data to points
+outside the onion routing network.  See `connection_edge.c`. They fall into two types:
+
+**Entry connections** are a type of edge connection. They receive data
+from the user running a Tor client, and deliver data to that user.
+They are used to implement SOCKSPort, TransPort, NATDPort, and so on.
+Sometimes they are called "AP" connections for historical reasons (it
+used to stand for "Application Proxy").
+
+**Exit connections** are a type of edge connection. They exist at an
+exit node, and transmit traffic to and from the network.
+
+(Entry connections and exit connections are also used as placeholders
+when performing a remote DNS request; they are not decoupled from the
+notion of "stream" in the Tor protocol. This is implemented partially
+in `connection_edge.c`, and partially in `dnsserv.c` and `dns.c`.)
+
+**OR connections** send and receive Tor cells over TLS, using some
+version of the Tor link protocol.  Their implementation is spread
+across `connection_or.c`, with a bit of logic in `command.c`,
+`relay.c`, and `channeltls.c`.
+
+**Extended OR connections** are a type of OR connection for use on
+bridges using pluggable transports, so that the PT can tell the bridge
+some information about the incoming connection before passing on its
+data.  They are implemented in `ext_orport.c`.
+
+**Directory connections** are server-side or client-side connections
+that implement Tor's HTTP-based directory protocol.  These are
+instantiated using a socket when Tor is making an unencrypted HTTP
+connection.  When Tor is tunneling a directory request over a Tor
+circuit, directory connections are implemented using a linked
+connection pair (see below).  Directory connections are implemented in
+`directory.c`; some of the server-side logic is implemented in
+`dirserver.c`.
+
+**Controller connections** are local connections to a controller
+process implementing the controller protocol from
+control-spec.txt. These are in `control.c`.
+
+**Listener connections** are not stream oriented!  Rather, they wrap a
+listening socket in order to detect new incoming connections.  They
+bypass most of stream logic.  They don't have associated buffers.
+They are implemented in `connection.c`.
+
+![structure hierarchy for connection types](./diagrams/02/02-connection-types.png "structure hierarchy for connection types")
+
+>**Note**: "History Time!" You might occasionally find reference to a couple types of connections
+> which no longer exist in modern Tor.  A *CPUWorker connection*
+>connected the main Tor process to a thread or process used for
+>computation.  (Nowadays we use in-process communication.)  Even more
+>anciently, a *DNSWorker connection* connected the main tor process to
+>a separate thread or process used for running `gethostbyname()` or
+>`getaddrinfo()`.  (Nowadays we use Libevent's evdns facility to
+>perform DNS requests asynchronously.)
+
+#### Linked connections ####
+
+Sometimes two channels are joined together, such that data which the
+Tor process sends on one should immediately be received by the same
+Tor process on the other.  (For example, when Tor makes a tunneled
+directory connection, this is implemented on the client side as a
+directory connection whose output goes, not to the network, but to a
+local entry connection. And when a directory receives a tunnelled
+directory connection, this is implemented as an exit connection whose
+output goes, not to the network, but to a local directory connection.)
+
+The earliest versions of Tor to support linked connections used
+socketpairs for the purpose.  But using socketpairs forced us to copy
+data through kernelspace, and wasted limited file descriptors.  So
+instead, a pair of connections can be linked in-process.  Each linked
+connection has a pointer to the other, such that data written on one
+is immediately readable on the other, and vice versa.
+
+### From connections to channels ###
+
+There's an abstraction layer above OR connections (the ones that
+handle cells) and below cells called **Channels**.  A channel's
+purpose is to transmit authenticated cells from one Tor instance
+(relay or client) to another.
+
+Currently, only one implementation exists: Channel_tls, which sends
+and receiveds cells over a TLS-based OR connection.
+
+Cells are sent on a channel using
+`channel_write_{,packed_,var_}cell()`. Incoming cells arrive on a
+channel from its backend using `channel_queue*_cell()`, and are
+immediately processed using `channel_process_cells()`.
+
+Some cell types are handled below the channel layer, such as those
+that affect handshaking only.  And some others are passed up to the
+generic cross-channel code in `command.c`: cells like `DESTROY` and
+`CREATED` are all trivial to handle.  But relay cells
+require special handling...
+
+### From channels through circuits ###
+
+When a relay cell arrives on an existing circuit, it is handled in
+`circuit_receive_relay_cell()` -- one of the innermost functions in
+Tor.  This function encrypts or decrypts the relay cell as
+appropriate, and decides whether the cell is intended for the current
+hop of the circuit.
+
+If the cell *is* intended for the current hop, we pass it to
+`connection_edge_process_relay_cell()` in `relay.c`, which acts on it
+based on its relay command, and (possibly) queues its data on an
+`edge_connection_t`.
+
+If the cell *is not* intended for the current hop, we queue it for the
+next channel in sequence with `append cell_to_circuit_queue()`.  This
+places the cell on a per-circuit queue for cells headed out on that
+particular channel.
+
+### Sending cells on circuits: the complicated bit.
+
+Relay cells are queued onto circuits from one of two (main) sources:
+reading data from edge connections, and receiving a cell to be relayed
+on a circuit.  Both of these sources place their cells on cell queue:
+each circuit has one cell queue for each direction that it travels.
+
+A naive implementation would skip using cell queues, and instead write
+each outgoing relay cell.  (Tor did this in its earlier versions.)
+But such an approach tends to give poor performance, because it allows
+high-volume circuits to clog channels, and it forces the Tor server to
+send data queued on a circuit even after that circuit has been closed.
+
+So by using queues on each circuit, we can add cells to each channel
+on a just-in-time basis, choosing the cell at each moment based on
+a performance-aware algorithm.
+
+This logic is implemented in two main modules: `scheduler.c` and
+`circuitmux*.c`.  The scheduler code is responsible for determining
+globally, across all channels that could write cells, which one should
+next receive queued cells.  The circuitmux code determines, for all
+of the circuits with queued cells for a channel, which one should
+queue the next cell.
+
+(This logic applies to outgoing relay cells only; incoming relay cells
+are processed as they arrive.)
+
diff --git a/src/core/or/destroy_cell_queue_st.h b/src/core/or/destroy_cell_queue_st.h
index 56630670ba..aa28289be5 100644
--- a/src/core/or/destroy_cell_queue_st.h
+++ b/src/core/or/destroy_cell_queue_st.h
@@ -1,12 +1,19 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file destroy_cell_queue_st.h
+ * @brief Destroy-cell queue structures
+ **/
+
 #ifndef DESTROY_CELL_QUEUE_ST_H
 #define DESTROY_CELL_QUEUE_ST_H
 
+#include "core/or/cell_queue_st.h"
+
 /** A single queued destroy cell. */
 struct destroy_cell_t {
   TOR_SIMPLEQ_ENTRY(destroy_cell_t) next;
@@ -19,9 +26,8 @@ struct destroy_cell_t {
 /** A queue of destroy cells on a channel. */
 struct destroy_cell_queue_t {
   /** Linked list of packed_cell_t */
-  TOR_SIMPLEQ_HEAD(dcell_simpleq, destroy_cell_t) head;
+  TOR_SIMPLEQ_HEAD(dcell_simpleq_t, destroy_cell_t) head;
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
-
+#endif /* !defined(DESTROY_CELL_QUEUE_ST_H) */
diff --git a/src/core/or/dos.c b/src/core/or/dos.c
index d06eaa6d05..5f99280030 100644
--- a/src/core/or/dos.c
+++ b/src/core/or/dos.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
@@ -15,6 +15,7 @@
 #include "core/or/channel.h"
 #include "core/or/connection_or.h"
 #include "core/or/relay.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/relay/routermode.h"
@@ -629,6 +630,7 @@ dos_log_heartbeat(void)
   char *cc_msg = NULL;
   char *single_hop_client_msg = NULL;
   char *circ_stats_msg = NULL;
+  char *hs_dos_intro2_msg = NULL;
 
   /* Stats number coming from relay.c append_cell_to_circuit_queue(). */
   tor_asprintf(&circ_stats_msg,
@@ -654,17 +656,24 @@ dos_log_heartbeat(void)
                  num_single_hop_client_refused);
   }
 
+  /* HS DoS stats. */
+  tor_asprintf(&hs_dos_intro2_msg,
+               " %" PRIu64 " INTRODUCE2 rejected.",
+               hs_dos_get_intro2_rejected_count());
+
   log_notice(LD_HEARTBEAT,
-             "DoS mitigation since startup:%s%s%s%s",
+             "DoS mitigation since startup:%s%s%s%s%s",
              circ_stats_msg,
              (cc_msg != NULL) ? cc_msg : " [cc not enabled]",
              (conn_msg != NULL) ? conn_msg : " [conn not enabled]",
-             (single_hop_client_msg != NULL) ? single_hop_client_msg : "");
+             (single_hop_client_msg != NULL) ? single_hop_client_msg : "",
+             (hs_dos_intro2_msg != NULL) ? hs_dos_intro2_msg : "");
 
   tor_free(conn_msg);
   tor_free(cc_msg);
   tor_free(single_hop_client_msg);
   tor_free(circ_stats_msg);
+  tor_free(hs_dos_intro2_msg);
   return;
 }
 
diff --git a/src/core/or/dos.h b/src/core/or/dos.h
index 058b7afce6..b3eca058b8 100644
--- a/src/core/or/dos.h
+++ b/src/core/or/dos.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
@@ -135,7 +135,7 @@ MOCK_DECL(STATIC unsigned int, get_param_cc_enabled,
 MOCK_DECL(STATIC unsigned int, get_param_conn_enabled,
           (const networkstatus_t *ns));
 
-#endif /* TOR_DOS_PRIVATE */
+#endif /* defined(DOS_PRIVATE) */
 
-#endif /* TOR_DOS_H */
+#endif /* !defined(TOR_DOS_H) */
 
diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h
index 1665b8589f..9b2f031b9d 100644
--- a/src/core/or/edge_connection_st.h
+++ b/src/core/or/edge_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file edge_connection_st.h
+ * @brief Edge-connection structure.
+ **/
+
 #ifndef EDGE_CONNECTION_ST_H
 #define EDGE_CONNECTION_ST_H
 
@@ -73,5 +78,4 @@ struct edge_connection_t {
   uint64_t dirreq_id;
 };
 
-#endif
-
+#endif /* !defined(EDGE_CONNECTION_ST_H) */
diff --git a/src/core/or/entry_connection_st.h b/src/core/or/entry_connection_st.h
index 45621fadbf..495ffd85dd 100644
--- a/src/core/or/entry_connection_st.h
+++ b/src/core/or/entry_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file entry_connection_st.h
+ * @brief Entry connection structure.
+ **/
+
 #ifndef ENTRY_CONNECTION_ST_H
 #define ENTRY_CONNECTION_ST_H
 
@@ -96,5 +101,4 @@ struct entry_connection_t {
 /** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
 #define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
 
-#endif
-
+#endif /* !defined(ENTRY_CONNECTION_ST_H) */
diff --git a/src/core/or/entry_port_cfg_st.h b/src/core/or/entry_port_cfg_st.h
index 87dfb331e5..ef1095086d 100644
--- a/src/core/or/entry_port_cfg_st.h
+++ b/src/core/or/entry_port_cfg_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file entry_port_cfg_st.h
+ * @brief Configuration structure for client ports.
+ **/
+
 #ifndef ENTRY_PORT_CFG_ST_H
 #define ENTRY_PORT_CFG_ST_H
 
@@ -48,7 +53,9 @@ struct entry_port_cfg_t {
    * do we prefer IPv6? */
   unsigned int prefer_ipv6_virtaddr : 1;
 
-};
+  /** For socks listeners: can we send back the extended SOCKS5 error code? */
+  unsigned int extended_socks5_codes : 1;
 
-#endif
+};
 
+#endif /* !defined(ENTRY_PORT_CFG_ST_H) */
diff --git a/src/core/or/extend_info_st.h b/src/core/or/extend_info_st.h
index bc7a77b1b2..a66ce24cfa 100644
--- a/src/core/or/extend_info_st.h
+++ b/src/core/or/extend_info_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file extend_info_st.h
+ * @brief Extend-info structure.
+ **/
+
 #ifndef EXTEND_INFO_ST_H
 #define EXTEND_INFO_ST_H
 
@@ -27,4 +32,4 @@ struct extend_info_t {
   curve25519_public_key_t curve25519_onion_key;
 };
 
-#endif
+#endif /* !defined(EXTEND_INFO_ST_H) */
diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h
index d4617be108..c6b6e518fc 100644
--- a/src/core/or/half_edge_st.h
+++ b/src/core/or/half_edge_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file half_edge_st.h
+ * @brief Half-open connection structure.
+ **/
+
 #ifndef HALF_EDGE_ST_H
 #define HALF_EDGE_ST_H
 
@@ -30,5 +35,4 @@ typedef struct half_edge_t {
   int connected_pending : 1;
 } half_edge_t;
 
-#endif
-
+#endif /* !defined(HALF_EDGE_ST_H) */
diff --git a/src/core/or/include.am b/src/core/or/include.am
new file mode 100644
index 0000000000..3626e76bed
--- /dev/null
+++ b/src/core/or/include.am
@@ -0,0 +1,96 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/or/address_set.c		\
+	src/core/or/channel.c			\
+	src/core/or/channelpadding.c		\
+	src/core/or/channeltls.c		\
+	src/core/or/circuitbuild.c		\
+	src/core/or/circuitlist.c		\
+	src/core/or/circuitmux.c		\
+	src/core/or/circuitmux_ewma.c		\
+	src/core/or/circuitpadding.c		\
+	src/core/or/circuitpadding_machines.c	\
+	src/core/or/circuitstats.c		\
+	src/core/or/circuituse.c		\
+	src/core/or/crypt_path.c		\
+	src/core/or/command.c			\
+	src/core/or/connection_edge.c		\
+	src/core/or/connection_or.c		\
+	src/core/or/dos.c			\
+	src/core/or/onion.c			\
+	src/core/or/ocirc_event.c		\
+	src/core/or/or_periodic.c		\
+	src/core/or/or_sys.c			\
+	src/core/or/orconn_event.c		\
+	src/core/or/policies.c			\
+	src/core/or/protover.c			\
+	src/core/or/protover_rust.c		\
+	src/core/or/reasons.c			\
+	src/core/or/relay.c			\
+	src/core/or/scheduler.c			\
+	src/core/or/scheduler_kist.c		\
+	src/core/or/scheduler_vanilla.c		\
+	src/core/or/sendme.c			\
+	src/core/or/status.c			\
+	src/core/or/versions.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/or/addr_policy_st.h			\
+	src/core/or/address_set.h			\
+	src/core/or/cell_queue_st.h			\
+	src/core/or/cell_st.h				\
+	src/core/or/channel.h				\
+	src/core/or/channelpadding.h			\
+	src/core/or/channeltls.h			\
+	src/core/or/circuit_st.h			\
+	src/core/or/circuitbuild.h			\
+	src/core/or/circuitlist.h			\
+	src/core/or/circuitmux.h			\
+	src/core/or/circuitmux_ewma.h			\
+	src/core/or/circuitstats.h			\
+	src/core/or/circuitpadding.h			\
+	src/core/or/circuitpadding_machines.h		\
+	src/core/or/circuituse.h			\
+	src/core/or/command.h				\
+	src/core/or/connection_edge.h			\
+	src/core/or/connection_or.h			\
+	src/core/or/connection_st.h			\
+	src/core/or/crypt_path.h			\
+	src/core/or/cpath_build_state_st.h		\
+	src/core/or/crypt_path_reference_st.h		\
+	src/core/or/crypt_path_st.h			\
+	src/core/or/destroy_cell_queue_st.h		\
+	src/core/or/dos.h				\
+	src/core/or/edge_connection_st.h		\
+	src/core/or/half_edge_st.h			\
+	src/core/or/entry_connection_st.h		\
+	src/core/or/entry_port_cfg_st.h			\
+	src/core/or/extend_info_st.h			\
+	src/core/or/listener_connection_st.h		\
+	src/core/or/onion.h				\
+	src/core/or/or.h				\
+	src/core/or/or_periodic.h			\
+	src/core/or/or_sys.h				\
+	src/core/or/orconn_event.h			\
+	src/core/or/or_circuit_st.h			\
+	src/core/or/or_connection_st.h			\
+	src/core/or/or_handshake_certs_st.h		\
+	src/core/or/or_handshake_state_st.h		\
+	src/core/or/ocirc_event.h			\
+	src/core/or/origin_circuit_st.h			\
+	src/core/or/policies.h				\
+	src/core/or/port_cfg_st.h			\
+	src/core/or/protover.h				\
+	src/core/or/reasons.h				\
+	src/core/or/relay.h				\
+	src/core/or/relay_crypto_st.h			\
+	src/core/or/scheduler.h				\
+	src/core/or/sendme.h				\
+	src/core/or/server_port_cfg_st.h		\
+	src/core/or/socks_request_st.h			\
+	src/core/or/status.h				\
+	src/core/or/tor_version_st.h			\
+	src/core/or/var_cell_st.h			\
+	src/core/or/versions.h
diff --git a/src/core/or/listener_connection_st.h b/src/core/or/listener_connection_st.h
index 8989a39dc8..78175ea061 100644
--- a/src/core/or/listener_connection_st.h
+++ b/src/core/or/listener_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file listener_connection_st.h
+ * @brief Listener connection structure.
+ **/
+
 #ifndef LISTENER_CONNECTION_ST_H
 #define LISTENER_CONNECTION_ST_H
 
@@ -21,5 +26,4 @@ struct listener_connection_t {
 
 };
 
-#endif
-
+#endif /* !defined(LISTENER_CONNECTION_ST_H) */
diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c
new file mode 100644
index 0000000000..fa16459175
--- /dev/null
+++ b/src/core/or/ocirc_event.c
@@ -0,0 +1,121 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.c
+ * \brief Publish state change messages for origin circuits
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of origin circuits.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/or_sys.h"
+#include "core/or/origin_circuit_st.h"
+#include "lib/subsys/subsys.h"
+
+DECLARE_PUBLISH(ocirc_state);
+DECLARE_PUBLISH(ocirc_chan);
+DECLARE_PUBLISH(ocirc_cevent);
+
+static void
+ocirc_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+ocirc_state_fmt(msg_aux_data_t u)
+{
+  ocirc_state_msg_t *msg = (ocirc_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" state=%d onehop=%d>",
+               msg->gid, msg->state, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_chan_fmt(msg_aux_data_t u)
+{
+  ocirc_chan_msg_t *msg = (ocirc_chan_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" chan=%"PRIu64" onehop=%d>",
+               msg->gid, msg->chan, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_cevent_fmt(msg_aux_data_t u)
+{
+  ocirc_cevent_msg_t *msg = (ocirc_cevent_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" evtype=%d reason=%d onehop=%d>",
+               msg->gid, msg->evtype, msg->reason, msg->onehop);
+  return s;
+}
+
+static dispatch_typefns_t ocirc_state_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_state_fmt,
+};
+
+static dispatch_typefns_t ocirc_chan_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_chan_fmt,
+};
+
+static dispatch_typefns_t ocirc_cevent_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_cevent_fmt,
+};
+
+int
+ocirc_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_state, &ocirc_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_chan, &ocirc_chan_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_cevent, &ocirc_cevent_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_state))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  return 0;
+}
+
+void
+ocirc_state_publish(ocirc_state_msg_t *msg)
+{
+  PUBLISH(ocirc_state, msg);
+}
+
+void
+ocirc_chan_publish(ocirc_chan_msg_t *msg)
+{
+  PUBLISH(ocirc_chan, msg);
+}
+
+void
+ocirc_cevent_publish(ocirc_cevent_msg_t *msg)
+{
+  PUBLISH(ocirc_cevent, msg);
+}
diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h
new file mode 100644
index 0000000000..10307a3664
--- /dev/null
+++ b/src/core/or/ocirc_event.h
@@ -0,0 +1,72 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.h
+ * \brief Header file for ocirc_event.c
+ **/
+
+#ifndef TOR_OCIRC_EVENT_H
+#define TOR_OCIRC_EVENT_H
+
+#include <stdbool.h>
+
+#include "lib/cc/torint.h"
+#include "lib/pubsub/pubsub.h"
+
+/** Used to indicate the type of a circuit event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum circuit_status_event_t {
+  CIRC_EVENT_LAUNCHED = 0,
+  CIRC_EVENT_BUILT    = 1,
+  CIRC_EVENT_EXTENDED = 2,
+  CIRC_EVENT_FAILED   = 3,
+  CIRC_EVENT_CLOSED   = 4,
+} circuit_status_event_t;
+
+/** Message for origin circuit state update */
+typedef struct ocirc_state_msg_t {
+  uint32_t gid;       /**< global ID (only origin circuits have them) */
+  int state;          /**< new circuit state */
+  bool onehop;        /**< one-hop circuit? */
+} ocirc_state_msg_t;
+
+DECLARE_MESSAGE(ocirc_state, ocirc_state, ocirc_state_msg_t *);
+
+/**
+ * Message when a channel gets associated to a circuit.
+ *
+ * This doesn't always correspond to something in circuitbuild.c
+ * setting the n_chan field in the circuit.  For some reason, if
+ * circuit_handle_first_hop() launches a new circuit, it doesn't set
+ * the n_chan field.
+ */
+typedef struct ocirc_chan_msg_t {
+  uint32_t gid;                 /**< global ID */
+  uint64_t chan;                /**< channel ID */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_chan_msg_t;
+
+DECLARE_MESSAGE(ocirc_chan, ocirc_chan, ocirc_chan_msg_t *);
+
+/**
+ * Message for origin circuit status event
+ *
+ * This contains information that ends up in CIRC control protocol events.
+ */
+typedef struct ocirc_cevent_msg_t {
+  uint32_t gid;                 /**< global ID */
+  int evtype;                   /**< event type */
+  int reason;                   /**< reason */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_cevent_msg_t;
+
+DECLARE_MESSAGE(ocirc_cevent, ocirc_cevent, ocirc_cevent_msg_t *);
+
+#ifdef OCIRC_EVENT_PRIVATE
+void ocirc_state_publish(ocirc_state_msg_t *msg);
+void ocirc_chan_publish(ocirc_chan_msg_t *msg);
+void ocirc_cevent_publish(ocirc_cevent_msg_t *msg);
+#endif
+
+#endif /* !defined(TOR_OCIRC_EVENT_H) */
diff --git a/src/core/or/onion.c b/src/core/or/onion.c
index aeddedd807..a3b5c6922d 100644
--- a/src/core/or/onion.c
+++ b/src/core/or/onion.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -240,11 +240,21 @@ created_cell_parse(created_cell_t *cell_out, const cell_t *cell_in)
 static int
 check_extend_cell(const extend_cell_t *cell)
 {
+  const bool is_extend2 = (cell->cell_type == RELAY_COMMAND_EXTEND2);
+
   if (tor_digest_is_zero((const char*)cell->node_id))
     return -1;
-  /* We don't currently allow EXTEND2 cells without an IPv4 address */
-  if (tor_addr_family(&cell->orport_ipv4.addr) == AF_UNSPEC)
-    return -1;
+  if (!tor_addr_port_is_valid_ap(&cell->orport_ipv4, 0)) {
+    /* EXTEND cells must have an IPv4 address. */
+    if (!is_extend2) {
+      return -1;
+    }
+    /* EXTEND2 cells must have at least one IP address.
+     * It can be IPv4 or IPv6. */
+    if (!tor_addr_port_is_valid_ap(&cell->orport_ipv6, 0)) {
+      return -1;
+    }
+  }
   if (cell->create_cell.cell_type == CELL_CREATE) {
     if (cell->cell_type != RELAY_COMMAND_EXTEND)
       return -1;
@@ -343,7 +353,7 @@ extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
           continue;
         found_ipv6 = 1;
         tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
-                                 (const char *)ls->un_ipv6_addr);
+                                 ls->un_ipv6_addr);
         cell_out->orport_ipv6.port = ls->un_ipv6_port;
         break;
       case LS_LEGACY_ID:
@@ -364,7 +374,12 @@ extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
     }
   }
 
-  if (!found_rsa_id || !found_ipv4) /* These are mandatory */
+  /* EXTEND2 cells must have an RSA ID */
+  if (!found_rsa_id)
+    return -1;
+
+  /* EXTEND2 cells must have at least one IP address */
+  if (!found_ipv4 && !found_ipv6)
     return -1;
 
   return create_cell_from_create2_cell_body(&cell_out->create_cell,
@@ -374,9 +389,11 @@ extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
 /** Parse an EXTEND or EXTEND2 cell (according to <b>command</b>) from the
  * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
  * 0 on success, -1 on failure. */
-int
-extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
-                  const uint8_t *payload, size_t payload_length)
+MOCK_IMPL(int,
+extend_cell_parse,(extend_cell_t *cell_out,
+                   const uint8_t command,
+                   const uint8_t *payload,
+                   size_t payload_length))
 {
 
   tor_assert(cell_out);
@@ -618,12 +635,13 @@ extend_cell_format(uint8_t *command_out, uint16_t *len_out,
     break;
   case RELAY_COMMAND_EXTEND2:
     {
-      uint8_t n_specifiers = 2;
+      uint8_t n_specifiers = 1;
       *command_out = RELAY_COMMAND_EXTEND2;
       extend2_cell_body_t *cell = extend2_cell_body_new();
       link_specifier_t *ls;
-      {
-        /* IPv4 specifier first. */
+      if (tor_addr_port_is_valid_ap(&cell_in->orport_ipv4, 0)) {
+        /* Maybe IPv4 specifier first. */
+        ++n_specifiers;
         ls = link_specifier_new();
         extend2_cell_body_add_ls(cell, ls);
         ls->ls_type = LS_IPV4;
@@ -649,6 +667,17 @@ extend_cell_format(uint8_t *command_out, uint16_t *len_out,
         ls->ls_len = 32;
         memcpy(ls->un_ed25519_id, cell_in->ed_pubkey.pubkey, 32);
       }
+      if (tor_addr_port_is_valid_ap(&cell_in->orport_ipv6, 0)) {
+        /* Then maybe IPv6 specifier. */
+        ++n_specifiers;
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_IPV6;
+        ls->ls_len = 18;
+        tor_addr_copy_ipv6_bytes(ls->un_ipv6_addr,
+                                 &cell_in->orport_ipv6.addr);
+        ls->un_ipv6_port = cell_in->orport_ipv6.port;
+      }
       cell->n_spec = n_specifiers;
 
       /* Now, the handshake */
diff --git a/src/core/or/onion.h b/src/core/or/onion.h
index bb0b5b8dfd..256f0a3f31 100644
--- a/src/core/or/onion.h
+++ b/src/core/or/onion.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -56,8 +56,8 @@ typedef struct extend_cell_t {
   /** Ed25519 public identity key. Zero if not set. */
   struct ed25519_public_key_t ed_pubkey;
   /** The "create cell" embedded in this extend cell. Note that unlike the
-   * create cells we generate ourself, this once can have a handshake type we
-   * don't recognize. */
+   * create cells we generate ourselves, this create cell can have a handshake
+   * type we don't recognize. */
   create_cell_t create_cell;
 } extend_cell_t;
 
@@ -74,8 +74,10 @@ void create_cell_init(create_cell_t *cell_out, uint8_t cell_type,
                       const uint8_t *onionskin);
 int create_cell_parse(create_cell_t *cell_out, const cell_t *cell_in);
 int created_cell_parse(created_cell_t *cell_out, const cell_t *cell_in);
-int extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
-                      const uint8_t *payload_in, size_t payload_len);
+MOCK_DECL(int,extend_cell_parse,(extend_cell_t *cell_out,
+                                 const uint8_t command,
+                                 const uint8_t *payload_in,
+                                 size_t payload_len));
 int extended_cell_parse(extended_cell_t *cell_out, const uint8_t command,
                         const uint8_t *payload_in, size_t payload_len);
 
diff --git a/src/core/or/or.h b/src/core/or/or.h
index 7c601e49b3..8758a2ec6f 100644
--- a/src/core/or/or.h
+++ b/src/core/or/or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,9 +24,8 @@
 
 #include "lib/arch/bytes.h"
 #include "lib/cc/compat_compiler.h"
-#include "lib/cc/torint.h"
 #include "lib/container/map.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/container/smartlist.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rsa.h"
@@ -97,6 +96,8 @@ struct curve25519_public_key_t;
 #define SIGNEWNYM 129
 #define SIGCLEARDNSCACHE 130
 #define SIGHEARTBEAT 131
+#define SIGACTIVE 132
+#define SIGDORMANT 133
 
 #if (SIZEOF_CELL_T != 0)
 /* On Irix, stdlib.h defines a cell_t type, so we need to make sure
@@ -166,12 +167,13 @@ struct curve25519_public_key_t;
 #define PROXY_CONNECT 1
 #define PROXY_SOCKS4 2
 #define PROXY_SOCKS5 3
-/* !!!! If there is ever a PROXY_* type over 3, we must grow the proxy_type
+#define PROXY_HAPROXY 4
+/* !!!! If there is ever a PROXY_* type over 7, we must grow the proxy_type
  * field in or_connection_t */
 
 /* Pluggable transport proxy type. Don't use this in or_connection_t,
  * instead use the actual underlying proxy type (see above).  */
-#define PROXY_PLUGGABLE 4
+#define PROXY_PLUGGABLE 5
 
 /** How many circuits do we want simultaneously in-progress to handle
  * a given stream? */
@@ -205,6 +207,9 @@ struct curve25519_public_key_t;
 #define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39
 #define RELAY_COMMAND_INTRODUCE_ACK 40
 
+#define RELAY_COMMAND_PADDING_NEGOTIATE 41
+#define RELAY_COMMAND_PADDING_NEGOTIATED 42
+
 /* Reasons why an OR connection is closed. */
 #define END_OR_CONN_REASON_DONE           1
 #define END_OR_CONN_REASON_REFUSED        2 /* connection refused */
@@ -215,7 +220,8 @@ struct curve25519_public_key_t;
 #define END_OR_CONN_REASON_IO_ERROR       7 /* read/write error */
 #define END_OR_CONN_REASON_RESOURCE_LIMIT 8 /* sockets, buffers, etc */
 #define END_OR_CONN_REASON_PT_MISSING     9 /* PT failed or not available */
-#define END_OR_CONN_REASON_MISC           10
+#define END_OR_CONN_REASON_TLS_ERROR      10 /* Problem in TLS protocol */
+#define END_OR_CONN_REASON_MISC           11
 
 /* Reasons why we (or a remote OR) might close a stream. See tor-spec.txt for
  * documentation of these.  The values must match. */
@@ -604,21 +610,21 @@ typedef uint32_t circid_t;
 /** Identifies a stream on a circuit */
 typedef uint16_t streamid_t;
 
-/* channel_t typedef; struct channel_s is in channel.h */
+/* channel_t typedef; struct channel_t is in channel.h */
 
-typedef struct channel_s channel_t;
+typedef struct channel_t channel_t;
 
-/* channel_listener_t typedef; struct channel_listener_s is in channel.h */
+/* channel_listener_t typedef; struct channel_listener_t is in channel.h */
 
-typedef struct channel_listener_s channel_listener_t;
+typedef struct channel_listener_t channel_listener_t;
 
 /* TLS channel stuff */
 
-typedef struct channel_tls_s channel_tls_t;
+typedef struct channel_tls_t channel_tls_t;
 
-/* circuitmux_t typedef; struct circuitmux_s is in circuitmux.h */
+/* circuitmux_t typedef; struct circuitmux_t is in circuitmux.h */
 
-typedef struct circuitmux_s circuitmux_t;
+typedef struct circuitmux_t circuitmux_t;
 
 typedef struct cell_t cell_t;
 typedef struct var_cell_t var_cell_t;
@@ -834,6 +840,14 @@ typedef struct protover_summary_flags_t {
    * service rendezvous point supporting version 3 as seen in proposal 224.
    * This requires HSRend=2. */
   unsigned int supports_v3_rendezvous_point: 1;
+
+  /** True iff this router has a protocol list that allows clients to
+   * negotiate hs circuit setup padding. Requires Padding>=2. */
+  unsigned int supports_hs_setup_padding : 1;
+
+  /** True iff this router has a protocol list that allows it to support the
+   * ESTABLISH_INTRO DoS cell extension. Requires HSIntro>=5. */
+  unsigned int supports_establish_intro_dos_extension : 1;
 } protover_summary_flags_t;
 
 typedef struct routerinfo_t routerinfo_t;
@@ -982,8 +996,6 @@ typedef struct routerset_t routerset_t;
 
 typedef struct or_options_t or_options_t;
 
-#define LOG_PROTOCOL_WARN (get_protocol_warning_severity_level())
-
 typedef struct or_state_t or_state_t;
 
 #define MAX_SOCKS_ADDR_LEN 256
@@ -1000,7 +1012,7 @@ typedef struct or_state_t or_state_t;
 #define BW_MIN_WEIGHT_SCALE 1
 #define BW_MAX_WEIGHT_SCALE INT32_MAX
 
-typedef struct circuit_build_times_s circuit_build_times_t;
+typedef struct circuit_build_times_t circuit_build_times_t;
 
 /********************************* config.c ***************************/
 
diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h
index 6b6feb9d89..4e17b1c143 100644
--- a/src/core/or/or_circuit_st.h
+++ b/src/core/or/or_circuit_st.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef OR_CIRCUIT_ST_H
@@ -12,6 +12,8 @@
 #include "core/or/circuit_st.h"
 #include "core/or/crypt_path_st.h"
 
+#include "lib/evloop/token_bucket.h"
+
 struct onion_queue_t;
 
 /** An or_circuit_t holds information needed to implement a circuit at an
@@ -25,7 +27,7 @@ struct or_circuit_t {
   /** Pointer to a workqueue entry, if this circuit has given an onionskin to
    * a cpuworker and is waiting for a response. Used to decide whether it is
    * safe to free a circuit or if it is still in use by a cpuworker. */
-  struct workqueue_entry_s *workqueue_entry;
+  struct workqueue_entry_t *workqueue_entry;
 
   /** The circuit_id used in the previous (backward) hop of this circuit. */
   circid_t p_circ_id;
@@ -33,11 +35,6 @@ struct or_circuit_t {
   cell_queue_t p_chan_cells;
   /** The channel that is previous in this circuit. */
   channel_t *p_chan;
-  /**
-   * Circuit mux associated with p_chan to which this circuit is attached;
-   * NULL if we have no p_chan.
-   */
-  circuitmux_t *p_mux;
   /** Linked list of Exit streams associated with this circuit. */
   edge_connection_t *n_streams;
   /** Linked list of Exit streams associated with this circuit that are
@@ -74,7 +71,20 @@ struct or_circuit_t {
    * exit-ward queues of this circuit; reset every time when writing
    * buffer stats to disk. */
   uint64_t total_cell_waiting_time;
+
+  /** If set, the DoS defenses are enabled on this circuit meaning that the
+   * introduce2_bucket is initialized and used. */
+  unsigned int introduce2_dos_defense_enabled : 1;
+  /** If set, the DoS defenses were explicitly enabled through the
+   * ESTABLISH_INTRO cell extension. If unset, the consensus is used to learn
+   * if the defenses can be enabled or not. */
+  unsigned int introduce2_dos_defense_explicit : 1;
+
+  /** INTRODUCE2 cell bucket controlling how much can go on this circuit. Only
+   * used if this is a service introduction circuit at the intro point
+   * (purpose = CIRCUIT_PURPOSE_INTRO_POINT). */
+  token_bucket_ctr_t introduce2_bucket;
 };
 
-#endif
+#endif /* !defined(OR_CIRCUIT_ST_H) */
 
diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h
index d5db5e8694..92956c2847 100644
--- a/src/core/or/or_connection_st.h
+++ b/src/core/or/or_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file or_connection_st.h
+ * @brief OR connection structure.
+ **/
+
 #ifndef OR_CONNECTION_ST_H
 #define OR_CONNECTION_ST_H
 
@@ -58,7 +63,7 @@ struct or_connection_t {
 
   /** True iff this is an outgoing connection. */
   unsigned int is_outgoing:1;
-  unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */
+  unsigned int proxy_type:3; /**< One of PROXY_NONE...PROXY_HAPROXY */
   unsigned int wide_circ_ids:1;
   /** True iff this connection has had its bootstrap failure logged with
    * control_event_bootstrap_problem. */
@@ -67,6 +72,8 @@ struct or_connection_t {
    * geoip cache and handled by the DoS mitigation subsystem. We use this to
    * insure we have a coherent count of concurrent connection. */
   unsigned int tracked_for_dos_mitigation : 1;
+  /** True iff this connection is using a pluggable transport */
+  unsigned int is_pt : 1;
 
   uint16_t link_proto; /**< What protocol version are we using? 0 for
                         * "none negotiated yet." */
@@ -89,4 +96,4 @@ struct or_connection_t {
   uint64_t bytes_xmitted, bytes_xmitted_by_tls;
 };
 
-#endif
+#endif /* !defined(OR_CONNECTION_ST_H) */
diff --git a/src/core/or/or_handshake_certs_st.h b/src/core/or/or_handshake_certs_st.h
index a93b7104aa..31755f04a1 100644
--- a/src/core/or/or_handshake_certs_st.h
+++ b/src/core/or/or_handshake_certs_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file or_handshake_certs_st.h
+ * @brief OR handshake certs structure
+ **/
+
 #ifndef OR_HANDSHAKE_CERTS_ST
 #define OR_HANDSHAKE_CERTS_ST
 
@@ -37,4 +42,4 @@ struct or_handshake_certs_t {
   size_t ed_rsa_crosscert_len;
 };
 
-#endif
+#endif /* !defined(OR_HANDSHAKE_CERTS_ST) */
diff --git a/src/core/or/or_handshake_state_st.h b/src/core/or/or_handshake_state_st.h
index 09a8a34179..050404046d 100644
--- a/src/core/or/or_handshake_state_st.h
+++ b/src/core/or/or_handshake_state_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file or_handshake_state_st.h
+ * @brief OR handshake state structure
+ **/
+
 #ifndef OR_HANDSHAKE_STATE_ST
 #define OR_HANDSHAKE_STATE_ST
 
@@ -74,5 +79,4 @@ struct or_handshake_state_t {
   or_handshake_certs_t *certs;
 };
 
-#endif
-
+#endif /* !defined(OR_HANDSHAKE_STATE_ST) */
diff --git a/src/core/or/or_periodic.c b/src/core/or/or_periodic.c
new file mode 100644
index 0000000000..4dfdce14ca
--- /dev/null
+++ b/src/core/or/or_periodic.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.c
+ * @brief Periodic callbacks for the onion routing subsystem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+
+#include "core/or/channel.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_periodic.h"
+
+#include "feature/relay/routermode.h"
+
+#ifndef COCCI
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+#endif /* !defined(COCCI) */
+
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+
+#define CHANNEL_CHECK_INTERVAL (60*60)
+static int
+check_canonical_channels_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (public_server_mode(options))
+    channel_check_for_duplicates();
+
+  return CHANNEL_CHECK_INTERVAL;
+}
+
+DECLARE_EVENT(check_canonical_channels, RELAY, FL(NEED_NET));
+
+/**
+ * Periodic callback: as a server, see if we have any old unused circuits
+ * that should be expired */
+static int
+expire_old_circuits_serverside_callback(time_t now,
+                                        const or_options_t *options)
+{
+  (void)options;
+  /* every 11 seconds, so not usually the same second as other such events */
+  circuit_expire_old_circuits_serverside(now);
+  return 11;
+}
+
+DECLARE_EVENT(expire_old_circuits_serverside, ROUTER, FL(NEED_NET));
+
+void
+or_register_periodic_events(void)
+{
+  // These are router-only events, but they're owned by the OR subsystem. */
+  periodic_events_register(&check_canonical_channels_event);
+  periodic_events_register(&expire_old_circuits_serverside_event);
+}
diff --git a/src/core/or/or_periodic.h b/src/core/or/or_periodic.h
new file mode 100644
index 0000000000..080573a838
--- /dev/null
+++ b/src/core/or/or_periodic.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.h
+ * @brief Header for core/or/or_periodic.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_PERIODIC_H
+#define TOR_CORE_OR_OR_PERIODIC_H
+
+void or_register_periodic_events(void);
+
+#endif /* !defined(TOR_CORE_OR_OR_PERIODIC_H) */
diff --git a/src/core/or/or_sys.c b/src/core/or/or_sys.c
new file mode 100644
index 0000000000..73c6087dce
--- /dev/null
+++ b/src/core/or/or_sys.c
@@ -0,0 +1,56 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.c
+ * @brief Subsystem definitions for OR module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/or/or_periodic.h"
+#include "core/or/or_sys.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/versions.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_or_initialize(void)
+{
+  or_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_or_shutdown(void)
+{
+  protover_free_all();
+  protover_summary_cache_free_all();
+  policies_free_all();
+}
+
+static int
+subsys_or_add_pubsub(struct pubsub_connector_t *connector)
+{
+  int rv = 0;
+  if (orconn_add_pubsub(connector) < 0)
+    rv = -1;
+  if (ocirc_add_pubsub(connector) < 0)
+    rv = -1;
+  return rv;
+}
+
+const struct subsys_fns_t sys_or = {
+  .name = "or",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = 20,
+  .initialize = subsys_or_initialize,
+  .shutdown = subsys_or_shutdown,
+  .add_pubsub = subsys_or_add_pubsub,
+};
diff --git a/src/core/or/or_sys.h b/src/core/or/or_sys.h
new file mode 100644
index 0000000000..7ee56c8682
--- /dev/null
+++ b/src/core/or/or_sys.h
@@ -0,0 +1,21 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.h
+ * @brief Header for core/or/or_sys.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_SYS_H
+#define TOR_CORE_OR_OR_SYS_H
+
+extern const struct subsys_fns_t sys_or;
+
+struct pubsub_connector_t;
+int ocirc_add_pubsub(struct pubsub_connector_t *connector);
+int orconn_add_pubsub(struct pubsub_connector_t *connector);
+
+#endif /* !defined(TOR_CORE_OR_OR_SYS_H) */
diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c
new file mode 100644
index 0000000000..c30e2dd22f
--- /dev/null
+++ b/src/core/or/orconn_event.c
@@ -0,0 +1,92 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.c
+ * \brief Publish state change messages for OR connections
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of OR connections.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/orconn_event.h"
+#include "core/or/or_sys.h"
+
+DECLARE_PUBLISH(orconn_state);
+DECLARE_PUBLISH(orconn_status);
+
+static void
+orconn_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+orconn_state_fmt(msg_aux_data_t u)
+{
+  orconn_state_msg_t *msg = (orconn_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" chan=%"PRIu64" proxy_type=%d state=%d>",
+               msg->gid, msg->chan, msg->proxy_type, msg->state);
+  return s;
+}
+
+static char *
+orconn_status_fmt(msg_aux_data_t u)
+{
+  orconn_status_msg_t *msg = (orconn_status_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" status=%d reason=%d>",
+               msg->gid, msg->status, msg->reason);
+  return s;
+}
+
+static dispatch_typefns_t orconn_state_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_state_fmt,
+};
+
+static dispatch_typefns_t orconn_status_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_status_fmt,
+};
+
+int
+orconn_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_state, &orconn_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_status, &orconn_status_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_state) != 0)
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_status) != 0)
+    return -1;
+  return 0;
+}
+
+void
+orconn_state_publish(orconn_state_msg_t *msg)
+{
+  PUBLISH(orconn_state, msg);
+}
+
+void
+orconn_status_publish(orconn_status_msg_t *msg)
+{
+  PUBLISH(orconn_status, msg);
+}
diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h
new file mode 100644
index 0000000000..2653b20d6e
--- /dev/null
+++ b/src/core/or/orconn_event.h
@@ -0,0 +1,103 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.h
+ * \brief Header file for orconn_event.c
+ *
+ * The OR_CONN_STATE_* symbols are here to make it easier for
+ * subscribers to make decisions based on the messages that they
+ * receive.
+ **/
+
+#ifndef TOR_ORCONN_EVENT_H
+#define TOR_ORCONN_EVENT_H
+
+#include "lib/pubsub/pubsub.h"
+
+/**
+ * @name States of OR connections
+ *
+ * These must be in a partial ordering such that usually no OR
+ * connection will transition from a higher-numbered state to a
+ * lower-numbered one.  Code such as bto_update_best() depends on this
+ * ordering to determine the best state it's seen so far.
+ * @{ */
+#define OR_CONN_STATE_MIN_ 1
+/** State for a connection to an OR: waiting for connect() to finish. */
+#define OR_CONN_STATE_CONNECTING 1
+/** State for a connection to an OR: waiting for proxy handshake to complete */
+#define OR_CONN_STATE_PROXY_HANDSHAKING 2
+/** State for an OR connection client: SSL is handshaking, not done
+ * yet. */
+#define OR_CONN_STATE_TLS_HANDSHAKING 3
+/** State for a connection to an OR: We're doing a second SSL handshake for
+ * renegotiation purposes. (V2 handshake only.) */
+#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
+/** State for a connection at an OR: We're waiting for the client to
+ * renegotiate (to indicate a v2 handshake) or send a versions cell (to
+ * indicate a v3 handshake) */
+#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
+/** State for an OR connection: We're done with our SSL handshake, we've done
+ * renegotiation, but we haven't yet negotiated link protocol versions and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
+/** State for an OR connection: We're done with our SSL handshake, but we
+ * haven't yet negotiated link protocol versions, done a V3 handshake, and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
+/** State for an OR connection: Ready to send/receive cells. */
+#define OR_CONN_STATE_OPEN 8
+#define OR_CONN_STATE_MAX_ 8
+/** @} */
+
+/** Used to indicate the type of an OR connection event passed to the
+ * controller.  The various types are defined in control-spec.txt */
+typedef enum or_conn_status_event_t {
+  OR_CONN_EVENT_LAUNCHED     = 0,
+  OR_CONN_EVENT_CONNECTED    = 1,
+  OR_CONN_EVENT_FAILED       = 2,
+  OR_CONN_EVENT_CLOSED       = 3,
+  OR_CONN_EVENT_NEW          = 4,
+} or_conn_status_event_t;
+
+/**
+ * Message for orconn state update
+ *
+ * This contains information about internal state changes of
+ * or_connection_t objects.  The chan and proxy_type fields are
+ * additional information that a subscriber may need to make
+ * decisions.
+ **/
+typedef struct orconn_state_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  uint64_t chan;                /**< associated channel ID */
+  int proxy_type;               /**< connection's proxy type */
+  uint8_t state;                /**< new connection state */
+} orconn_state_msg_t;
+
+DECLARE_MESSAGE(orconn_state, orconn_state, orconn_state_msg_t *);
+
+/**
+ * Message for orconn status event
+ *
+ * This contains information that ends up in ORCONN control protocol
+ * events.
+ **/
+typedef struct orconn_status_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  int status;                   /**< or_conn_status_event_t */
+  int reason;                   /**< reason */
+} orconn_status_msg_t;
+
+DECLARE_MESSAGE(orconn_status, orconn_status, orconn_status_msg_t *);
+
+#ifdef ORCONN_EVENT_PRIVATE
+void orconn_state_publish(orconn_state_msg_t *);
+void orconn_status_publish(orconn_status_msg_t *);
+#endif
+
+#endif /* !defined(TOR_ORCONN_EVENT_H) */
diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h
index f55416db14..79e250cd59 100644
--- a/src/core/or/origin_circuit_st.h
+++ b/src/core/or/origin_circuit_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file origin_circuit_st.h
+ * @brief Origin circuit structure.
+ **/
+
 #ifndef ORIGIN_CIRCUIT_ST_H
 #define ORIGIN_CIRCUIT_ST_H
 
@@ -161,6 +166,10 @@ struct origin_circuit_t {
    * connections to this circuit. */
   unsigned int unusable_for_new_conns : 1;
 
+  /* If this flag is set (due to padding negotiation failure), we should
+   * not try to negotiate further circuit padding. */
+  unsigned padding_negotiation_failed : 1;
+
   /**
    * Tristate variable to guard against pathbias miscounting
    * due to circuit purpose transitions changing the decision
@@ -291,4 +300,4 @@ struct origin_circuit_t {
 
 };
 
-#endif
+#endif /* !defined(ORIGIN_CIRCUIT_ST_H) */
diff --git a/src/core/or/policies.c b/src/core/or/policies.c
index 3ed282d785..2bf2dc7005 100644
--- a/src/core/or/policies.c
+++ b/src/core/or/policies.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,9 @@
 #include "feature/relay/routermode.h"
 #include "lib/geoip/geoip.h"
 #include "ht.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/encoding/confline.h"
+#include "trunnel/ed25519_cert.h"
 
 #include "core/or/addr_policy_st.h"
 #include "feature/dirclient/dir_server_st.h"
@@ -165,7 +167,7 @@ policy_expand_unspec(smartlist_t **policy)
       }
       tor_addr_from_ipv4h(&newpolicy_ipv4.addr, 0);
       tor_addr_from_ipv6_bytes(&newpolicy_ipv6.addr,
-                               "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
+          (const uint8_t *)"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
       smartlist_add(tmp, addr_policy_get_canonical_entry(&newpolicy_ipv4));
       smartlist_add(tmp, addr_policy_get_canonical_entry(&newpolicy_ipv6));
       addr_policy_free(p);
@@ -918,49 +920,6 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
                                        pref_ipv6, ap);
 }
 
-/* Some microdescriptor consensus methods have no IPv6 addresses in rs: they
- * are in the microdescriptors. For these consensus methods, we can't rely on
- * the node's IPv6 address until its microdescriptor is available (when using
- * microdescs).
- * But for bridges, rewrite_node_address_for_bridge() updates node->ri with
- * the configured address, so we can trust bridge addresses.
- * (Bridges could gain an IPv6 address if their microdescriptor arrives, but
- * this will never be their preferred address: that is in the config.)
- * Returns true if the node needs a microdescriptor for its IPv6 address, and
- * false if the addresses in the node are already up-to-date.
- */
-static int
-node_awaiting_ipv6(const or_options_t* options, const node_t *node)
-{
-  tor_assert(node);
-
-  /* There's no point waiting for an IPv6 address if we'd never use it */
-  if (!fascist_firewall_use_ipv6(options)) {
-    return 0;
-  }
-
-  /* If the node has an IPv6 address, we're not waiting */
-  if (node_has_ipv6_addr(node)) {
-    return 0;
-  }
-
-  /* If the current consensus method and flavour has IPv6 addresses, we're not
-   * waiting */
-  if (networkstatus_consensus_has_ipv6(options)) {
-    return 0;
-  }
-
-  /* Bridge clients never use the address from a bridge's md, so there's no
-   * need to wait for it. */
-  if (node_is_a_configured_bridge(node)) {
-    return 0;
-  }
-
-  /* We are waiting if we_use_microdescriptors_for_circuits() and we have no
-   * md. */
-  return (!node->md && we_use_microdescriptors_for_circuits(options));
-}
-
 /** Like fascist_firewall_choose_address_base(), but takes <b>rs</b>.
  * Consults the corresponding node, then falls back to rs if node is NULL.
  * This should only happen when there's no valid consensus, and rs doesn't
@@ -983,7 +942,7 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs,
   const or_options_t *options = get_options();
   const node_t *node = node_get_by_id(rs->identity_digest);
 
-  if (node && !node_awaiting_ipv6(options, node)) {
+  if (node) {
     fascist_firewall_choose_address_node(node, fw_connection, pref_only, ap);
   } else {
     /* There's no node-specific IPv6 preference, so use the generic IPv6
@@ -1001,6 +960,83 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs,
   }
 }
 
+/** Like fascist_firewall_choose_address_base(), but takes in a smartlist
+ * <b>lspecs</b> consisting of one or more link specifiers. We assume
+ * fw_connection is FIREWALL_OR_CONNECTION as link specifiers cannot
+ * contain DirPorts.
+ */
+void
+fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                   int pref_only, tor_addr_port_t* ap)
+{
+  int have_v4 = 0, have_v6 = 0;
+  uint16_t port_v4 = 0, port_v6 = 0;
+  tor_addr_t addr_v4, addr_v6;
+
+  tor_assert(ap);
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Unknown or missing link specifiers");
+    return;
+  }
+  if (smartlist_len(lspecs) == 0) {
+    log_warn(LD_PROTOCOL, "Link specifiers are empty");
+    return;
+  }
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
+  tor_addr_make_null(&addr_v4, AF_INET);
+  tor_addr_make_null(&addr_v6, AF_INET6);
+
+  SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
+    switch (link_specifier_get_ls_type(ls)) {
+    case LS_IPV4:
+      /* Skip if we already seen a v4. */
+      if (have_v4) continue;
+      tor_addr_from_ipv4h(&addr_v4,
+                          link_specifier_get_un_ipv4_addr(ls));
+      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      have_v4 = 1;
+      break;
+    case LS_IPV6:
+      /* Skip if we already seen a v6, or deliberately skip it if we're not a
+       * direct connection. */
+      if (have_v6) continue;
+      tor_addr_from_ipv6_bytes(&addr_v6,
+          link_specifier_getconstarray_un_ipv6_addr(ls));
+      port_v6 = link_specifier_get_un_ipv6_port(ls);
+      have_v6 = 1;
+      break;
+    default:
+      /* Ignore unknown. */
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* If we don't have IPv4 or IPv6 in link specifiers, log a bug and return. */
+  if (!have_v4 && !have_v6) {
+    if (!have_v6) {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4 or IPv6");
+    } else {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4");
+    }
+    return;
+  }
+
+  /* Here, don't check for DirPorts as link specifiers are only used for
+   * ORPorts. */
+  const or_options_t *options = get_options();
+  int pref_ipv6 = fascist_firewall_prefer_ipv6_orport(options);
+  /* Assume that the DirPorts are zero as link specifiers only use ORPorts. */
+  fascist_firewall_choose_address_base(&addr_v4, port_v4, 0,
+                                       &addr_v6, port_v6, 0,
+                                       FIREWALL_OR_CONNECTION,
+                                       pref_only, pref_ipv6,
+                                       ap);
+}
+
 /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and
  * looks up the node's IPv6 preference rather than taking an argument
  * for pref_ipv6. */
@@ -1019,17 +1055,6 @@ fascist_firewall_choose_address_node(const node_t *node,
   }
 
   node_assert_ok(node);
-  /* Calling fascist_firewall_choose_address_node() when the node is missing
-   * IPv6 information breaks IPv6-only clients.
-   * If the node is a hard-coded fallback directory or authority, call
-   * fascist_firewall_choose_address_rs() on the fake (hard-coded) routerstatus
-   * for the node.
-   * If it is not hard-coded, check that the node has a microdescriptor, full
-   * descriptor (routerinfo), or is one of our configured bridges before
-   * calling this function. */
-  if (BUG(node_awaiting_ipv6(get_options(), node))) {
-    return;
-  }
 
   const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION
                               ? node_ipv6_or_preferred(node)
@@ -1150,6 +1175,15 @@ authdir_policy_badexit_address(uint32_t addr, uint16_t port)
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END
 
+/** Check <b>or_options</b> to determine whether or not we are using the
+ * default options for exit policy. Return true if so, false otherwise. */
+static int
+policy_using_default_exit_options(const or_options_t *or_options)
+{
+  return (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 &&
+          or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0);
+}
+
 /** Config helper: If there's any problem with the policy configuration
  * options in <b>options</b>, return -1 and set <b>msg</b> to a newly
  * allocated description of the error. Else return 0. */
@@ -1168,9 +1202,8 @@ validate_addr_policies(const or_options_t *options, char **msg)
 
   static int warned_about_nonexit = 0;
 
-  if (public_server_mode(options) &&
-      !warned_about_nonexit && options->ExitPolicy == NULL &&
-      options->ExitRelay == -1 && options->ReducedExitPolicy == 0) {
+  if (public_server_mode(options) && !warned_about_nonexit &&
+      policy_using_default_exit_options(options)) {
     warned_about_nonexit = 1;
     log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. "
                "If you want to be an exit relay, "
@@ -1359,9 +1392,9 @@ policy_hash(const policy_map_ent_t *ent)
 }
 
 HT_PROTOTYPE(policy_map, policy_map_ent_t, node, policy_hash,
-             policy_eq)
+             policy_eq);
 HT_GENERATE2(policy_map, policy_map_ent_t, node, policy_hash,
-             policy_eq, 0.6, tor_reallocarray_, tor_free_)
+             policy_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** Given a pointer to an addr_policy_t, return a copy of the pointer to the
  * "canonical" copy of that addr_policy_t; the canonical copy is a single
@@ -2127,9 +2160,9 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options,
   int rv = 0;
 
   /* Short-circuit for non-exit relays, or for relays where we didn't specify
-   * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */
-  if (or_options->ExitRelay == 0 || (or_options->ExitPolicy == NULL &&
-      or_options->ExitRelay == -1 && or_options->ReducedExitPolicy == 0)) {
+   * ExitPolicy or ReducedExitPolicy or IPv6Exit and ExitRelay is auto. */
+  if (or_options->ExitRelay == 0 ||
+      policy_using_default_exit_options(or_options)) {
     append_exit_policy_string(result, "reject *4:*");
     append_exit_policy_string(result, "reject *6:*");
     return 0;
@@ -2706,7 +2739,7 @@ parse_short_policy(const char *summary)
   int is_accept;
   int n_entries;
   short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */
-  const char *next;
+  char *next;
 
   if (!strcmpstart(summary, "accept ")) {
     is_accept = 1;
@@ -2721,57 +2754,56 @@ parse_short_policy(const char *summary)
 
   n_entries = 0;
   for ( ; *summary; summary = next) {
-    const char *comma = strchr(summary, ',');
-    unsigned low, high;
-    char dummy;
-    char ent_buf[32];
-    size_t len;
-
-    next = comma ? comma+1 : strchr(summary, '\0');
-    len = comma ? (size_t)(comma - summary) : strlen(summary);
-
     if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) {
       log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s",
              escaped(orig_summary));
       return NULL;
     }
 
-    if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) {
-      /* unrecognized entry format. skip it. */
-      continue;
-    }
-    if (len < 1) {
-      /* empty; skip it. */
-      /* XXX This happens to be unreachable, since if len==0, then *summary is
-       * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */
-      continue;
+    unsigned low, high;
+    int ok;
+    low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next);
+    if (!ok) {
+      if (! TOR_ISDIGIT(*summary) || *summary == ',') {
+        /* Unrecognized format: skip it. */
+        goto skip_ent;
+      } else {
+        goto bad_ent;
+      }
     }
 
-    memcpy(ent_buf, summary, len);
-    ent_buf[len] = '\0';
+    switch (*next) {
+      case ',':
+        ++next;
+        FALLTHROUGH;
+      case '\0':
+        high = low;
+        break;
+      case '-':
+        high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next);
+        if (!ok)
+          goto bad_ent;
 
-    if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) {
-      if (low<1 || low>65535 || high<1 || high>65535 || low>high) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-    } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) {
-      if (low<1 || low>65535) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-      high = low;
-    } else {
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
-             escaped(orig_summary));
-      return NULL;
+        if (*next == ',')
+          ++next;
+        else if (*next != '\0')
+          goto bad_ent;
+
+        break;
+      default:
+        goto bad_ent;
     }
 
     entries[n_entries].min_port = low;
     entries[n_entries].max_port = high;
     n_entries++;
+
+    continue;
+  skip_ent:
+    next = strchr(next, ',');
+    if (!next)
+      break;
+    ++next;
   }
 
   if (n_entries == 0) {
@@ -2792,6 +2824,11 @@ parse_short_policy(const char *summary)
   result->n_entries = n_entries;
   memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries);
   return result;
+
+ bad_ent:
+  log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
+         escaped(orig_summary));
+  return NULL;
 }
 
 /** Write <b>policy</b> back out into a string. */
diff --git a/src/core/or/policies.h b/src/core/or/policies.h
index 2c38de362f..72a37d62b0 100644
--- a/src/core/or/policies.h
+++ b/src/core/or/policies.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -91,6 +91,8 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds,
 void fascist_firewall_choose_address_rs(const routerstatus_t *rs,
                                         firewall_connection_t fw_connection,
                                         int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                        int pref_only, tor_addr_port_t* ap);
 void fascist_firewall_choose_address_node(const node_t *node,
                                           firewall_connection_t fw_connection,
                                           int pref_only, tor_addr_port_t* ap);
diff --git a/src/core/or/port_cfg_st.h b/src/core/or/port_cfg_st.h
index b67091ce32..064e679d78 100644
--- a/src/core/or/port_cfg_st.h
+++ b/src/core/or/port_cfg_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file port_cfg_st.h
+ * @brief Listener port configuration structure.
+ **/
+
 #ifndef PORT_CFG_ST_H
 #define PORT_CFG_ST_H
 
@@ -31,5 +36,4 @@ struct port_cfg_t {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
-
+#endif /* !defined(PORT_CFG_ST_H) */
diff --git a/src/core/or/protover.c b/src/core/or/protover.c
index dfb0e9e303..0d03e9a06b 100644
--- a/src/core/or/protover.c
+++ b/src/core/or/protover.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -39,6 +39,9 @@ static int protocol_list_contains(const smartlist_t *protos,
 static const struct {
   protocol_type_t protover_type;
   const char *name;
+/* If you add a new protocol here, you probably also want to add
+ * parsing for it in summarize_protover_flags(), so that it has a
+ * summary flag in routerstatus_t */
 } PROTOCOL_NAMES[] = {
   { PRT_LINK, "Link" },
   { PRT_LINKAUTH, "LinkAuth" },
@@ -49,7 +52,9 @@ static const struct {
   { PRT_HSREND, "HSRend" },
   { PRT_DESC, "Desc" },
   { PRT_MICRODESC, "Microdesc"},
-  { PRT_CONS, "Cons" }
+  { PRT_PADDING, "Padding"},
+  { PRT_CONS, "Cons" },
+  { PRT_FLOWCTRL, "FlowCtrl"},
 };
 
 #define N_PROTOCOL_NAMES ARRAY_LENGTH(PROTOCOL_NAMES)
@@ -386,8 +391,9 @@ protover_get_supported_protocols(void)
     "Cons=1-2 "
     "Desc=1-2 "
     "DirCache=1-2 "
+    "FlowCtrl=1 "
     "HSDir=1-2 "
-    "HSIntro=3-4 "
+    "HSIntro=3-5 "
     "HSRend=1-2 "
     "Link=1-5 "
 #ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
@@ -396,6 +402,7 @@ protover_get_supported_protocols(void)
     "LinkAuth=3 "
 #endif
     "Microdesc=1-2 "
+    "Padding=2 "
     "Relay=1-2";
 }
 
@@ -815,6 +822,8 @@ protover_all_supported(const char *s, char **missing_out)
        * ones and, if so, add them to unsupported->ranges. */
       if (versions->low != 0 && versions->high != 0) {
         smartlist_add(unsupported->ranges, versions);
+      } else {
+        tor_free(versions);
       }
       /* Finally, if we had something unsupported, add it to the list of
        * missing_some things and mark that there was something missing. */
@@ -823,7 +832,6 @@ protover_all_supported(const char *s, char **missing_out)
         all_supported = 0;
       } else {
         proto_entry_free(unsupported);
-        tor_free(versions);
       }
     } SMARTLIST_FOREACH_END(range);
 
diff --git a/src/core/or/protover.h b/src/core/or/protover.h
index 7e181ba97a..9509f3e8a3 100644
--- a/src/core/or/protover.h
+++ b/src/core/or/protover.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,21 +28,25 @@ struct smartlist_t;
 #define PROTOVER_HS_INTRO_V3 4
 /** The protover version number that signifies HSv3 rendezvous point support */
 #define PROTOVER_HS_RENDEZVOUS_POINT_V3 2
+/** The protover that signals support for HS circuit setup padding machines */
+#define PROTOVER_HS_SETUP_PADDING 2
 
 /** List of recognized subprotocols. */
 /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust`
 /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto`
 typedef enum protocol_type_t {
-  PRT_LINK,
-  PRT_LINKAUTH,
-  PRT_RELAY,
-  PRT_DIRCACHE,
-  PRT_HSDIR,
-  PRT_HSINTRO,
-  PRT_HSREND,
-  PRT_DESC,
-  PRT_MICRODESC,
-  PRT_CONS,
+  PRT_LINK      = 0,
+  PRT_LINKAUTH  = 1,
+  PRT_RELAY     = 2,
+  PRT_DIRCACHE  = 3,
+  PRT_HSDIR     = 4,
+  PRT_HSINTRO   = 5,
+  PRT_HSREND    = 6,
+  PRT_DESC      = 7,
+  PRT_MICRODESC = 8,
+  PRT_CONS      = 9,
+  PRT_PADDING   = 10,
+  PRT_FLOWCTRL  = 11,
 } protocol_type_t;
 
 bool protover_contains_long_protocol_names(const char *s);
diff --git a/src/core/or/protover_rust.c b/src/core/or/protover_rust.c
index bc56ea11d0..f44746b6da 100644
--- a/src/core/or/protover_rust.c
+++ b/src/core/or/protover_rust.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
diff --git a/src/core/or/reasons.c b/src/core/or/reasons.c
index e21bfa670a..708f43a689 100644
--- a/src/core/or/reasons.c
+++ b/src/core/or/reasons.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -244,6 +244,8 @@ orconn_end_reason_to_control_string(int r)
       return "IOERROR";
     case END_OR_CONN_REASON_RESOURCE_LIMIT:
       return "RESOURCELIMIT";
+    case END_OR_CONN_REASON_TLS_ERROR:
+      return "TLS_ERROR";
     case END_OR_CONN_REASON_MISC:
       return "MISC";
     case END_OR_CONN_REASON_PT_MISSING:
@@ -276,6 +278,8 @@ tls_error_to_orconn_end_reason(int e)
     case TOR_TLS_CLOSE:
     case TOR_TLS_DONE:
       return END_OR_CONN_REASON_DONE;
+    case TOR_TLS_ERROR_MISC:
+      return END_OR_CONN_REASON_TLS_ERROR;
     default:
       return END_OR_CONN_REASON_MISC;
   }
diff --git a/src/core/or/reasons.h b/src/core/or/reasons.h
index c45a8bc38d..2e534aab73 100644
--- a/src/core/or/reasons.h
+++ b/src/core/or/reasons.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/or/relay.c b/src/core/or/relay.c
index 00353f47a9..892dbe2d0a 100644
--- a/src/core/or/relay.c
+++ b/src/core/or/relay.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,22 +49,24 @@
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "feature/client/circpathbias.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "lib/compress/compress.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircommon/directory.h"
 #include "feature/relay/dns.h"
+#include "feature/relay/circuitbuild_relay.h"
 #include "feature/stats/geoip_stats.h"
 #include "feature/hs/hs_cache.h"
 #include "core/mainloop/mainloop.h"
@@ -80,7 +82,6 @@
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/routerlist.h"
 #include "core/or/scheduler.h"
-#include "feature/stats/rephist.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/cell_queue_st.h"
@@ -93,15 +94,12 @@
 #include "core/or/origin_circuit_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "core/or/socks_request_st.h"
-
-#include "lib/intmath/weakrng.h"
+#include "core/or/sendme.h"
 
 static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell,
                                             cell_direction_t cell_direction,
                                             crypt_path_t *layer_hint);
 
-static void circuit_consider_sending_sendme(circuit_t *circ,
-                                            crypt_path_t *layer_hint);
 static void circuit_resume_edge_reading(circuit_t *circ,
                                         crypt_path_t *layer_hint);
 static int circuit_resume_edge_reading_helper(edge_connection_t *conn,
@@ -134,9 +132,6 @@ uint64_t stats_n_relay_cells_delivered = 0;
  * reached (see append_cell_to_circuit_queue()) */
 uint64_t stats_n_circ_max_cell_reached = 0;
 
-/** Used to tell which stream to read from first on a circuit. */
-static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT;
-
 /**
  * Update channel usage state based on the type of relay cell and
  * circuit properties.
@@ -253,6 +248,10 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   if (recognized) {
     edge_connection_t *conn = NULL;
 
+    /* Recognized cell, the cell digest has been updated, we'll record it for
+     * the SENDME if need be. */
+    sendme_record_received_cell_digest(circ, layer_hint);
+
     if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
       if (pathbias_check_probe_response(circ, cell) == -1) {
         pathbias_count_valid_cells(circ, cell);
@@ -267,8 +266,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_OUT) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending away from origin.");
-      if ((reason=connection_edge_process_relay_cell(cell, circ, conn, NULL))
-          < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn, NULL);
+      if (reason < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "connection_edge_process_relay_cell (away from origin) "
                "failed.");
@@ -278,8 +277,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_IN) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending to origin.");
-      if ((reason = connection_edge_process_relay_cell(cell, circ, conn,
-                                                       layer_hint)) < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn,
+                                                  layer_hint);
+      if (reason < 0) {
         /* If a client is trying to connect to unknown hidden service port,
          * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit.
          * Do not log warn as this is an expected behavior for a service. */
@@ -293,7 +293,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     return 0;
   }
 
-  /* not recognized. pass it on. */
+  /* not recognized. inform circpad and pass it on. */
+  circpad_deliver_unrecognized_cell_events(circ, cell_direction);
+
   if (cell_direction == CELL_DIRECTION_OUT) {
     cell->circ_id = circ->n_circ_id; /* switch it */
     chan = circ->n_chan;
@@ -353,11 +355,11 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
  *  - Encrypt it to the right layer
  *  - Append it to the appropriate cell_queue on <b>circ</b>.
  */
-static int
-circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
+MOCK_IMPL(int,
+circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
                            cell_direction_t cell_direction,
                            crypt_path_t *layer_hint, streamid_t on_stream,
-                           const char *filename, int lineno)
+                           const char *filename, int lineno))
 {
   channel_t *chan; /* where to send the cell */
 
@@ -524,6 +526,8 @@ relay_command_to_string(uint8_t command)
     case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK";
     case RELAY_COMMAND_EXTEND2: return "EXTEND2";
     case RELAY_COMMAND_EXTENDED2: return "EXTENDED2";
+    case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE";
+    case RELAY_COMMAND_PADDING_NEGOTIATED: return "PADDING_NEGOTIATED";
     default:
       tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u",
                    (unsigned)command);
@@ -531,6 +535,64 @@ relay_command_to_string(uint8_t command)
   }
 }
 
+/** When padding a cell with randomness, leave this many zeros after the
+ * payload. */
+#define CELL_PADDING_GAP 4
+
+/** Return the offset where the padding should start. The <b>data_len</b> is
+ * the relay payload length expected to be put in the cell. It can not be
+ * bigger than RELAY_PAYLOAD_SIZE else this function assert().
+ *
+ * Value will always be smaller than CELL_PAYLOAD_SIZE because this offset is
+ * for the entire cell length not just the data payload length. Zero is
+ * returned if there is no room for padding.
+ *
+ * This function always skips the first 4 bytes after the payload because
+ * having some unused zero bytes has saved us a lot of times in the past. */
+
+STATIC size_t
+get_pad_cell_offset(size_t data_len)
+{
+  /* This is never supposed to happen but in case it does, stop right away
+   * because if tor is tricked somehow into not adding random bytes to the
+   * payload with this function returning 0 for a bad data_len, the entire
+   * authenticated SENDME design can be bypassed leading to bad denial of
+   * service attacks. */
+  tor_assert(data_len <= RELAY_PAYLOAD_SIZE);
+
+  /* If the offset is larger than the cell payload size, we return an offset
+   * of zero indicating that no padding needs to be added. */
+  size_t offset = RELAY_HEADER_SIZE + data_len + CELL_PADDING_GAP;
+  if (offset >= CELL_PAYLOAD_SIZE) {
+    return 0;
+  }
+  return offset;
+}
+
+/* Add random bytes to the unused portion of the payload, to foil attacks
+ * where the other side can predict all of the bytes in the payload and thus
+ * compute the authenticated SENDME cells without seeing the traffic. See
+ * proposal 289. */
+static void
+pad_cell_payload(uint8_t *cell_payload, size_t data_len)
+{
+  size_t pad_offset, pad_len;
+
+  tor_assert(cell_payload);
+
+  pad_offset = get_pad_cell_offset(data_len);
+  if (pad_offset == 0) {
+    /* We can't add padding so we are done. */
+    return;
+  }
+
+  /* Remember here that the cell_payload is the length of the header and
+   * payload size so we offset it using the full length of the cell. */
+  pad_len = CELL_PAYLOAD_SIZE - pad_offset;
+  crypto_fast_rng_getbytes(get_thread_fast_rng(),
+                           cell_payload + pad_offset, pad_len);
+}
+
 /** Make a relay cell out of <b>relay_command</b> and <b>payload</b>, and send
  * it onto the open circuit <b>circ</b>. <b>stream_id</b> is the ID on
  * <b>circ</b> for the stream that's sending the relay cell, or 0 if it's a
@@ -574,11 +636,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
   if (payload_len)
     memcpy(cell.payload+RELAY_HEADER_SIZE, payload, payload_len);
 
+  /* Add random padding to the cell if we can. */
+  pad_cell_payload(cell.payload, payload_len);
+
   log_debug(LD_OR,"delivering %d cell %s.", relay_command,
             cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward");
 
-  if (relay_command == RELAY_COMMAND_DROP)
-    rep_hist_padding_count_write(PADDING_TYPE_DROP);
+  /* Tell circpad we're sending a relay cell */
+  circpad_deliver_sent_relay_cell_events(circ, relay_command);
 
   /* If we are sending an END cell and this circuit is used for a tunneled
    * directory request, advance its state. */
@@ -602,7 +667,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
        * one of them.  Don't worry about the conn protocol version:
        * append_cell_to_circuit_queue will fix it up. */
       cell.command = CELL_RELAY_EARLY;
-      --origin_circ->remaining_relay_early_cells;
+      /* If we're out of relay early cells, tell circpad */
+      if (--origin_circ->remaining_relay_early_cells == 0)
+        circpad_machine_event_circ_has_no_relay_early(origin_circ);
       log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.",
                 (int)origin_circ->remaining_relay_early_cells);
       /* Memorize the command that is sent as RELAY_EARLY cell; helps debug
@@ -639,6 +706,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
     circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL);
     return -1;
   }
+
+  /* If applicable, note the cell digest for the SENDME version 1 purpose if
+   * we need to. This call needs to be after the circuit_package_relay_cell()
+   * because the cell digest is set within that function. */
+  if (relay_command == RELAY_COMMAND_DATA) {
+    sendme_record_cell_digest_on_circ(circ, cpath_layer);
+  }
+
   return 0;
 }
 
@@ -792,7 +867,7 @@ connection_ap_process_end_not_open(
               ttl = (int)ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+5));
           } else if (rh->length == 17 || rh->length == 21) {
             tor_addr_from_ipv6_bytes(&addr,
-                                (char*)(cell->payload+RELAY_HEADER_SIZE+1));
+                                     (cell->payload+RELAY_HEADER_SIZE+1));
             if (rh->length == 21)
               ttl = (int)ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+17));
           }
@@ -1017,7 +1092,7 @@ connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
       return -1;
     if (get_uint8(payload + 4) != 6)
       return -1;
-    tor_addr_from_ipv6_bytes(addr_out, (char*)(payload + 5));
+    tor_addr_from_ipv6_bytes(addr_out, (payload + 5));
     bytes = ntohl(get_uint32(payload + 21));
     if (bytes <= INT32_MAX)
       *ttl_out = (int) bytes;
@@ -1090,7 +1165,7 @@ resolved_cell_parse(const cell_t *cell, const relay_header_t *rh,
       if (answer_len != 16)
         goto err;
       addr = tor_malloc_zero(sizeof(*addr));
-      tor_addr_from_ipv6_bytes(&addr->addr, (const char*) cp);
+      tor_addr_from_ipv6_bytes(&addr->addr, cp);
       cp += 16;
       addr->ttl = ntohl(get_uint32(cp));
       cp += 4;
@@ -1447,85 +1522,108 @@ relay_crypt_from_last_hop(origin_circuit_t *circ, crypt_path_t *layer_hint)
   return true;
 }
 
-/** An incoming relay cell has arrived on circuit <b>circ</b>. If
- * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
- * destined for <b>conn</b>.
+/** Process a SENDME cell that arrived on <b>circ</b>. If it is a stream level
+ * cell, it is destined for the given <b>conn</b>. If it is a circuit level
+ * cell, it is destined for the <b>layer_hint</b>. The <b>domain</b> is the
+ * logging domain that should be used.
  *
- * If <b>layer_hint</b> is defined, then we're the origin of the
- * circuit, and it specifies the hop that packaged <b>cell</b>.
- *
- * Return -reason if you want to warn and tear down the circuit, else 0.
- */
-STATIC int
-connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
-                                   edge_connection_t *conn,
-                                   crypt_path_t *layer_hint)
+ * Return 0 if everything went well or a negative value representing a circuit
+ * end reason on error for which the caller is responsible for closing it. */
+static int
+process_sendme_cell(const relay_header_t *rh, const cell_t *cell,
+                    circuit_t *circ, edge_connection_t *conn,
+                    crypt_path_t *layer_hint, int domain)
 {
-  static int num_seen=0;
-  relay_header_t rh;
-  unsigned domain = layer_hint?LD_APP:LD_EXIT;
-  int reason;
-  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
-                            * that's in the EXIT_CONN_STATE_RESOLVING
-                            * or EXIT_CONN_STATE_CONNECTING states. */
-
-  tor_assert(cell);
-  tor_assert(circ);
+  int ret;
 
-  relay_header_unpack(&rh, cell->payload);
-//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
-  num_seen++;
-  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
-            num_seen, rh.command, rh.stream_id);
+  tor_assert(rh);
 
-  if (rh.length > RELAY_PAYLOAD_SIZE) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Relay cell length field too long. Closing circuit.");
-    return - END_CIRC_REASON_TORPROTOCOL;
+  if (!rh->stream_id) {
+    /* Circuit level SENDME cell. */
+    ret = sendme_process_circuit_level(layer_hint, circ,
+                                       cell->payload + RELAY_HEADER_SIZE,
+                                       rh->length);
+    if (ret < 0) {
+      return ret;
+    }
+    /* Resume reading on any streams now that we've processed a valid
+     * SENDME cell that updated our package window. */
+    circuit_resume_edge_reading(circ, layer_hint);
+    /* We are done, the rest of the code is for the stream level. */
+    return 0;
   }
 
-  if (rh.stream_id == 0) {
-    switch (rh.command) {
-      case RELAY_COMMAND_BEGIN:
-      case RELAY_COMMAND_CONNECTED:
-      case RELAY_COMMAND_END:
-      case RELAY_COMMAND_RESOLVE:
-      case RELAY_COMMAND_RESOLVED:
-      case RELAY_COMMAND_BEGIN_DIR:
-        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
-               "stream_id. Dropping.", (int)rh.command);
-        return 0;
-      default:
-        ;
+  /* No connection, might be half edge state. We are done if so. */
+  if (!conn) {
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+      if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                               rh->stream_id)) {
+        circuit_read_valid_data(ocirc, rh->length);
+        log_info(domain, "Sendme cell on circ %u valid on half-closed "
+                         "stream id %d",
+                 ocirc->global_identifier, rh->stream_id);
+      }
     }
+
+    log_info(domain, "SENDME cell dropped, unknown stream (streamid %d).",
+             rh->stream_id);
+    return 0;
   }
 
-  /* either conn is NULL, in which case we've got a control cell, or else
-   * conn points to the recognized stream. */
+  /* Stream level SENDME cell. */
+  ret = sendme_process_stream_level(conn, circ, rh->length);
+  if (ret < 0) {
+    /* Means we need to close the circuit with reason ret. */
+    return ret;
+  }
 
-  if (conn && !connection_state_is_open(TO_CONN(conn))) {
-    if (conn->base_.type == CONN_TYPE_EXIT &&
-        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
-         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
-        rh.command == RELAY_COMMAND_DATA) {
-      /* Allow DATA cells to be delivered to an exit node in state
-       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
-       * This speeds up HTTP, for example. */
-      optimistic_data = 1;
-    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
-      log_warn(LD_BUG, "Somehow I had a connection that matched a "
-               "data cell with stream ID 0.");
-    } else {
-      return connection_edge_process_relay_cell_not_open(
-               &rh, cell, circ, conn, layer_hint);
-    }
+  /* We've now processed properly a SENDME cell, all windows have been
+   * properly updated, we'll read on the edge connection to see if we can
+   * get data out towards the end point (Exit or client) since we are now
+   * allowed to deliver more cells. */
+
+  if (circuit_queue_streams_are_blocked(circ)) {
+    /* Still waiting for queue to flush; don't touch conn */
+    return 0;
+  }
+  connection_start_reading(TO_CONN(conn));
+  /* handle whatever might still be on the inbuf */
+  if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
+    /* (We already sent an end cell if possible) */
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
   }
+  return 0;
+}
 
-  switch (rh.command) {
-    case RELAY_COMMAND_DROP:
-      rep_hist_padding_count_read(PADDING_TYPE_DROP);
-//      log_info(domain,"Got a relay-level padding cell. Dropping.");
-      return 0;
+/** A helper for connection_edge_process_relay_cell(): Actually handles the
+ *  cell that we received on the connection.
+ *
+ *  The arguments are the same as in the parent function
+ *  connection_edge_process_relay_cell(), plus the relay header <b>rh</b> as
+ *  unpacked by the parent function, and <b>optimistic_data</b> as set by the
+ *  parent function.
+ */
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data)
+{
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int reason;
+
+  tor_assert(rh);
+
+  /* First pass the cell to the circuit padding subsystem, in case it's a
+   * padding cell or circuit that should be handled there. */
+  if (circpad_check_received_cell(cell, circ, layer_hint, rh) == 0) {
+    log_debug(domain, "Cell handled as circuit padding");
+    return 0;
+  }
+
+  /* Now handle all the other commands */
+  switch (rh->command) {
     case RELAY_COMMAND_BEGIN:
     case RELAY_COMMAND_BEGIN_DIR:
       if (layer_hint &&
@@ -1546,7 +1644,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "Begin cell for known stream. Dropping.");
         return 0;
       }
-      if (rh.command == RELAY_COMMAND_BEGIN_DIR &&
+      if (rh->command == RELAY_COMMAND_BEGIN_DIR &&
           circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) {
         /* Assign this circuit and its app-ward OR connection a unique ID,
          * so that we can measure download times. The local edge and dir
@@ -1559,24 +1657,21 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       return connection_exit_begin_conn(cell, circ);
     case RELAY_COMMAND_DATA:
       ++stats_n_data_cells_received;
-      if (( layer_hint && --layer_hint->deliver_window < 0) ||
-          (!layer_hint && --circ->deliver_window < 0)) {
+
+      /* Update our circuit-level deliver window that we received a DATA cell.
+       * If the deliver window goes below 0, we end the circuit and stream due
+       * to a protocol failure. */
+      if (sendme_circuit_data_received(circ, layer_hint) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) circ deliver_window below 0. Killing.");
-        if (conn) {
-          /* XXXX Do we actually need to do this?  Will killing the circuit
-           * not send an END and mark the stream for close as appropriate? */
-          connection_edge_end(conn, END_STREAM_REASON_TORPROTOCOL);
-          connection_mark_for_close(TO_CONN(conn));
-        }
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
-      log_debug(domain,"circ deliver_window now %d.", layer_hint ?
-                layer_hint->deliver_window : circ->deliver_window);
 
-      circuit_consider_sending_sendme(circ, layer_hint);
+      /* Consider sending a circuit-level SENDME cell. */
+      sendme_circuit_consider_sending(circ, layer_hint);
 
-      if (rh.stream_id == 0) {
+      if (rh->stream_id == 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero "
                "stream_id. Dropping.");
         return 0;
@@ -1584,32 +1679,37 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (connection_half_edge_is_valid_data(ocirc->half_streams,
-                                                 rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
+                                                 rh->stream_id)) {
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "data cell on circ %u valid on half-closed "
-                     "stream id %d", ocirc->global_identifier, rh.stream_id);
+                     "stream id %d", ocirc->global_identifier, rh->stream_id);
           }
         }
 
         log_info(domain,"data cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
+                 rh->stream_id);
         return 0;
       }
 
-      if (--conn->deliver_window < 0) { /* is it below 0 after decrement? */
+      /* Update our stream-level deliver window that we just received a DATA
+       * cell. Going below 0 means we have a protocol level error so the
+       * stream and circuit are closed. */
+
+      if (sendme_stream_data_received(conn) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) conn deliver_window below 0. Killing.");
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
       /* Total all valid application bytes delivered */
-      if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      if (CIRCUIT_IS_ORIGIN(circ) && rh->length > 0) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
 
-      stats_n_data_bytes_received += rh.length;
+      stats_n_data_bytes_received += rh->length;
       connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE),
-                              rh.length, TO_CONN(conn));
+                              rh->length, TO_CONN(conn));
 
 #ifdef MEASUREMENTS_21206
       /* Count number of RELAY_DATA cells received on a linked directory
@@ -1625,26 +1725,26 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         /* Only send a SENDME if we're not getting optimistic data; otherwise
          * a SENDME could arrive before the CONNECTED.
          */
-        connection_edge_consider_sending_sendme(conn);
+        sendme_connection_edge_consider_sending(conn);
       }
 
       return 0;
     case RELAY_COMMAND_END:
-      reason = rh.length > 0 ?
+      reason = rh->length > 0 ?
         get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC;
       if (!conn) {
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (relay_crypt_from_last_hop(ocirc, layer_hint) &&
               connection_half_edge_is_valid_end(ocirc->half_streams,
-                                                rh.stream_id)) {
+                                                rh->stream_id)) {
 
-            circuit_read_valid_data(ocirc, rh.length);
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "end cell (%s) on circ %u valid on half-closed "
                      "stream id %d",
                      stream_end_reason_to_string(reason),
-                     ocirc->global_identifier, rh.stream_id);
+                     ocirc->global_identifier, rh->stream_id);
             return 0;
           }
         }
@@ -1676,7 +1776,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
 
         /* Total all valid application bytes delivered */
         if (CIRCUIT_IS_ORIGIN(circ)) {
-          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
         }
       }
       return 0;
@@ -1684,7 +1784,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_EXTEND2: {
       static uint64_t total_n_extend=0, total_nonearly=0;
       total_n_extend++;
-      if (rh.stream_id) {
+      if (rh->stream_id) {
         log_fn(LOG_PROTOCOL_WARN, domain,
                "'extend' cell received for non-zero stream. Dropping.");
         return 0;
@@ -1725,9 +1825,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_debug(domain,"Got an extended cell! Yay.");
       {
         extended_cell_t extended_cell;
-        if (extended_cell_parse(&extended_cell, rh.command,
+        if (extended_cell_parse(&extended_cell, rh->command,
                         (const uint8_t*)cell->payload+RELAY_HEADER_SIZE,
-                        rh.length)<0) {
+                        rh->length)<0) {
           log_warn(LD_PROTOCOL,
                    "Can't parse EXTENDED cell; killing circuit.");
           return -END_CIRC_REASON_TORPROTOCOL;
@@ -1745,7 +1845,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       }
       /* Total all valid bytes delivered. */
       if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
       return 0;
     case RELAY_COMMAND_TRUNCATE:
@@ -1789,7 +1889,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
        * circuit is being torn down anyway, though.  */
       if (CIRCUIT_IS_ORIGIN(circ)) {
         circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
+                                rh->length);
       }
       circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
                         get_uint8(cell->payload + RELAY_HEADER_SIZE));
@@ -1804,11 +1904,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       if (CIRCUIT_IS_ORIGIN(circ)) {
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (connection_half_edge_is_valid_connected(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "connected cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1816,102 +1916,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_info(domain,
                "'connected' received on circid %u for streamid %d, "
                "no conn attached anymore. Ignoring.",
-               (unsigned)circ->n_circ_id, rh.stream_id);
+               (unsigned)circ->n_circ_id, rh->stream_id);
       return 0;
     case RELAY_COMMAND_SENDME:
-      if (!rh.stream_id) {
-        if (layer_hint) {
-          if (layer_hint->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from exit relay. "
-                   "Closing circ.");
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          layer_hint->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.",
-                    layer_hint->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-
-          /* We count circuit-level sendme's as valid delivered data because
-           * they are rate limited.
-           */
-          if (CIRCUIT_IS_ORIGIN(circ)) {
-            circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                    rh.length);
-          }
-
-        } else {
-          if (circ->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&client_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from client. "
-                   "Closing circ (window %d).",
-                   circ->package_window);
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          circ->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,
-                    "circ-level sendme at non-origin, packagewindow %d.",
-                    circ->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-        }
-        return 0;
-      }
-      if (!conn) {
-        if (CIRCUIT_IS_ORIGIN(circ)) {
-          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-          if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
-                                                   rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
-            log_info(domain,
-                    "sendme cell on circ %u valid on half-closed "
-                    "stream id %d", ocirc->global_identifier, rh.stream_id);
-          }
-        }
-
-        log_info(domain,"sendme cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
-        return 0;
-      }
-
-      /* Don't allow the other endpoint to request more than our maximum
-       * (i.e. initial) stream SENDME window worth of data. Well-behaved
-       * stock clients will not request more than this max (as per the check
-       * in the while loop of connection_edge_consider_sending_sendme()).
-       */
-      if (conn->package_window + STREAMWINDOW_INCREMENT >
-          STREAMWINDOW_START_MAX) {
-        static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
-        log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
-               "Unexpected stream sendme cell. Closing circ (window %d).",
-               conn->package_window);
-        return -END_CIRC_REASON_TORPROTOCOL;
-      }
-
-      /* At this point, the stream sendme is valid */
-      if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
-      }
-
-      conn->package_window += STREAMWINDOW_INCREMENT;
-      log_debug(domain,"stream-level sendme, packagewindow now %d.",
-                conn->package_window);
-      if (circuit_queue_streams_are_blocked(circ)) {
-        /* Still waiting for queue to flush; don't touch conn */
-        return 0;
-      }
-      connection_start_reading(TO_CONN(conn));
-      /* handle whatever might still be on the inbuf */
-      if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
-        /* (We already sent an end cell if possible) */
-        connection_mark_for_close(TO_CONN(conn));
-        return 0;
-      }
-      return 0;
+      return process_sendme_cell(rh, cell, circ, conn, layer_hint, domain);
     case RELAY_COMMAND_RESOLVE:
       if (layer_hint) {
         log_fn(LOG_PROTOCOL_WARN, LD_APP,
@@ -1940,11 +1948,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (relay_crypt_from_last_hop(ocirc, layer_hint) &&
             connection_half_edge_is_valid_resolved(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "resolved cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1962,17 +1970,96 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_INTRO_ESTABLISHED:
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       rend_process_relay_cell(circ, layer_hint,
-                              rh.command, rh.length,
+                              rh->command, rh->length,
                               cell->payload+RELAY_HEADER_SIZE);
       return 0;
   }
   log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Received unknown relay command %d. Perhaps the other side is using "
          "a newer version of Tor? Dropping.",
-         rh.command);
+         rh->command);
   return 0; /* for forward compatibility, don't kill the circuit */
 }
 
+/** An incoming relay cell has arrived on circuit <b>circ</b>. If
+ * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
+ * destined for <b>conn</b>.
+ *
+ * If <b>layer_hint</b> is defined, then we're the origin of the
+ * circuit, and it specifies the hop that packaged <b>cell</b>.
+ *
+ * Return -reason if you want to warn and tear down the circuit, else 0.
+ */
+STATIC int
+connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
+                                   edge_connection_t *conn,
+                                   crypt_path_t *layer_hint)
+{
+  static int num_seen=0;
+  relay_header_t rh;
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
+                            * that's in the EXIT_CONN_STATE_RESOLVING
+                            * or EXIT_CONN_STATE_CONNECTING states. */
+
+  tor_assert(cell);
+  tor_assert(circ);
+
+  relay_header_unpack(&rh, cell->payload);
+//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
+  num_seen++;
+  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
+            num_seen, rh.command, rh.stream_id);
+
+  if (rh.length > RELAY_PAYLOAD_SIZE) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Relay cell length field too long. Closing circuit.");
+    return - END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  if (rh.stream_id == 0) {
+    switch (rh.command) {
+      case RELAY_COMMAND_BEGIN:
+      case RELAY_COMMAND_CONNECTED:
+      case RELAY_COMMAND_END:
+      case RELAY_COMMAND_RESOLVE:
+      case RELAY_COMMAND_RESOLVED:
+      case RELAY_COMMAND_BEGIN_DIR:
+        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
+               "stream_id. Dropping.", (int)rh.command);
+        return 0;
+      default:
+        ;
+    }
+  }
+
+  /* Tell circpad that we've received a recognized cell */
+  circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint);
+
+  /* either conn is NULL, in which case we've got a control cell, or else
+   * conn points to the recognized stream. */
+  if (conn && !connection_state_is_open(TO_CONN(conn))) {
+    if (conn->base_.type == CONN_TYPE_EXIT &&
+        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
+         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
+        rh.command == RELAY_COMMAND_DATA) {
+      /* Allow DATA cells to be delivered to an exit node in state
+       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
+       * This speeds up HTTP, for example. */
+      optimistic_data = 1;
+    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
+      log_warn(LD_BUG, "Somehow I had a connection that matched a "
+               "data cell with stream ID 0.");
+    } else {
+      return connection_edge_process_relay_cell_not_open(
+               &rh, cell, circ, conn, layer_hint);
+    }
+  }
+
+  return handle_relay_cell_command(cell, circ, conn, layer_hint,
+                              &rh, optimistic_data);
+}
+
 /** How many relay_data cells have we built, ever? */
 uint64_t stats_n_data_cells_packaged = 0;
 /** How many bytes of data have we put in relay_data cells have we built,
@@ -1986,6 +2073,84 @@ uint64_t stats_n_data_cells_received = 0;
  * ever received were completely full of data. */
 uint64_t stats_n_data_bytes_received = 0;
 
+/**
+ * Called when initializing a circuit, or when we have reached the end of the
+ * window in which we need to send some randomness so that incoming sendme
+ * cells will be unpredictable.  Resets the flags and picks a new window.
+ */
+void
+circuit_reset_sendme_randomness(circuit_t *circ)
+{
+  circ->have_sent_sufficiently_random_cell = 0;
+  circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 +
+    crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2);
+}
+
+/**
+ * Any relay data payload containing fewer than this many real bytes is
+ * considered to have enough randomness to.
+ **/
+#define RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES \
+  (RELAY_PAYLOAD_SIZE - CELL_PADDING_GAP - 16)
+
+/**
+ * Helper. Return the number of bytes that should be put into a cell from a
+ * given edge connection on which <b>n_available</b> bytes are available.
+ */
+STATIC size_t
+connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                           int package_partial,
+                                           circuit_t *on_circuit)
+{
+  if (!n_available)
+    return 0;
+
+  /* Do we need to force this payload to have space for randomness? */
+  const bool force_random_bytes =
+    (on_circuit->send_randomness_after_n_cells == 0) &&
+    (! on_circuit->have_sent_sufficiently_random_cell);
+
+  /* At most how much would we like to send in this cell? */
+  size_t target_length;
+  if (force_random_bytes) {
+    target_length = RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES;
+  } else {
+    target_length = RELAY_PAYLOAD_SIZE;
+  }
+
+  /* Decide how many bytes we will actually put into this cell. */
+  size_t package_length;
+  if (n_available >= target_length) { /* A full payload is available. */
+    package_length = target_length;
+  } else { /* not a full payload available */
+    if (package_partial)
+      package_length = n_available; /* just take whatever's available now */
+    else
+      return 0; /* nothing to do until we have a full payload */
+  }
+
+  /* If we reach this point, we will be definitely sending the cell. */
+  tor_assert_nonfatal(package_length > 0);
+
+  if (package_length <= RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES) {
+    /* This cell will have enough randomness in the padding to make a future
+     * sendme cell unpredictable. */
+    on_circuit->have_sent_sufficiently_random_cell = 1;
+  }
+
+  if (on_circuit->send_randomness_after_n_cells == 0) {
+    /* Either this cell, or some previous cell, had enough padding to
+     * ensure sendme unpredictability. */
+    tor_assert_nonfatal(on_circuit->have_sent_sufficiently_random_cell);
+    /* Pick a new interval in which we need to send randomness. */
+    circuit_reset_sendme_randomness(on_circuit);
+  }
+
+  --on_circuit->send_randomness_after_n_cells;
+
+  return package_length;
+}
+
 /** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or
  * <b>package_partial</b> is true), and the appropriate package windows aren't
  * empty, grab a cell and send it down the circuit.
@@ -2058,17 +2223,14 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     bytes_to_process = connection_get_inbuf_len(TO_CONN(conn));
   }
 
-  if (!bytes_to_process)
+  length = connection_edge_get_inbuf_bytes_to_package(bytes_to_process,
+                                                      package_partial, circ);
+  if (!length)
     return 0;
 
-  if (!package_partial && bytes_to_process < RELAY_PAYLOAD_SIZE)
-    return 0;
+  /* If we reach this point, we will definitely be packaging bytes into
+   * a cell. */
 
-  if (bytes_to_process > RELAY_PAYLOAD_SIZE) {
-    length = RELAY_PAYLOAD_SIZE;
-  } else {
-    length = bytes_to_process;
-  }
   stats_n_data_bytes_packaged += length;
   stats_n_data_cells_packaged += 1;
 
@@ -2103,15 +2265,17 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     return 0;
   }
 
-  if (!cpath_layer) { /* non-rendezvous exit */
-    tor_assert(circ->package_window > 0);
-    circ->package_window--;
-  } else { /* we're an AP, or an exit on a rendezvous circ */
-    tor_assert(cpath_layer->package_window > 0);
-    cpath_layer->package_window--;
+  /* Handle the circuit-level SENDME package window. */
+  if (sendme_note_circuit_data_packaged(circ, cpath_layer) < 0) {
+    /* Package window has gone under 0. Protocol issue. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Circuit package window is below 0. Closing circuit.");
+    conn->end_reason = END_STREAM_REASON_TORPROTOCOL;
+    return -1;
   }
 
-  if (--conn->package_window <= 0) { /* is it 0 after decrement? */
+  /* Handle the stream-level SENDME package window. */
+  if (sendme_note_stream_data_packaged(conn) < 0) {
     connection_stop_reading(TO_CONN(conn));
     log_debug(domain,"conn->package_window reached 0.");
     circuit_consider_stop_edge_reading(circ, cpath_layer);
@@ -2129,42 +2293,6 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
   goto repeat_connection_edge_package_raw_inbuf;
 }
 
-/** Called when we've just received a relay data cell, when
- * we've just finished flushing all bytes to stream <b>conn</b>,
- * or when we've flushed *some* bytes to the stream <b>conn</b>.
- *
- * If conn->outbuf is not too full, and our deliver window is
- * low, send back a suitable number of stream-level sendme cells.
- */
-void
-connection_edge_consider_sending_sendme(edge_connection_t *conn)
-{
-  circuit_t *circ;
-
-  if (connection_outbuf_too_full(TO_CONN(conn)))
-    return;
-
-  circ = circuit_get_by_edge_conn(conn);
-  if (!circ) {
-    /* this can legitimately happen if the destroy has already
-     * arrived and torn down the circuit */
-    log_info(LD_APP,"No circuit associated with conn. Skipping.");
-    return;
-  }
-
-  while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) {
-    log_debug(conn->base_.type == CONN_TYPE_AP ?LD_APP:LD_EXIT,
-              "Outbuf %d, Queuing stream sendme.",
-              (int)conn->base_.outbuf_flushlen);
-    conn->deliver_window += STREAMWINDOW_INCREMENT;
-    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
-                                     NULL, 0) < 0) {
-      log_warn(LD_APP,"connection_edge_send_command failed. Skipping.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The circuit <b>circ</b> has received a circuit-level sendme
  * (on hop <b>layer_hint</b>, if we're the OP). Go through all the
  * attached streams and let them resume reading and packaging, if
@@ -2187,12 +2315,6 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
                                        circ, layer_hint);
 }
 
-void
-stream_choice_seed_weak_rng(void)
-{
-  crypto_seed_weak_rng(&stream_choice_rng);
-}
-
 /** A helper function for circuit_resume_edge_reading() above.
  * The arguments are the same, except that <b>conn</b> is the head
  * of a linked list of edge streams that should each be considered.
@@ -2244,7 +2366,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn,
     int num_streams = 0;
     for (conn = first_conn; conn; conn = conn->next_stream) {
       num_streams++;
-      if (tor_weak_random_one_in_n(&stream_choice_rng, num_streams)) {
+
+      if (crypto_fast_rng_one_in_n(get_thread_fast_rng(), num_streams)) {
         chosen_stream = conn;
       }
       /* Invariant: chosen_stream has been chosen uniformly at random from
@@ -2386,33 +2509,6 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
   return 0;
 }
 
-/** Check if the deliver_window for circuit <b>circ</b> (at hop
- * <b>layer_hint</b> if it's defined) is low enough that we should
- * send a circuit-level sendme back down the circuit. If so, send
- * enough sendmes that the window would be overfull if we sent any
- * more.
- */
-static void
-circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint)
-{
-//  log_fn(LOG_INFO,"Considering: layer_hint is %s",
-//         layer_hint ? "defined" : "null");
-  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
-          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
-    log_debug(LD_CIRC,"Queuing circuit sendme.");
-    if (layer_hint)
-      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
-    else
-      circ->deliver_window += CIRCWINDOW_INCREMENT;
-    if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
-                                     NULL, 0, layer_hint) < 0) {
-      log_warn(LD_CIRC,
-               "relay_send_command_from_edge failed. Circuit's closed.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The total number of cells we have allocated. */
 static size_t total_cells_allocated = 0;
 
@@ -2787,7 +2883,7 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan,
 }
 
 /** Extract the command from a packed cell. */
-static uint8_t
+uint8_t
 packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids)
 {
   if (wide_circ_ids) {
@@ -3142,7 +3238,7 @@ decode_address_from_payload(tor_addr_t *addr_out, const uint8_t *payload,
   case RESOLVED_TYPE_IPV6:
     if (payload[1] != 16)
       return NULL;
-    tor_addr_from_ipv6_bytes(addr_out, (char*)(payload+2));
+    tor_addr_from_ipv6_bytes(addr_out, (payload+2));
     break;
   default:
     tor_addr_make_unspec(addr_out);
diff --git a/src/core/or/relay.h b/src/core/or/relay.h
index 7cc3c43e43..31bed0e01b 100644
--- a/src/core/or/relay.h
+++ b/src/core/or/relay.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,6 +42,7 @@ int connection_edge_package_raw_inbuf(edge_connection_t *conn,
                                       int package_partial,
                                       int *max_cells);
 void connection_edge_consider_sending_sendme(edge_connection_t *conn);
+void circuit_reset_sendme_randomness(circuit_t *circ);
 
 extern uint64_t stats_n_data_cells_packaged;
 extern uint64_t stats_n_data_bytes_packaged;
@@ -78,6 +79,11 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue,
 void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out);
 MOCK_DECL(int, channel_flush_from_first_active_circuit,
           (channel_t *chan, int max));
+MOCK_DECL(int, circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno));
+
 void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
                              const char *file, int lineno);
 #define update_circuit_on_cmux(circ, direction) \
@@ -89,15 +95,19 @@ const uint8_t *decode_address_from_payload(tor_addr_t *addr_out,
                                         int payload_len);
 void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan);
 
-void stream_choice_seed_weak_rng(void);
-
 circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids);
+uint8_t packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids);
 
 #ifdef RELAY_PRIVATE
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data);
+
 STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
                          tor_addr_t *addr_out, int *ttl_out);
 /** An address-and-ttl tuple as yielded by resolved_cell_parse */
-typedef struct address_ttl_s {
+typedef struct address_ttl_t {
   tor_addr_t addr;
   char *hostname;
   int ttl;
@@ -117,8 +127,11 @@ STATIC int cell_queues_check_size(void);
 STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                                    edge_connection_t *conn,
                                    crypt_path_t *layer_hint);
+STATIC size_t get_pad_cell_offset(size_t payload_len);
+STATIC size_t connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                                      int package_partial,
+                                                      circuit_t *on_circuit);
 
 #endif /* defined(RELAY_PRIVATE) */
 
 #endif /* !defined(TOR_RELAY_H) */
-
diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h
index dafce257c7..d92f486a90 100644
--- a/src/core/or/relay_crypto_st.h
+++ b/src/core/or/relay_crypto_st.h
@@ -1,13 +1,18 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file relay_crypto_st.h
+ * @brief Relay-cell encryption state structure.
+ **/
+
 #ifndef RELAY_CRYPTO_ST_H
 #define RELAY_CRYPTO_ST_H
 
-#define crypto_cipher_t aes_cnt_cipher
+#define crypto_cipher_t aes_cnt_cipher_t
 struct crypto_cipher_t;
 struct crypto_digest_t;
 
@@ -25,7 +30,9 @@ struct relay_crypto_t {
   /** Digest state for cells heading away from the OR at this step. */
   struct crypto_digest_t *b_digest;
 
+  /** Digest used for the next SENDME cell if any. */
+  uint8_t sendme_digest[DIGEST_LEN];
 };
 #undef crypto_cipher_t
 
-#endif
+#endif /* !defined(RELAY_CRYPTO_ST_H) */
diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c
index 9f1a27d501..072d78128b 100644
--- a/src/core/or/scheduler.c
+++ b/src/core/or/scheduler.c
@@ -1,18 +1,17 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
 #include "app/config/config.h"
 
 #include "lib/evloop/compat_libevent.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #define SCHEDULER_KIST_PRIVATE
 #include "core/or/scheduler.h"
 #include "core/mainloop/mainloop.h"
-#include "lib/container/buffers.h"
-#define TOR_CHANNEL_INTERNAL_
+#include "lib/buf/buffers.h"
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/channeltls.h"
-#include "lib/evloop/compat_libevent.h"
 
 #include "core/or/or_connection_st.h"
 
@@ -43,7 +42,7 @@
  * circuit scheduler. It was supposed to prioritize circuits across many
  * channels, but wasn't effective. It is preserved in scheduler_vanilla.c.
  *
- * [0]: http://www.robgjansen.com/publications/kist-sec2014.pdf
+ * [0]: https://www.robgjansen.com/publications/kist-sec2014.pdf
  *
  * Then we actually got around to implementing KIST for real. We decided to
  * modularize the scheduler so new ones can be implemented. You can find KIST
@@ -267,7 +266,7 @@ select_scheduler(void)
           log_notice(LD_SCHED, "Scheduler type KIST has been disabled by "
                                "the consensus or no kernel support.");
         }
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
         log_info(LD_SCHED, "Scheduler type KIST not built in");
 #endif /* defined(HAVE_KIST_SUPPORT) */
         continue;
@@ -503,7 +502,12 @@ scheduler_free_all(void)
   the_scheduler = NULL;
 }
 
-/** Mark a channel as no longer ready to accept writes. */
+/** Mark a channel as no longer ready to accept writes.
+  *
+  * Possible state changes:
+  *  - SCHED_CHAN_PENDING -> SCHED_CHAN_WAITING_TO_WRITE
+  *  - SCHED_CHAN_WAITING_FOR_CELLS -> SCHED_CHAN_IDLE
+  */
 MOCK_IMPL(void,
 scheduler_channel_doesnt_want_writes,(channel_t *chan))
 {
@@ -514,31 +518,32 @@ scheduler_channel_doesnt_want_writes,(channel_t *chan))
     return;
   }
 
-  /* If it's already in pending, we can put it in waiting_to_write */
   if (chan->scheduler_state == SCHED_CHAN_PENDING) {
     /*
-     * It's in channels_pending, so it shouldn't be in any of
-     * the other lists.  It can't write any more, so it goes to
-     * channels_waiting_to_write.
+     * It has cells but no longer can write, so it becomes
+     * SCHED_CHAN_WAITING_TO_WRITE. It's in channels_pending, so we
+     * should remove it from the list.
      */
     smartlist_pqueue_remove(channels_pending,
                             scheduler_compare_channels,
                             offsetof(channel_t, sched_heap_idx),
                             chan);
     scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
-  } else {
+  } else if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
     /*
-     * It's not in pending, so it can't become waiting_to_write; it's
-     * either not in any of the lists (nothing to do) or it's already in
-     * waiting_for_cells (remove it, can't write any more).
+     * It does not have cells and no longer can write, so it becomes
+     * SCHED_CHAN_IDLE.
      */
-    if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
-      scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
-    }
+    scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
   }
 }
 
-/** Mark a channel as having waiting cells. */
+/** Mark a channel as having waiting cells.
+  *
+  * Possible state changes:
+  *  - SCHED_CHAN_WAITING_FOR_CELLS -> SCHED_CHAN_PENDING
+  *  - SCHED_CHAN_IDLE -> SCHED_CHAN_WAITING_TO_WRITE
+  */
 MOCK_IMPL(void,
 scheduler_channel_has_waiting_cells,(channel_t *chan))
 {
@@ -549,12 +554,11 @@ scheduler_channel_has_waiting_cells,(channel_t *chan))
     return;
   }
 
-  /* First, check if it's also writeable */
   if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
     /*
-     * It's in channels_waiting_for_cells, so it shouldn't be in any of
-     * the other lists.  It has waiting cells now, so it goes to
-     * channels_pending.
+     * It is able to write and now has cells, so it becomes
+     * SCHED_CHAN_PENDING. It must be added to the channels_pending
+     * list.
      */
     scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
     if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
@@ -566,16 +570,12 @@ scheduler_channel_has_waiting_cells,(channel_t *chan))
     /* If we made a channel pending, we potentially have scheduling work to
      * do. */
     the_scheduler->schedule();
-  } else {
+  } else if (chan->scheduler_state == SCHED_CHAN_IDLE) {
     /*
-     * It's not in waiting_for_cells, so it can't become pending; it's
-     * either not in any of the lists (we add it to waiting_to_write)
-     * or it's already in waiting_to_write or pending (we do nothing)
+     * It is not able to write but now has cells, so it becomes
+     * SCHED_CHAN_WAITING_TO_WRITE.
      */
-    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE ||
-          chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
-    }
+    scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
   }
 }
 
@@ -663,8 +663,12 @@ scheduler_release_channel,(channel_t *chan))
   scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
 }
 
-/** Mark a channel as ready to accept writes */
-
+/** Mark a channel as ready to accept writes.
+  * Possible state changes:
+  *
+  *  - SCHED_CHAN_WAITING_TO_WRITE -> SCHED_CHAN_PENDING
+  *  - SCHED_CHAN_IDLE -> SCHED_CHAN_WAITING_FOR_CELLS
+  */
 void
 scheduler_channel_wants_writes(channel_t *chan)
 {
@@ -675,10 +679,11 @@ scheduler_channel_wants_writes(channel_t *chan)
     return;
   }
 
-  /* If it's already in waiting_to_write, we can put it in pending */
   if (chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE) {
     /*
-     * It can write now, so it goes to channels_pending.
+     * It has cells and can now write, so it becomes
+     * SCHED_CHAN_PENDING. It must be added to the channels_pending
+     * list.
      */
     scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
     if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
@@ -689,15 +694,12 @@ scheduler_channel_wants_writes(channel_t *chan)
     }
     /* We just made a channel pending, we have scheduling work to do. */
     the_scheduler->schedule();
-  } else {
+  } else if (chan->scheduler_state == SCHED_CHAN_IDLE) {
     /*
-     * It's not in SCHED_CHAN_WAITING_TO_WRITE, so it can't become pending;
-     * it's either idle and goes to WAITING_FOR_CELLS, or it's a no-op.
+     * It does not have cells but can now write, so it becomes
+     * SCHED_CHAN_WAITING_FOR_CELLS.
      */
-    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS ||
-          chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
-    }
+    scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
   }
 }
 
diff --git a/src/core/or/scheduler.h b/src/core/or/scheduler.h
index 843be2603c..82df2b0b0f 100644
--- a/src/core/or/scheduler.h
+++ b/src/core/or/scheduler.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,7 +40,7 @@ typedef enum {
  * doesn't create any state for itself, thus it has nothing to free when Tor
  * is shutting down), then set that function pointer to NULL.
  */
-typedef struct scheduler_s {
+typedef struct scheduler_t {
   /* Scheduler type. This is used for logging when the scheduler is switched
    * during runtime. */
   scheduler_types_t type;
@@ -136,7 +136,9 @@ MOCK_DECL(void, scheduler_channel_has_waiting_cells, (channel_t *chan));
  * These functions are only visible to the scheduling system, the current
  * scheduler implementation, and tests.
  *****************************************************************************/
-#ifdef SCHEDULER_PRIVATE_
+#ifdef SCHEDULER_PRIVATE
+
+#include "ext/ht.h"
 
 /*********************************
  * Defined in scheduler.c
@@ -173,8 +175,8 @@ void scheduler_touch_channel(channel_t *chan);
 
 /* Socket table entry which holds information of a channel's socket and kernel
  * TCP information. Only used by KIST. */
-typedef struct socket_table_ent_s {
-  HT_ENTRY(socket_table_ent_s) node;
+typedef struct socket_table_ent_t {
+  HT_ENTRY(socket_table_ent_t) node;
   const channel_t *chan;
   /* Amount written this scheduling run */
   uint64_t written;
@@ -187,7 +189,7 @@ typedef struct socket_table_ent_s {
   uint32_t notsent;
 } socket_table_ent_t;
 
-typedef HT_HEAD(outbuf_table_s, outbuf_table_ent_s) outbuf_table_t;
+typedef HT_HEAD(outbuf_table_s, outbuf_table_ent_t) outbuf_table_t;
 
 MOCK_DECL(int, channel_should_write_to_kernel,
           (outbuf_table_t *table, channel_t *chan));
@@ -212,7 +214,6 @@ extern int32_t sched_run_interval;
 
 scheduler_t *get_vanilla_scheduler(void);
 
-#endif /* defined(SCHEDULER_PRIVATE_) */
+#endif /* defined(SCHEDULER_PRIVATE) */
 
 #endif /* !defined(TOR_SCHEDULER_H) */
-
diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c
index 79ecb0bc7e..c73d768f88 100644
--- a/src/core/or/scheduler_kist.c
+++ b/src/core/or/scheduler_kist.c
@@ -1,17 +1,22 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file scheduler_kist.c
+ * @brief Implements the KIST cell scheduler.
+ **/
+
 #define SCHEDULER_KIST_PRIVATE
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/nodelist/networkstatus.h"
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #include "core/or/scheduler.h"
 #include "lib/math/fp.h"
 
@@ -46,22 +51,22 @@ socket_table_ent_eq(const socket_table_ent_t *a, const socket_table_ent_t *b)
   return a->chan == b->chan;
 }
 
-typedef HT_HEAD(socket_table_s, socket_table_ent_s) socket_table_t;
+typedef HT_HEAD(socket_table_s, socket_table_ent_t) socket_table_t;
 
 static socket_table_t socket_table = HT_INITIALIZER();
 
-HT_PROTOTYPE(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
-             socket_table_ent_eq)
-HT_GENERATE2(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
-             socket_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+HT_PROTOTYPE(socket_table_s, socket_table_ent_t, node, socket_table_ent_hash,
+             socket_table_ent_eq);
+HT_GENERATE2(socket_table_s, socket_table_ent_t, node, socket_table_ent_hash,
+             socket_table_ent_eq, 0.6, tor_reallocarray, tor_free_);
 
 /* outbuf_table hash table stuff. The outbuf_table keeps track of which
  * channels have data sitting in their outbuf so the kist scheduler can force
  * a write from outbuf to kernel periodically during a run and at the end of a
  * run. */
 
-typedef struct outbuf_table_ent_s {
-  HT_ENTRY(outbuf_table_ent_s) node;
+typedef struct outbuf_table_ent_t {
+  HT_ENTRY(outbuf_table_ent_t) node;
   channel_t *chan;
 } outbuf_table_ent_t;
 
@@ -77,10 +82,10 @@ outbuf_table_ent_eq(const outbuf_table_ent_t *a, const outbuf_table_ent_t *b)
   return a->chan->global_identifier == b->chan->global_identifier;
 }
 
-HT_PROTOTYPE(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
-             outbuf_table_ent_eq)
-HT_GENERATE2(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
-             outbuf_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+HT_PROTOTYPE(outbuf_table_s, outbuf_table_ent_t, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq);
+HT_GENERATE2(outbuf_table_s, outbuf_table_ent_t, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq, 0.6, tor_reallocarray, tor_free_);
 
 /*****************************************************************************
  * Other internal data
@@ -104,7 +109,7 @@ static unsigned int kist_lite_mode = 0;
  * changed and it doesn't recognized the values passed to the syscalls needed
  * by KIST. In that case, fallback to the naive approach. */
 static unsigned int kist_no_kernel_support = 0;
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 static unsigned int kist_lite_mode = 1;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -298,7 +303,7 @@ update_socket_info_impl, (socket_table_ent_t *ent))
   }
   return;
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
   goto fallback;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -458,6 +463,13 @@ MOCK_IMPL(void, channel_write_to_kernel, (channel_t *chan))
   log_debug(LD_SCHED, "Writing %lu bytes to kernel for chan %" PRIu64,
             (unsigned long)channel_outbuf_length(chan),
             chan->global_identifier);
+  /* Note that 'connection_handle_write()' may change the scheduler state of
+   * the channel during the scheduling loop with
+   * 'connection_or_flushed_some()' -> 'scheduler_channel_wants_writes()'.
+   * This side-effect will only occur if the channel is currently in the
+   * 'SCHED_CHAN_WAITING_TO_WRITE' or 'SCHED_CHAN_IDLE' states, which KIST
+   * rarely uses, so it should be fine unless KIST begins using these states
+   * in the future. */
   connection_handle_write(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), 0);
 }
 
@@ -724,7 +736,7 @@ kist_scheduler_run(void)
     SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
       scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
       if (!smartlist_contains(cp, readd_chan)) {
-        if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+        if (!SCHED_BUG(readd_chan->sched_heap_idx != -1, readd_chan)) {
           /* XXXX Note that the check above is in theory redundant with
            * the smartlist_contains check.  But let's make sure we're
            * not messing anything up, and leave them both for now. */
@@ -833,7 +845,7 @@ scheduler_can_use_kist(void)
   return run_interval > 0;
 }
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 
 int
 scheduler_can_use_kist(void)
diff --git a/src/core/or/scheduler_vanilla.c b/src/core/or/scheduler_vanilla.c
index 33536ae04b..d862ff8710 100644
--- a/src/core/or/scheduler_vanilla.c
+++ b/src/core/or/scheduler_vanilla.c
@@ -1,11 +1,16 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file scheduler_vanilla.c
+ * @brief "Vanilla" (pre-KIST) cell scheduler code.
+ **/
+
 #include "core/or/or.h"
 #include "app/config/config.h"
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/channel.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #include "core/or/scheduler.h"
 
 /*****************************************************************************
@@ -172,4 +177,3 @@ get_vanilla_scheduler(void)
 {
   return &vanilla_scheduler;
 }
-
diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c
new file mode 100644
index 0000000000..788f56088c
--- /dev/null
+++ b/src/core/or/sendme.c
@@ -0,0 +1,710 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.c
+ * \brief Code that is related to SENDME cells both in terms of
+ *        creating/parsing cells and handling the content.
+ */
+
+#define SENDME_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/mainloop/connection.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+#include "feature/nodelist/networkstatus.h"
+#include "lib/ctime/di_ops.h"
+#include "trunnel/sendme_cell.h"
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * used when emitting a SENDME cell. */
+STATIC int
+get_emit_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_emit_min_version",
+                                 SENDME_EMIT_MIN_VERSION_DEFAULT,
+                                 SENDME_EMIT_MIN_VERSION_MIN,
+                                 SENDME_EMIT_MIN_VERSION_MAX);
+}
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * accepted when receiving a SENDME cell. */
+STATIC int
+get_accept_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_accept_min_version",
+                                 SENDME_ACCEPT_MIN_VERSION_DEFAULT,
+                                 SENDME_ACCEPT_MIN_VERSION_MIN,
+                                 SENDME_ACCEPT_MIN_VERSION_MAX);
+}
+
+/* Pop the first cell digset on the given circuit from the SENDME last digests
+ * list. NULL is returned if the list is uninitialized or empty.
+ *
+ * The caller gets ownership of the returned digest thus is responsible for
+ * freeing the memory. */
+static uint8_t *
+pop_first_cell_digest(const circuit_t *circ)
+{
+  uint8_t *circ_digest;
+
+  tor_assert(circ);
+
+  if (circ->sendme_last_digests == NULL ||
+      smartlist_len(circ->sendme_last_digests) == 0) {
+    return NULL;
+  }
+
+  /* More cell digest than the SENDME window is never suppose to happen. The
+   * cell should have been rejected before reaching this point due to its
+   * package_window down to 0 leading to a circuit close. Scream loudly but
+   * still pop the element so we don't memory leak. */
+  tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <=
+                      CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT);
+
+  circ_digest = smartlist_get(circ->sendme_last_digests, 0);
+  smartlist_del_keeporder(circ->sendme_last_digests, 0);
+  return circ_digest;
+}
+
+/* Return true iff the given cell digest matches the first digest in the
+ * circuit sendme list. */
+static bool
+v1_digest_matches(const uint8_t *circ_digest, const uint8_t *cell_digest)
+{
+  tor_assert(circ_digest);
+  tor_assert(cell_digest);
+
+  /* Compare the digest with the one in the SENDME. This cell is invalid
+   * without a perfect match. */
+  if (tor_memneq(circ_digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "SENDME v1 cell digest do not match.");
+    return false;
+  }
+
+  /* Digests matches! */
+  return true;
+}
+
+/* Return true iff the given decoded SENDME version 1 cell is valid and
+ * matches the expected digest on the circuit.
+ *
+ * Validation is done by comparing the digest in the cell from the previous
+ * cell we saw which tells us that the other side has in fact seen that cell.
+ * See proposal 289 for more details. */
+static bool
+cell_v1_is_valid(const sendme_cell_t *cell, const uint8_t *circ_digest)
+{
+  tor_assert(cell);
+  tor_assert(circ_digest);
+
+  const uint8_t *cell_digest = sendme_cell_getconstarray_data_v1_digest(cell);
+  return v1_digest_matches(circ_digest, cell_digest);
+}
+
+/* Return true iff the given cell version can be handled or if the minimum
+ * accepted version from the consensus is known to us. */
+STATIC bool
+cell_version_can_be_handled(uint8_t cell_version)
+{
+  int accept_version = get_accept_min_version();
+
+  /* We will first check if the consensus minimum accepted version can be
+   * handled by us and if not, regardless of the cell version we got, we can't
+   * continue. */
+  if (accept_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Unable to accept SENDME version %u (from consensus). "
+           "We only support <= %u. Probably your tor is too old?",
+           accept_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  /* Then, is this version below the accepted version from the consensus? If
+   * yes, we must not handle it. */
+  if (cell_version < accept_version) {
+    log_info(LD_PROTOCOL, "Unacceptable SENDME version %u. Only "
+                          "accepting %u (from consensus). Closing circuit.",
+             cell_version, accept_version);
+    goto invalid;
+  }
+
+  /* Is this cell version supported by us? */
+  if (cell_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_info(LD_PROTOCOL, "SENDME cell version %u is not supported by us. "
+                          "We only support <= %u",
+             cell_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  return true;
+ invalid:
+  return false;
+}
+
+/* Return true iff the encoded SENDME cell in cell_payload of length
+ * cell_payload_len is valid. For each version:
+ *
+ *  0: No validation
+ *  1: Authenticated with last cell digest.
+ *
+ * This is the main critical function to make sure we can continue to
+ * send/recv cells on a circuit. If the SENDME is invalid, the circuit should
+ * be marked for close by the caller. */
+STATIC bool
+sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload,
+                size_t cell_payload_len)
+{
+  uint8_t cell_version;
+  uint8_t *circ_digest = NULL;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* An empty payload means version 0 so skip trunnel parsing. We won't be
+   * able to parse a 0 length buffer into a valid SENDME cell. */
+  if (cell_payload_len == 0) {
+    cell_version = 0;
+  } else {
+    /* First we'll decode the cell so we can get the version. */
+    if (sendme_cell_parse(&cell, cell_payload, cell_payload_len) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unparseable SENDME cell received. Closing circuit.");
+      goto invalid;
+    }
+    cell_version = sendme_cell_get_version(cell);
+  }
+
+  /* Validate that we can handle this cell version. */
+  if (!cell_version_can_be_handled(cell_version)) {
+    goto invalid;
+  }
+
+  /* Pop the first element that was added (FIFO). We do that regardless of the
+   * version so we don't accumulate on the circuit if v0 is used by the other
+   * end point. */
+  circ_digest = pop_first_cell_digest(circ);
+  if (circ_digest == NULL) {
+    /* We shouldn't have received a SENDME if we have no digests. Log at
+     * protocol warning because it can be tricked by sending many SENDMEs
+     * without prior data cell. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "We received a SENDME but we have no cell digests to match. "
+           "Closing circuit.");
+    goto invalid;
+  }
+
+  /* Validate depending on the version now. */
+  switch (cell_version) {
+  case 0x01:
+    if (!cell_v1_is_valid(cell, circ_digest)) {
+      goto invalid;
+    }
+    break;
+  case 0x00:
+    /* Version 0, there is no work to be done on the payload so it is
+     * necessarily valid if we pass the version validation. */
+    break;
+  default:
+    log_warn(LD_PROTOCOL, "Unknown SENDME cell version %d received.",
+             cell_version);
+    tor_assert_nonfatal_unreached();
+    break;
+  }
+
+  /* Valid cell. */
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return true;
+ invalid:
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return false;
+}
+
+/* Build and encode a version 1 SENDME cell into payload, which must be at
+ * least of RELAY_PAYLOAD_SIZE bytes, using the digest for the cell data.
+ *
+ * Return the size in bytes of the encoded cell in payload. A negative value
+ * is returned on encoding failure. */
+STATIC ssize_t
+build_cell_payload_v1(const uint8_t *cell_digest, uint8_t *payload)
+{
+  ssize_t len = -1;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(cell_digest);
+  tor_assert(payload);
+
+  cell = sendme_cell_new();
+
+  /* Building a payload for version 1. */
+  sendme_cell_set_version(cell, 0x01);
+  /* Set the data length field for v1. */
+  sendme_cell_set_data_len(cell, TRUNNEL_SENDME_V1_DIGEST_LEN);
+
+  /* Copy the digest into the data payload. */
+  memcpy(sendme_cell_getarray_data_v1_digest(cell), cell_digest,
+         sendme_cell_get_data_len(cell));
+
+  /* Finally, encode the cell into the payload. */
+  len = sendme_cell_encode(payload, RELAY_PAYLOAD_SIZE, cell);
+
+  sendme_cell_free(cell);
+  return len;
+}
+
+/* Send a circuit-level SENDME on the given circuit using the layer_hint if
+ * not NULL. The digest is only used for version 1.
+ *
+ * Return 0 on success else a negative value and the circuit will be closed
+ * because we failed to send the cell on it. */
+static int
+send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint,
+                          const uint8_t *cell_digest)
+{
+  uint8_t emit_version;
+  uint8_t payload[RELAY_PAYLOAD_SIZE];
+  ssize_t payload_len;
+
+  tor_assert(circ);
+  tor_assert(cell_digest);
+
+  emit_version = get_emit_min_version();
+  switch (emit_version) {
+  case 0x01:
+    payload_len = build_cell_payload_v1(cell_digest, payload);
+    if (BUG(payload_len < 0)) {
+      /* Unable to encode the cell, abort. We can recover from this by closing
+       * the circuit but in theory it should never happen. */
+      return -1;
+    }
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 1 cell.");
+    break;
+  case 0x00:
+    FALLTHROUGH;
+  default:
+    /* Unknown version, fallback to version 0 meaning no payload. */
+    payload_len = 0;
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 0 cell. "
+                           "Consensus emit version is %d", emit_version);
+    break;
+  }
+
+  if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
+                                   (char *) payload, payload_len,
+                                   layer_hint) < 0) {
+    log_warn(LD_CIRC,
+             "SENDME relay_send_command_from_edge failed. Circuit's closed.");
+    return -1; /* the circuit's closed, don't continue */
+  }
+  return 0;
+}
+
+/* Record the cell digest only if the next cell is expected to be a SENDME. */
+static void
+record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest)
+{
+  tor_assert(circ);
+  tor_assert(sendme_digest);
+
+  /* Add the digest to the last seen list in the circuit. */
+  if (circ->sendme_last_digests == NULL) {
+    circ->sendme_last_digests = smartlist_new();
+  }
+  smartlist_add(circ->sendme_last_digests,
+                tor_memdup(sendme_digest, DIGEST_LEN));
+}
+
+/*
+ * Public API
+ */
+
+/** Return true iff the next cell for the given cell window is expected to be
+ * a SENDME.
+ *
+ * We are able to know that because the package or deliver window value minus
+ * one cell (the possible SENDME cell) should be a multiple of the increment
+ * window value. */
+static bool
+circuit_sendme_cell_is_next(int window)
+{
+  /* At the start of the window, no SENDME will be expected. */
+  if (window == CIRCWINDOW_START) {
+    return false;
+  }
+
+  /* Are we at the limit of the increment and if not, we don't expect next
+   * cell is a SENDME.
+   *
+   * We test against the window minus 1 because when we are looking if the
+   * next cell is a SENDME, the window (either package or deliver) hasn't been
+   * decremented just yet so when this is called, we are currently processing
+   * the "window - 1" cell.
+   *
+   * This function is used when recording a cell digest and this is done quite
+   * low in the stack when decrypting or encrypting a cell. The window is only
+   * updated once the cell is actually put in the outbuf. */
+  if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) {
+    return false;
+  }
+
+  /* Next cell is expected to be a SENDME. */
+  return true;
+}
+
+/** Called when we've just received a relay data cell, when we've just
+ * finished flushing all bytes to stream <b>conn</b>, or when we've flushed
+ * *some* bytes to the stream <b>conn</b>.
+ *
+ * If conn->outbuf is not too full, and our deliver window is low, send back a
+ * suitable number of stream-level sendme cells.
+ */
+void
+sendme_connection_edge_consider_sending(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  int log_domain = TO_CONN(conn)->type == CONN_TYPE_AP ? LD_APP : LD_EXIT;
+
+  /* Don't send it if we still have data to deliver. */
+  if (connection_outbuf_too_full(TO_CONN(conn))) {
+    goto end;
+  }
+
+  if (circuit_get_by_edge_conn(conn) == NULL) {
+    /* This can legitimately happen if the destroy has already arrived and
+     * torn down the circuit. */
+    log_info(log_domain, "No circuit associated with edge connection. "
+                         "Skipping sending SENDME.");
+    goto end;
+  }
+
+  while (conn->deliver_window <=
+         (STREAMWINDOW_START - STREAMWINDOW_INCREMENT)) {
+    log_debug(log_domain, "Outbuf %" TOR_PRIuSZ ", queuing stream SENDME.",
+              TO_CONN(conn)->outbuf_flushlen);
+    conn->deliver_window += STREAMWINDOW_INCREMENT;
+    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
+                                     NULL, 0) < 0) {
+      log_warn(LD_BUG, "connection_edge_send_command failed while sending "
+                       "a SENDME. Circuit probably closed, skipping.");
+      goto end; /* The circuit's closed, don't continue */
+    }
+  }
+
+ end:
+  return;
+}
+
+/** Check if the deliver_window for circuit <b>circ</b> (at hop
+ * <b>layer_hint</b> if it's defined) is low enough that we should
+ * send a circuit-level sendme back down the circuit. If so, send
+ * enough sendmes that the window would be overfull if we sent any
+ * more.
+ */
+void
+sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  bool sent_one_sendme = false;
+  const uint8_t *digest;
+
+  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
+          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
+    log_debug(LD_CIRC,"Queuing circuit sendme.");
+    if (layer_hint) {
+      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = cpath_get_sendme_digest(layer_hint);
+    } else {
+      circ->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+    }
+    if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) {
+      return; /* The circuit's closed, don't continue */
+    }
+    /* Current implementation is not suppose to send multiple SENDME at once
+     * because this means we would use the same relay crypto digest for each
+     * SENDME leading to a mismatch on the other side and the circuit to
+     * collapse. Scream loudly if it ever happens so we can address it. */
+    tor_assert_nonfatal(!sent_one_sendme);
+    sent_one_sendme = true;
+  }
+}
+
+/* Process a circuit-level SENDME cell that we just received. The layer_hint,
+ * if not NULL, is the Exit hop of the connection which means that we are a
+ * client. In that case, circ must be an origin circuit. The cell_body_len is
+ * the length of the SENDME cell payload (excluding the header). The
+ * cell_payload is the payload.
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_circuit_level(crypt_path_t *layer_hint,
+                             circuit_t *circ, const uint8_t *cell_payload,
+                             uint16_t cell_payload_len)
+{
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* Validate the SENDME cell. Depending on the version, different validation
+   * can be done. An invalid SENDME requires us to close the circuit. */
+  if (!sendme_is_valid(circ, cell_payload, cell_payload_len)) {
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  /* If we are the origin of the circuit, we are the Client so we use the
+   * layer hint (the Exit hop) for the package window tracking. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* If we are the origin of the circuit, it is impossible to not have a
+     * cpath. Just in case, bug on it and close the circuit. */
+    if (BUG(layer_hint == NULL)) {
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    if ((layer_hint->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from exit relay. "
+                     "Closing circ.");
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    layer_hint->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.",
+              layer_hint->package_window);
+
+    /* We count circuit-level sendme's as valid delivered data because they
+     * are rate limited. */
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len);
+  } else {
+    /* We aren't the origin of this circuit so we are the Exit and thus we
+     * track the package window with the circuit object. */
+    if ((circ->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&client_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from client. "
+                     "Closing circ (window %d).", circ->package_window);
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    circ->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_EXIT, "circ-level sendme at non-origin, packagewindow %d.",
+              circ->package_window);
+  }
+
+  return 0;
+}
+
+/* Process a stream-level SENDME cell that we just received. The conn is the
+ * edge connection (stream) that the circuit circ is associated with. The
+ * cell_body_len is the length of the payload (excluding the header).
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                            uint16_t cell_body_len)
+{
+  tor_assert(conn);
+  tor_assert(circ);
+
+  /* Don't allow the other endpoint to request more than our maximum (i.e.
+   * initial) stream SENDME window worth of data. Well-behaved stock clients
+   * will not request more than this max (as per the check in the while loop
+   * of sendme_connection_edge_consider_sending()). */
+  if ((conn->package_window + STREAMWINDOW_INCREMENT) >
+      STREAMWINDOW_START_MAX) {
+    static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
+    log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                   "Unexpected stream sendme cell. Closing circ (window %d).",
+                   conn->package_window);
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+  /* At this point, the stream sendme is valid */
+  conn->package_window += STREAMWINDOW_INCREMENT;
+
+  /* We count circuit-level sendme's as valid delivered data because they are
+   * rate limited. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_body_len);
+  }
+
+  log_debug(CIRCUIT_IS_ORIGIN(circ) ? LD_APP : LD_EXIT,
+            "stream-level sendme, package_window now %d.",
+            conn->package_window);
+  return 0;
+}
+
+/* Called when a relay DATA cell is received on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the deliver window and return its new value. */
+int
+sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int deliver_window, domain;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    tor_assert(layer_hint);
+    --layer_hint->deliver_window;
+    deliver_window = layer_hint->deliver_window;
+    domain = LD_APP;
+  } else {
+    tor_assert(!layer_hint);
+    --circ->deliver_window;
+    deliver_window = circ->deliver_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit deliver_window now %d.", deliver_window);
+  return deliver_window;
+}
+
+/* Called when a relay DATA cell is received for the given edge connection
+ * conn. Update the deliver window and return its new value. */
+int
+sendme_stream_data_received(edge_connection_t *conn)
+{
+  tor_assert(conn);
+  return --conn->deliver_window;
+}
+
+/* Called when a relay DATA cell is packaged on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the package window and return its new value. */
+int
+sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int package_window, domain;
+
+  tor_assert(circ);
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* Client side. */
+    tor_assert(layer_hint);
+    --layer_hint->package_window;
+    package_window = layer_hint->package_window;
+    domain = LD_APP;
+  } else {
+    /* Exit side. */
+    tor_assert(!layer_hint);
+    --circ->package_window;
+    package_window = circ->package_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit package_window now %d.", package_window);
+  return package_window;
+}
+
+/* Called when a relay DATA cell is packaged for the given edge connection
+ * conn. Update the package window and return its new value. */
+int
+sendme_note_stream_data_packaged(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  --conn->package_window;
+  log_debug(LD_APP, "Stream package_window now %d.", conn->package_window);
+  return conn->package_window;
+}
+
+/* Record the cell digest into the circuit sendme digest list depending on
+ * which edge we are. The digest is recorded only if we expect the next cell
+ * that we will receive is a SENDME so we can match the digest. */
+void
+sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath)
+{
+  int package_window;
+  uint8_t *sendme_digest;
+
+  tor_assert(circ);
+
+  package_window = circ->package_window;
+  if (cpath) {
+    package_window = cpath->package_window;
+  }
+
+  /* Is this the last cell before a SENDME? The idea is that if the
+   * package_window reaches a multiple of the increment, after this cell, we
+   * should expect a SENDME. */
+  if (!circuit_sendme_cell_is_next(package_window)) {
+    return;
+  }
+
+  /* Getting the digest is expensive so we only do it once we are certain to
+   * record it on the circuit. */
+  if (cpath) {
+    sendme_digest = cpath_get_sendme_digest(cpath);
+  } else {
+    sendme_digest =
+      relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+  }
+
+  record_cell_digest_on_circ(circ, sendme_digest);
+}
+
+/* Called once we decrypted a cell and recognized it. Record the cell digest
+ * as the next sendme digest only if the next cell we'll send on the circuit
+ * is expected to be a SENDME. */
+void
+sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window :
+                                           circ->deliver_window)) {
+    return;
+  }
+
+  if (cpath) {
+    /* Record incoming digest. */
+    cpath_sendme_record_cell_digest(cpath, false);
+  } else {
+    /* Record foward digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, true);
+  }
+}
+
+/* Called once we encrypted a cell. Record the cell digest as the next sendme
+ * digest only if the next cell we expect to receive is a SENDME so we can
+ * match the digests. */
+void
+sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window :
+                                           circ->package_window)) {
+    goto end;
+  }
+
+  if (cpath) {
+    /* Record the forward digest. */
+    cpath_sendme_record_cell_digest(cpath, true);
+  } else {
+    /* Record the incoming digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, false);
+  }
+
+ end:
+  return;
+}
diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h
new file mode 100644
index 0000000000..05d37ec3bb
--- /dev/null
+++ b/src/core/or/sendme.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.h
+ * \brief Header file for sendme.c.
+ **/
+
+#ifndef TOR_SENDME_H
+#define TOR_SENDME_H
+
+#include "core/or/edge_connection_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+
+/* Sending SENDME cell. */
+void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn);
+void sendme_circuit_consider_sending(circuit_t *circ,
+                                     crypt_path_t *layer_hint);
+
+/* Processing SENDME cell. */
+int sendme_process_circuit_level(crypt_path_t *layer_hint,
+                                 circuit_t *circ, const uint8_t *cell_payload,
+                                 uint16_t cell_payload_len);
+int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                                uint16_t cell_body_len);
+
+/* Update deliver window functions. */
+int sendme_stream_data_received(edge_connection_t *conn);
+int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint);
+
+/* Update package window functions. */
+int sendme_note_circuit_data_packaged(circuit_t *circ,
+                                      crypt_path_t *layer_hint);
+int sendme_note_stream_data_packaged(edge_connection_t *conn);
+
+/* Record cell digest on circuit. */
+void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath);
+/* Record cell digest as the SENDME digest. */
+void sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+void sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+
+/* Private section starts. */
+#ifdef SENDME_PRIVATE
+
+/* The maximum supported version. Above that value, the cell can't be
+ * recognized as a valid SENDME. */
+#define SENDME_MAX_SUPPORTED_VERSION 1
+
+/* The cell version constants for when emitting a cell. */
+#define SENDME_EMIT_MIN_VERSION_DEFAULT 1
+#define SENDME_EMIT_MIN_VERSION_MIN 0
+#define SENDME_EMIT_MIN_VERSION_MAX UINT8_MAX
+
+/* The cell version constants for when accepting a cell. */
+#define SENDME_ACCEPT_MIN_VERSION_DEFAULT 0
+#define SENDME_ACCEPT_MIN_VERSION_MIN 0
+#define SENDME_ACCEPT_MIN_VERSION_MAX UINT8_MAX
+
+/*
+ * Unit tests declaractions.
+ */
+#ifdef TOR_UNIT_TESTS
+
+STATIC int get_emit_min_version(void);
+STATIC int get_accept_min_version(void);
+
+STATIC bool cell_version_can_be_handled(uint8_t cell_version);
+
+STATIC ssize_t build_cell_payload_v1(const uint8_t *cell_digest,
+                                     uint8_t *payload);
+STATIC bool sendme_is_valid(const circuit_t *circ,
+                            const uint8_t *cell_payload,
+                            size_t cell_payload_len);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(SENDME_PRIVATE) */
+
+#endif /* !defined(TOR_SENDME_H) */
diff --git a/src/core/or/server_port_cfg_st.h b/src/core/or/server_port_cfg_st.h
index bd026af7ee..9a005eccdf 100644
--- a/src/core/or/server_port_cfg_st.h
+++ b/src/core/or/server_port_cfg_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file server_port_cfg_st.h
+ * @brief Cnfiguration structure for server ports.
+ **/
+
 #ifndef SERVER_PORT_CFG_ST_H
 #define SERVER_PORT_CFG_ST_H
 
@@ -16,5 +21,4 @@ struct server_port_cfg_t {
   unsigned int bind_ipv6_only : 1;
 };
 
-#endif
-
+#endif /* !defined(SERVER_PORT_CFG_ST_H) */
diff --git a/src/core/or/socks_request_st.h b/src/core/or/socks_request_st.h
index 5922870c61..4bcdb48b92 100644
--- a/src/core/or/socks_request_st.h
+++ b/src/core/or/socks_request_st.h
@@ -1,12 +1,19 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file socks_request_st.h
+ * @brief Client request structure.
+ **/
+
 #ifndef SOCKS_REQUEST_ST_H
 #define SOCKS_REQUEST_ST_H
 
+#include "lib/net/socks5_status.h"
+
 #define MAX_SOCKS_REPLY_LEN 1024
 
 #define SOCKS_NO_AUTH 0x00
@@ -58,6 +65,11 @@ struct socks_request_t {
    * "username/password" authentication if both are offered. Used as input to
    * parse_socks. */
   unsigned int socks_prefer_no_auth : 1;
+  /** If set, we can send back the extended error code in the reply. */
+  unsigned int socks_use_extended_errors : 1;
+  /** If non zero, this contains the extended error code that should be used
+   * if the port was configured to use them. */
+  socks5_reply_status_t socks_extended_error_code;
 
   /** Number of bytes in username; 0 if username is NULL */
   size_t usernamelen;
@@ -74,4 +86,4 @@ struct socks_request_t {
   uint8_t socks5_atyp; /* SOCKS5 address type */
 };
 
-#endif
+#endif /* !defined(SOCKS_REQUEST_ST_H) */
diff --git a/src/core/or/status.c b/src/core/or/status.c
index 46494ca76c..ed8448883c 100644
--- a/src/core/or/status.c
+++ b/src/core/or/status.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,7 @@
 #include "core/or/or.h"
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
+#include "feature/dirclient/dirclient.h"
 #include "core/or/status.h"
 #include "feature/nodelist/nodelist.h"
 #include "core/or/relay.h"
@@ -146,6 +147,8 @@ log_heartbeat(time_t now)
          uptime, count_circuits(), bw_sent, bw_rcvd,
          hibernating?" We are currently hibernating.":"");
 
+  dirclient_dump_total_dls();
+
   if (server_mode(options) && accounting_is_enabled(options) && !hibernating) {
     log_accounting(now, options);
   }
diff --git a/src/core/or/status.h b/src/core/or/status.h
index 3467501ebb..639f8cdf51 100644
--- a/src/core/or/status.h
+++ b/src/core/or/status.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file status.h
+ * @brief Header for status.c
+ **/
+
 #ifndef TOR_STATUS_H
 #define TOR_STATUS_H
 
@@ -15,4 +20,3 @@ STATIC char *bytes_to_usage(uint64_t bytes);
 #endif
 
 #endif /* !defined(TOR_STATUS_H) */
-
diff --git a/src/core/or/tor_version_st.h b/src/core/or/tor_version_st.h
index 716429bd32..46644c5eb8 100644
--- a/src/core/or/tor_version_st.h
+++ b/src/core/or/tor_version_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tor_version_st.h
+ * @brief Parsed Tor version structure.
+ **/
+
 #ifndef TOR_VERSION_ST_H
 #define TOR_VERSION_ST_H
 
@@ -28,5 +33,4 @@ struct tor_version_t {
   char git_tag[DIGEST_LEN];
 };
 
-#endif
-
+#endif /* !defined(TOR_VERSION_ST_H) */
diff --git a/src/core/or/var_cell_st.h b/src/core/or/var_cell_st.h
index 4287c83f6d..caf64c63a5 100644
--- a/src/core/or/var_cell_st.h
+++ b/src/core/or/var_cell_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file var_cell_st.h
+ * @brief Variable-length cell structure.
+ **/
+
 #ifndef VAR_CELL_ST_H
 #define VAR_CELL_ST_H
 
@@ -19,5 +24,4 @@ struct var_cell_t {
   uint8_t payload[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
-
+#endif /* !defined(VAR_CELL_ST_H) */
diff --git a/src/core/or/versions.c b/src/core/or/versions.c
index 33273a5294..31f1f5b997 100644
--- a/src/core/or/versions.c
+++ b/src/core/or/versions.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,25 @@
 
 #include "core/or/tor_version_st.h"
 
+/**
+ * Return the approximate date when this release came out, or was
+ * scheduled to come out, according to the APPROX_RELEASE_DATE set in
+ * configure.ac
+ **/
+time_t
+tor_get_approx_release_date(void)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  tor_snprintf(tbuf, sizeof(tbuf),
+               "%s 00:00:00", APPROX_RELEASE_DATE);
+  time_t result = 0;
+  int r = parse_iso_time(tbuf, &result);
+  if (BUG(r < 0)) {
+    result = 0;
+  }
+  return result;
+}
+
 /** Return VS_RECOMMENDED if <b>myversion</b> is contained in
  * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
  * entries. Else, return VS_OLD if every member of
@@ -277,7 +296,7 @@ tor_version_parse(const char *s, tor_version_t *out)
       return -1;
     hexlen = (int)(close_paren-cp);
     memwipe(digest, 0, sizeof(digest));
-    if ( hexlen == 0 || (hexlen % 2) == 1)
+    if (hexlen == 0 || (hexlen % 2) == 1)
       return -1;
     if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2)
       return -1;
@@ -377,6 +396,69 @@ sort_version_list(smartlist_t *versions, int remove_duplicates)
     smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
 }
 
+/** If there are more than this many entries, we're probably under
+ * some kind of weird DoS. */
+static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024;
+
+/**
+ * Map from protover string to protover_summary_flags_t.
+ */
+static strmap_t *protover_summary_map = NULL;
+
+/**
+ * Helper.  Given a non-NULL protover string <b>protocols</b>, set <b>out</b>
+ * to its summary, and memoize the result in <b>protover_summary_map</b>.
+ */
+static void
+memoize_protover_summary(protover_summary_flags_t *out,
+                         const char *protocols)
+{
+  if (!protover_summary_map)
+    protover_summary_map = strmap_new();
+
+  if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) {
+    protover_summary_cache_free_all();
+    tor_assert(protover_summary_map == NULL);
+    protover_summary_map = strmap_new();
+  }
+
+  const protover_summary_flags_t *cached =
+    strmap_get(protover_summary_map, protocols);
+
+  if (cached != NULL) {
+    /* We found a cached entry; no need to parse this one. */
+    memcpy(out, cached, sizeof(protover_summary_flags_t));
+    tor_assert(out->protocols_known);
+    return;
+  }
+
+  memset(out, 0, sizeof(*out));
+  out->protocols_known = 1;
+  out->supports_extend2_cells =
+    protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+  out->supports_ed25519_link_handshake_compat =
+    protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_link_handshake_any =
+    protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_hs_intro =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+  out->supports_v3_hsdir =
+    protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                    PROTOVER_HSDIR_V3);
+  out->supports_v3_rendezvous_point =
+    protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                    PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  out->supports_hs_setup_padding =
+    protocol_list_supports_protocol(protocols, PRT_PADDING,
+                                    PROTOVER_HS_SETUP_PADDING);
+  out->supports_establish_intro_dos_extension =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 5);
+
+  protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out));
+  cached = strmap_set(protover_summary_map, protocols, new_cached);
+  tor_assert(!cached);
+}
+
 /** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
  * falling back or correcting them based on <b>version</b> as appropriate.
  */
@@ -388,21 +470,7 @@ summarize_protover_flags(protover_summary_flags_t *out,
   tor_assert(out);
   memset(out, 0, sizeof(*out));
   if (protocols) {
-    out->protocols_known = 1;
-    out->supports_extend2_cells =
-      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
-    out->supports_ed25519_link_handshake_compat =
-      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_link_handshake_any =
-      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_hs_intro =
-      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
-    out->supports_v3_hsdir =
-      protocol_list_supports_protocol(protocols, PRT_HSDIR,
-                                      PROTOVER_HSDIR_V3);
-    out->supports_v3_rendezvous_point =
-      protocol_list_supports_protocol(protocols, PRT_HSREND,
-                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+    memoize_protover_summary(out, protocols);
   }
   if (version && !strcmpstart(version, "Tor ")) {
     if (!out->protocols_known) {
@@ -420,3 +488,13 @@ summarize_protover_flags(protover_summary_flags_t *out,
     }
   }
 }
+
+/**
+ * Free all space held in the protover_summary_map.
+ */
+void
+protover_summary_cache_free_all(void)
+{
+  strmap_free(protover_summary_map, tor_free_);
+  protover_summary_map = NULL;
+}
diff --git a/src/core/or/versions.h b/src/core/or/versions.h
index 22f3be176f..75dc17f9c7 100644
--- a/src/core/or/versions.h
+++ b/src/core/or/versions.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,6 +26,8 @@ typedef enum version_status_t {
   VS_UNKNOWN, /**< We have no idea. */
 } version_status_t;
 
+time_t tor_get_approx_release_date(void);
+
 version_status_t tor_version_is_obsolete(const char *myversion,
                                          const char *versionlist);
 int tor_version_parse_platform(const char *platform,
@@ -41,4 +43,6 @@ void summarize_protover_flags(protover_summary_flags_t *out,
                               const char *protocols,
                               const char *version);
 
+void protover_summary_cache_free_all(void);
+
 #endif /* !defined(TOR_VERSIONS_H) */
diff --git a/src/core/proto/.may_include b/src/core/proto/.may_include
new file mode 100644
index 0000000000..a66c3f83a6
--- /dev/null
+++ b/src/core/proto/.may_include
@@ -0,0 +1,14 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/buf/*.h
+lib/malloc/*.h
+lib/string/*.h
+
+lib/net/address.h
+
+trunnel/*.h
+
+core/proto/*.h
diff --git a/src/core/proto/core_proto.md b/src/core/proto/core_proto.md
new file mode 100644
index 0000000000..ad43bc5846
--- /dev/null
+++ b/src/core/proto/core_proto.md
@@ -0,0 +1,6 @@
+@dir /core/proto
+@brief core/proto: Protocol encoding/decoding
+
+These functions should (but do not always) exist at a lower level than most
+of the rest of core.
+
diff --git a/src/core/proto/include.am b/src/core/proto/include.am
new file mode 100644
index 0000000000..726ef924cf
--- /dev/null
+++ b/src/core/proto/include.am
@@ -0,0 +1,18 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/proto/proto_cell.c		\
+	src/core/proto/proto_control0.c		\
+	src/core/proto/proto_ext_or.c		\
+	src/core/proto/proto_haproxy.c		\
+	src/core/proto/proto_http.c		\
+	src/core/proto/proto_socks.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/proto/proto_cell.h			\
+	src/core/proto/proto_control0.h			\
+	src/core/proto/proto_ext_or.h			\
+	src/core/proto/proto_haproxy.h			\
+	src/core/proto/proto_http.h			\
+	src/core/proto/proto_socks.h
diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c
index 0442e2c6ee..5c1a2c24d7 100644
--- a/src/core/proto/proto_cell.c
+++ b/src/core/proto/proto_cell.c
@@ -1,11 +1,18 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_cell.c
+ * @brief Decodes Tor cells from buffers.
+ **/
+/* Right now it only handles variable-length cells, but eventually
+ * we should refactor other cell-reading code into here. */
+
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_cell.h"
 
 #include "core/or/connection_or.h"
@@ -83,4 +90,3 @@ fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
   *out = result;
   return 1;
 }
-
diff --git a/src/core/proto/proto_cell.h b/src/core/proto/proto_cell.h
index 4f3982ea43..0e9cfbfed3 100644
--- a/src/core/proto/proto_cell.h
+++ b/src/core/proto/proto_cell.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_cell.h
+ * @brief Header for proto_cell.c
+ **/
+
 #ifndef TOR_PROTO_CELL_H
 #define TOR_PROTO_CELL_H
 
@@ -14,4 +19,3 @@ int fetch_var_cell_from_buf(struct buf_t *buf, struct var_cell_t **out,
                             int linkproto);
 
 #endif /* !defined(TOR_PROTO_CELL_H) */
-
diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c
index 21fa328f02..323b37c539 100644
--- a/src/core/proto/proto_control0.c
+++ b/src/core/proto/proto_control0.c
@@ -1,11 +1,16 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_control0.c
+ * @brief Code to detect the obsolete v0 control protocol.
+ **/
+
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_control0.h"
 
 /** Return 1 iff buf looks more like it has an (obsolete) v0 controller
@@ -23,4 +28,3 @@ peek_buf_has_control0_command(buf_t *buf)
   }
   return 0;
 }
-
diff --git a/src/core/proto/proto_control0.h b/src/core/proto/proto_control0.h
index 162e513a1b..561bf00d19 100644
--- a/src/core/proto/proto_control0.h
+++ b/src/core/proto/proto_control0.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_control0.h
+ * @brief Header for proto_control0.c
+ **/
+
 #ifndef TOR_PROTO_CONTROL0_H
 #define TOR_PROTO_CONTROL0_H
 
@@ -11,4 +16,3 @@ struct buf_t;
 int peek_buf_has_control0_command(struct buf_t *buf);
 
 #endif /* !defined(TOR_PROTO_CONTROL0_H) */
-
diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c
index edbc51b10c..23fc2393b7 100644
--- a/src/core/proto/proto_ext_or.c
+++ b/src/core/proto/proto_ext_or.c
@@ -1,11 +1,16 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_ext_or.c
+ * @brief Parsing/encoding for the extended OR protocol.
+ **/
+
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_ext_or.h"
 
@@ -37,4 +42,3 @@ fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
   buf_get_bytes(buf, (*out)->body, len);
   return 1;
 }
-
diff --git a/src/core/proto/proto_ext_or.h b/src/core/proto/proto_ext_or.h
index b2bc64af85..3408599fb7 100644
--- a/src/core/proto/proto_ext_or.h
+++ b/src/core/proto/proto_ext_or.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_ext_or.h
+ * @brief Header for proto_ext_or.c
+ **/
+
 #ifndef TOR_PROTO_EXT_OR_H
 #define TOR_PROTO_EXT_OR_H
 
@@ -19,4 +24,11 @@ struct ext_or_cmd_t {
 int fetch_ext_or_command_from_buf(struct buf_t *buf,
                                   struct ext_or_cmd_t **out);
 
+ext_or_cmd_t *ext_or_cmd_new(uint16_t len);
+
+#define ext_or_cmd_free(cmd)                            \
+  FREE_AND_NULL(ext_or_cmd_t, ext_or_cmd_free_, (cmd))
+
+void ext_or_cmd_free_(ext_or_cmd_t *cmd);
+
 #endif /* !defined(TOR_PROTO_EXT_OR_H) */
diff --git a/src/core/proto/proto_haproxy.c b/src/core/proto/proto_haproxy.c
new file mode 100644
index 0000000000..9129fc55bf
--- /dev/null
+++ b/src/core/proto/proto_haproxy.c
@@ -0,0 +1,45 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define PROTO_HAPROXY_PRIVATE
+#include "lib/malloc/malloc.h"
+#include "lib/net/address.h"
+#include "lib/string/printf.h"
+#include "core/proto/proto_haproxy.h"
+
+/** Return a newly allocated PROXY header null-terminated string. Returns NULL
+ * if addr_port->addr is incompatible with the proxy protocol.
+ */
+char *
+haproxy_format_proxy_header_line(const tor_addr_port_t *addr_port)
+{
+  tor_assert(addr_port);
+
+  sa_family_t family = tor_addr_family(&addr_port->addr);
+  const char *family_string = NULL;
+  const char *src_addr_string = NULL;
+
+  switch (family) {
+    case AF_INET:
+      family_string = "TCP4";
+      src_addr_string = "0.0.0.0";
+      break;
+    case AF_INET6:
+      family_string = "TCP6";
+      src_addr_string = "::";
+      break;
+    default:
+      /* Unknown family. */
+      return NULL;
+  }
+
+  char *buf;
+  char addrbuf[TOR_ADDR_BUF_LEN];
+
+  tor_addr_to_str(addrbuf, &addr_port->addr, sizeof(addrbuf), 0);
+
+  tor_asprintf(&buf, "PROXY %s %s %s 0 %d\r\n", family_string, src_addr_string,
+                                                addrbuf, addr_port->port);
+
+  return buf;
+}
diff --git a/src/core/proto/proto_haproxy.h b/src/core/proto/proto_haproxy.h
new file mode 100644
index 0000000000..63c164e1a1
--- /dev/null
+++ b/src/core/proto/proto_haproxy.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_HAPROXY_H
+#define TOR_PROTO_HAPROXY_H
+
+struct tor_addr_port_t;
+
+char *haproxy_format_proxy_header_line(
+                                    const struct tor_addr_port_t *addr_port);
+
+#endif /* !defined(TOR_PROTO_HAPROXY_H) */
diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c
index 5c86fc4979..ef4b897fcc 100644
--- a/src/core/proto/proto_http.c
+++ b/src/core/proto/proto_http.c
@@ -1,12 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_http.c
+ * @brief Parse a subset of the HTTP protocol.
+ **/
+
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 
 /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
@@ -168,4 +173,3 @@ buf_http_find_content_length(const char *headers, size_t headerlen,
 
   return ok ? 1 : -1;
 }
-
diff --git a/src/core/proto/proto_http.h b/src/core/proto/proto_http.h
index cd70050205..e0c5135346 100644
--- a/src/core/proto/proto_http.h
+++ b/src/core/proto/proto_http.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_http.h
+ * @brief Header for proto_http.c
+ **/
+
 #ifndef TOR_PROTO_HTTP_H
 #define TOR_PROTO_HTTP_H
 
@@ -21,4 +26,3 @@ STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
 #endif
 
 #endif /* !defined(TOR_PROTO_HTTP_H) */
-
diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c
index 5a7d7ac9be..bcb0c2b2f9 100644
--- a/src/core/proto/proto_socks.c
+++ b/src/core/proto/proto_socks.c
@@ -1,14 +1,19 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_socks.c
+ * @brief Implementations for SOCKS4 and SOCKS5 protocols.
+ **/
+
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/relay/ext_orport.h"
@@ -105,7 +110,7 @@ socks_request_free_(socks_request_t *req)
 /**
  * Parse a single SOCKS4 request from buffer <b>raw_data</b> of length
  * <b>datalen</b> and update relevant fields of <b>req</b>. If SOCKS4a
- * request is detected, set <b>*is_socks4a<b> to true. Set <b>*drain_out</b>
+ * request is detected, set <b>*is_socks4a</b> to true. Set <b>*drain_out</b>
  * to number of bytes we parsed so far.
  *
  * Return SOCKS_RESULT_DONE if parsing succeeded, SOCKS_RESULT_INVALID if
@@ -584,9 +589,8 @@ parse_socks5_client_request(const uint8_t *raw_data, socks_request_t *req,
       strlcpy(req->address, hostname, sizeof(req->address));
     } break;
     case 4: {
-      const char *ipv6 =
-        (const char *)socks5_client_request_getarray_dest_addr_ipv6(
-          trunnel_req);
+      const uint8_t *ipv6 =
+          socks5_client_request_getarray_dest_addr_ipv6(trunnel_req);
       tor_addr_from_ipv6_bytes(&destaddr, ipv6);
 
       tor_addr_to_str(req->address, &destaddr, sizeof(req->address), 1);
@@ -618,6 +622,7 @@ process_socks5_client_request(socks_request_t *req,
                               int safe_socks)
 {
   socks_result_t res = SOCKS_RESULT_DONE;
+  tor_addr_t tmpaddr;
 
   if (req->command != SOCKS_COMMAND_CONNECT &&
       req->command != SOCKS_COMMAND_RESOLVE &&
@@ -628,11 +633,10 @@ process_socks5_client_request(socks_request_t *req,
   }
 
   if (req->command == SOCKS_COMMAND_RESOLVE_PTR &&
-      !string_is_valid_ipv4_address(req->address) &&
-      !string_is_valid_ipv6_address(req->address)) {
+      tor_addr_parse(&tmpaddr, req->address) < 0) {
     socks_request_set_socks5_error(req, SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
     log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
-                     "hostname type. Rejecting.");
+                     "a malformed address. Rejecting.");
 
     res = SOCKS_RESULT_INVALID;
     goto end;
diff --git a/src/core/proto/proto_socks.h b/src/core/proto/proto_socks.h
index 2a387bf848..f3af0d988e 100644
--- a/src/core/proto/proto_socks.h
+++ b/src/core/proto/proto_socks.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_socks.h
+ * @brief Header for proto_socks.c
+ **/
+
 #ifndef TOR_PROTO_SOCKS_H
 #define TOR_PROTO_SOCKS_H
 
diff --git a/src/core/stA1RajU b/src/core/stA1RajU
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/core/stA1RajU
diff --git a/src/core/stiysZND b/src/core/stiysZND
new file mode 100644
index 0000000000..faa365b769
--- /dev/null
+++ b/src/core/stiysZND
diff --git a/src/ext/.may_include b/src/ext/.may_include
new file mode 100644
index 0000000000..1eafff2eeb
--- /dev/null
+++ b/src/ext/.may_include
@@ -0,0 +1,10 @@
+
+orconfig.h
+
+lib/err/*.h
+lib/cc/*.h
+
+tinytest*.h
+ext/siphash.h
+ext/byteorder.h
+ext/tor_readpassphrase.h
\ No newline at end of file
diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c
index a65b6fcbe6..e0f5b2e5c9 100644
--- a/src/ext/csiphash.c
+++ b/src/ext/csiphash.c
@@ -30,12 +30,12 @@
 */
 
 #include "lib/cc/torint.h"
-#include "lib/log/util_bug.h"
+#include "lib/err/torerr.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 #include <string.h>
 #include <stdlib.h>
-#include "byteorder.h"
+#include "ext/byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
@@ -87,6 +87,13 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		v0 ^= mi;
 	}
 
+#ifdef __COVERITY__
+	{
+		uint64_t mi = 0;
+		memcpy(&mi, m+i, (src_sz-blocks));
+		last7 = _le64toh(mi) | (uint64_t)(src_sz & 0xff) << 56;
+	}
+#else
 	switch (src_sz - blocks) {
 		case 7: last7 |= (uint64_t)m[i + 6] << 48; FALLTHROUGH;
 		case 6: last7 |= (uint64_t)m[i + 5] << 40; FALLTHROUGH;
@@ -98,6 +105,7 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		case 0:
 		default:;
 	}
+#endif
 	v3 ^= last7;
 	DOUBLE_ROUND(v0,v1,v2,v3);
 	v0 ^= last7;
@@ -112,13 +120,13 @@ static int the_siphash_key_is_set = 0;
 static struct sipkey the_siphash_key;
 
 uint64_t siphash24g(const void *src, unsigned long src_sz) {
-	tor_assert(the_siphash_key_is_set);
+	raw_assert(the_siphash_key_is_set);
 	return siphash24(src, src_sz, &the_siphash_key);
 }
 
 void siphash_set_global_key(const struct sipkey *key)
 {
-	tor_assert(! the_siphash_key_is_set);
+	raw_assert(! the_siphash_key_is_set);
 	the_siphash_key.k0 = key->k0;
 	the_siphash_key.k1 = key->k1;
 	the_siphash_key_is_set = 1;
diff --git a/src/ext/ed25519/ref10/base.py b/src/ext/ed25519/ref10/base.py
index 84accc8580..3d477c5c39 100644
--- a/src/ext/ed25519/ref10/base.py
+++ b/src/ext/ed25519/ref10/base.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 b = 256
 q = 2**255 - 19
 l = 2**252 + 27742317777372353535851937790883648493
@@ -51,15 +56,15 @@ def radix255(x):
 
 Bi = B
 for i in range(32):
-  print "{"
+  print("{")
   Bij = Bi
   for j in range(8):
-    print " {"
-    print "  {",radix255(Bij[1]+Bij[0]),"},"
-    print "  {",radix255(Bij[1]-Bij[0]),"},"
-    print "  {",radix255(2*d*Bij[0]*Bij[1]),"},"
+    print(" {")
+    print("  {",radix255(Bij[1]+Bij[0]),"},")
+    print("  {",radix255(Bij[1]-Bij[0]),"},")
+    print("  {",radix255(2*d*Bij[0]*Bij[1]),"},")
     Bij = edwards(Bij,Bi)
-    print " },"
-  print "},"
+    print(" },")
+  print("},")
   for k in range(8):
     Bi = edwards(Bi,Bi)
diff --git a/src/ext/ed25519/ref10/base2.py b/src/ext/ed25519/ref10/base2.py
index 5e4e8739d0..3f8e3d25d2 100644
--- a/src/ext/ed25519/ref10/base2.py
+++ b/src/ext/ed25519/ref10/base2.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 b = 256
 q = 2**255 - 19
 l = 2**252 + 27742317777372353535851937790883648493
@@ -52,9 +57,9 @@ def radix255(x):
 Bi = B
 
 for i in range(8):
-  print " {"
-  print "  {",radix255(Bi[1]+Bi[0]),"},"
-  print "  {",radix255(Bi[1]-Bi[0]),"},"
-  print "  {",radix255(2*d*Bi[0]*Bi[1]),"},"
-  print " },"
+  print(" {")
+  print("  {",radix255(Bi[1]+Bi[0]),"},")
+  print("  {",radix255(Bi[1]-Bi[0]),"},")
+  print("  {",radix255(2*d*Bi[0]*Bi[1]),"},")
+  print(" },")
   Bi = edwards(B,edwards(B,Bi))
diff --git a/src/ext/ed25519/ref10/d.py b/src/ext/ed25519/ref10/d.py
index 8995bb86a3..5b875de666 100644
--- a/src/ext/ed25519/ref10/d.py
+++ b/src/ext/ed25519/ref10/d.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 q = 2**255 - 19
 
 def expmod(b,e,m):
@@ -25,4 +30,4 @@ def radix255(x):
   return result
 
 d = -121665 * inv(121666)
-print radix255(d)
+print(radix255(d))
diff --git a/src/ext/ed25519/ref10/d2.py b/src/ext/ed25519/ref10/d2.py
index 79841758be..f59a1bc62a 100644
--- a/src/ext/ed25519/ref10/d2.py
+++ b/src/ext/ed25519/ref10/d2.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 q = 2**255 - 19
 
 def expmod(b,e,m):
@@ -25,4 +30,4 @@ def radix255(x):
   return result
 
 d = -121665 * inv(121666)
-print radix255(d*2)
+print(radix255(d*2))
diff --git a/src/ext/ed25519/ref10/sqrtm1.py b/src/ext/ed25519/ref10/sqrtm1.py
index 9a47fbc12a..df9f26ee1d 100644
--- a/src/ext/ed25519/ref10/sqrtm1.py
+++ b/src/ext/ed25519/ref10/sqrtm1.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 q = 2**255 - 19
 
 def expmod(b,e,m):
@@ -25,4 +30,4 @@ def radix255(x):
   return result
 
 I = expmod(2,(q-1)/4,q)
-print radix255(I)
+print(radix255(I))
diff --git a/src/ext/ht.h b/src/ext/ht.h
index 54e5eb7cba..4bfce36903 100644
--- a/src/ext/ht.h
+++ b/src/ext/ht.h
@@ -226,11 +226,16 @@ ht_string_hash(const char *s)
        (x) = HT_NEXT(name, head, x))
 
 #ifndef HT_NDEBUG
-#define HT_ASSERT_(x) tor_assert(x)
+#include "lib/err/torerr.h"
+#define HT_ASSERT_(x) raw_assert(x)
 #else
 #define HT_ASSERT_(x) (void)0
 #endif
 
+/* Macro put at the end of the end of a macro definition so that it
+ * consumes the following semicolon at file scope. Used only inside ht.h. */
+#define HT_EAT_SEMICOLON__ struct ht_semicolon_eater
+
 #define HT_PROTOTYPE(name, type, field, hashfn, eqfn)                   \
   int name##_HT_GROW(struct name *ht, unsigned min_capacity);           \
   void name##_HT_CLEAR(struct name *ht);                                \
@@ -370,7 +375,8 @@ ht_string_hash(const char *s)
   /* Return the next element in 'head' after 'elm', under the arbitrary \
    * order used by HT_START.  If there are no more elements, return     \
    * NULL.  If 'elm' is to be removed from the table, you must call     \
-   * this function for the next value before you remove it.             \
+   * this function for the next value before you remove it, or use      \
+   * HT_NEXT_RMV instead.                                               \
    */                                                                   \
   ATTR_UNUSED static inline struct type **                              \
   name##_HT_NEXT(struct name *head, struct type **elm)                  \
@@ -392,6 +398,8 @@ ht_string_hash(const char *s)
       return NULL;                                                      \
     }                                                                   \
   }                                                                     \
+  /* As HT_NEXT, but also remove the current element 'elm' from the     \
+   * table. */                                                          \
   ATTR_UNUSED static inline struct type **                              \
   name##_HT_NEXT_RMV(struct name *head, struct type **elm)              \
   {                                                                     \
@@ -409,7 +417,8 @@ ht_string_hash(const char *s)
       }                                                                 \
       return NULL;                                                      \
     }                                                                   \
-  }
+  }                                                                     \
+  HT_EAT_SEMICOLON__
 
 #define HT_GENERATE2(name, type, field, hashfn, eqfn, load, reallocarrayfn, \
                      freefn)                                            \
@@ -534,7 +543,8 @@ ht_string_hash(const char *s)
     if (n != head->hth_n_entries)                                       \
       return 6;                                                         \
     return 0;                                                           \
-  }
+  }                                                                     \
+  HT_EAT_SEMICOLON__
 
 #define HT_GENERATE(name, type, field, hashfn, eqfn, load, mallocfn,    \
                     reallocfn, freefn)                                  \
@@ -617,4 +627,3 @@ ht_string_hash(const char *s)
 */
 
 #endif
-
diff --git a/src/ext/include.am b/src/ext/include.am
index 6bdce2d79e..317e25d78e 100644
--- a/src/ext/include.am
+++ b/src/ext/include.am
@@ -143,6 +143,7 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS)
 LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a
 noinst_LIBRARIES += $(LIBED25519_DONNA)
 
+if BUILD_KECCAK_TINY
 src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\
   @CFLAGS_CONSTTIME@
 
@@ -156,6 +157,7 @@ noinst_HEADERS += $(LIBKECCAK_TINY_HDRS)
 
 LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a
 noinst_LIBRARIES += $(LIBKECCAK_TINY)
+endif
 
 EXTRA_DIST += \
 	src/ext/timeouts/bench/bench-add.lua 		\
diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c
index e0df05d7b7..16611af1e2 100644
--- a/src/ext/readpassphrase.c
+++ b/src/ext/readpassphrase.c
@@ -30,7 +30,7 @@
 #include <signal.h>
 #include <ctype.h>
 #include <fcntl.h>
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
diff --git a/src/ext/siphash.h b/src/ext/siphash.h
index 730e49937d..0207a959ff 100644
--- a/src/ext/siphash.h
+++ b/src/ext/siphash.h
@@ -1,6 +1,8 @@
 #ifndef SIPHASH_H
 #define SIPHASH_H
 
+#include <stdint.h>
+
 struct sipkey {
   uint64_t k0;
   uint64_t k1;
diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include
new file mode 100644
index 0000000000..92c7116555
--- /dev/null
+++ b/src/ext/timeouts/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+
+ext/tor_queue.h
+ext/timeouts/*.h
+ext/timeouts/timeout-bitops.c
diff --git a/src/ext/timeouts/test-timeout.c b/src/ext/timeouts/test-timeout.c
index 8077129376..52d2e31e0c 100644
--- a/src/ext/timeouts/test-timeout.c
+++ b/src/ext/timeouts/test-timeout.c
@@ -4,7 +4,7 @@
 #include <assert.h>
 #include <limits.h>
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #define THE_END_OF_TIME ((timeout_t)-1)
 
diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c
index d4b514d2c5..79fcc168ed 100644
--- a/src/ext/timeouts/timeout.c
+++ b/src/ext/timeouts/timeout.c
@@ -38,16 +38,16 @@
 
 #include <errno.h>     /* errno */
 
-#include "tor_queue.h" /* TAILQ(3) */
+#include "ext/tor_queue.h" /* TAILQ(3) */
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #ifndef TIMEOUT_DEBUG
 #define TIMEOUT_DEBUG 0
 #endif
 
 #if TIMEOUT_DEBUG - 0
-#include "timeout-debug.h"
+#include "ext/timeouts/timeout-debug.h"
 #endif
 
 #ifdef TIMEOUT_DISABLE_RELATIVE_ACCESS
@@ -141,7 +141,7 @@
 #define WHEEL_MASK (WHEEL_LEN - 1)
 #define TIMEOUT_MAX ((TIMEOUT_C(1) << (WHEEL_BIT * WHEEL_NUM)) - 1)
 
-#include "timeout-bitops.c"
+#include "ext/timeouts/timeout-bitops.c"
 
 #if WHEEL_BIT == 6
 #define ctz(n) ctz64(n)
@@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) {
 			timeout = MIN(_timeout, timeout);
 		}
 
-		relmask <<= WHEEL_BIT; 
+		relmask <<= WHEEL_BIT;
 		relmask |= WHEEL_MASK;
 	}
 
@@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) {
 TIMEOUT_PUBLIC int timeout_v_api(void) {
 	return TIMEOUT_V_API;
 } /* timeout_version() */
-
diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h
index b35874e153..f1028bfc80 100644
--- a/src/ext/timeouts/timeout.h
+++ b/src/ext/timeouts/timeout.h
@@ -31,7 +31,7 @@
 
 #include <inttypes.h>   /* PRIu64 PRIx64 PRIX64 uint64_t */
 
-#include "tor_queue.h"  /* TAILQ(3) */
+#include "ext/tor_queue.h"  /* TAILQ(3) */
 
 
 /*
@@ -89,10 +89,10 @@ typedef uint64_t timeout_t;
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #ifndef TIMEOUT_CB_OVERRIDE
-struct timeout_cb {
+struct timeout_cb_t {
 	void (*fn)(void);
 	void *arg;
-}; /* struct timeout_cb */
+}; /* struct timeout_cb_t */
 #endif
 
 /*
@@ -125,7 +125,7 @@ struct timeout {
 	/* entry member for struct timeout_list lists */
 
 #ifndef TIMEOUT_DISABLE_CALLBACKS
-	struct timeout_cb callback;
+	struct timeout_cb_t callback;
 	/* optional callback information */
 #endif
 
@@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int);
 #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS
 TIMEOUT_PUBLIC bool timeout_pending(struct timeout *);
 /* true if on timing wheel, false otherwise */
- 
+
 TIMEOUT_PUBLIC bool timeout_expired(struct timeout *);
 /* true if on expired queue, false otherwise */
 
diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c
index 16f11e4639..239fdd0a38 100644
--- a/src/ext/tinytest.c
+++ b/src/ext/tinytest.c
@@ -492,6 +492,12 @@ tinytest_set_test_skipped_(void)
 		cur_test_outcome = SKIP;
 }
 
+int
+tinytest_cur_test_has_failed(void)
+{
+	return (cur_test_outcome == FAIL);
+}
+
 char *
 tinytest_format_hex_(const void *val_, unsigned long len)
 {
diff --git a/src/ext/tinytest.h b/src/ext/tinytest.h
index ed07b26bc0..05c2fda052 100644
--- a/src/ext/tinytest.h
+++ b/src/ext/tinytest.h
@@ -72,6 +72,9 @@ struct testlist_alias_t {
 };
 #define END_OF_ALIASES { NULL, NULL }
 
+/** Return true iff the current test has failed. */
+int tinytest_cur_test_has_failed(void);
+
 /** Implementation: called from a test to indicate failure, before logging. */
 void tinytest_set_test_failed_(void);
 /** Implementation: called from a test to indicate that we're skipping. */
diff --git a/src/ext/tinytest_macros.h b/src/ext/tinytest_macros.h
index c3728d1fdd..6fc2cea2da 100644
--- a/src/ext/tinytest_macros.h
+++ b/src/ext/tinytest_macros.h
@@ -99,11 +99,11 @@
 
 /* Assert b, but do not stop the test if b fails.  Log msg on failure. */
 #define tt_want_msg(b, msg)			\
-	tt_want_(b, msg, );
+	tt_want_(b, msg, )
 
 /* Assert b and stop the test if b fails.  Log msg on failure. */
 #define tt_assert_msg(b, msg)			\
-	tt_want_(b, msg, TT_EXIT_TEST_FUNCTION);
+	tt_want_(b, msg, TT_EXIT_TEST_FUNCTION)
 
 /* Assert b, but do not stop the test if b fails. */
 #define tt_want(b)   tt_want_msg( (b), "want("#b")")
diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h
index 15d1c8633e..52afa9ccd4 100644
--- a/src/ext/trunnel/trunnel-impl.h
+++ b/src/ext/trunnel/trunnel-impl.h
@@ -1,4 +1,4 @@
-/* trunnel-impl.h -- copied from Trunnel v1.5.2
+/* trunnel-impl.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c
index 3ae3fe02c8..01a55c5bec 100644
--- a/src/ext/trunnel/trunnel.c
+++ b/src/ext/trunnel/trunnel.c
@@ -1,4 +1,4 @@
-/* trunnel.c -- copied from Trunnel v1.5.2
+/* trunnel.c -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h
index 9b708437b8..87c75f4ec3 100644
--- a/src/ext/trunnel/trunnel.h
+++ b/src/ext/trunnel/trunnel.h
@@ -1,4 +1,4 @@
-/* trunnel.h -- copied from Trunnel v1.5.2
+/* trunnel.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/feature/api/.may_include b/src/feature/api/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/api/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/api/feature_api.md b/src/feature/api/feature_api.md
new file mode 100644
index 0000000000..3065c000aa
--- /dev/null
+++ b/src/feature/api/feature_api.md
@@ -0,0 +1,2 @@
+@dir /feature/api
+@brief feature/api: In-process interface to starting/stopping Tor.
diff --git a/src/feature/api/include.am b/src/feature/api/include.am
new file mode 100644
index 0000000000..8d490458d4
--- /dev/null
+++ b/src/feature/api/include.am
@@ -0,0 +1,11 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/api/tor_api.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/api/tor_api_internal.h
+
+# This may someday want to be an installed file?
+noinst_HEADERS += src/feature/api/tor_api.h
diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c
index 697397d46b..531793301e 100644
--- a/src/feature/api/tor_api.c
+++ b/src/feature/api/tor_api.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,10 +40,10 @@
 #define raw_socketpair tor_ersatz_socketpair
 #define raw_closesocket closesocket
 #define snprintf _snprintf
-#else
+#else /* !defined(_WIN32) */
 #define raw_socketpair socketpair
 #define raw_closesocket close
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h
index 2bf130c376..e9993bb0d5 100644
--- a/src/feature/api/tor_api.h
+++ b/src/feature/api/tor_api.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -55,7 +55,7 @@ typedef SOCKET tor_control_socket_t;
 #else
 typedef int tor_control_socket_t;
 #define INVALID_TOR_CONTROL_SOCKET (-1)
-#endif
+#endif /* defined(_WIN32) */
 
 /** DOCDOC */
 tor_control_socket_t tor_main_configuration_setup_control_socket(
diff --git a/src/feature/api/tor_api_internal.h b/src/feature/api/tor_api_internal.h
index 60e0f3aa59..d52b2caf44 100644
--- a/src/feature/api/tor_api_internal.h
+++ b/src/feature/api/tor_api_internal.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tor_api_internal.h
+ * @brief Internal declarations for in-process Tor API.
+ **/
+
 #ifndef TOR_API_INTERNAL_H
 #define TOR_API_INTERNAL_H
 
diff --git a/src/feature/client/.may_include b/src/feature/client/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/client/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/client/addressmap.c b/src/feature/client/addressmap.c
index bbe786a6a2..9ad2d7f934 100644
--- a/src/feature/client/addressmap.c
+++ b/src/feature/client/addressmap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,8 +22,7 @@
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
-#include "feature/relay/dns.h"
+#include "feature/control/control_events.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
 
@@ -689,7 +688,7 @@ client_dns_set_addressmap_impl(entry_connection_t *for_conn,
   if (ttl<0)
     ttl = DEFAULT_DNS_TTL;
   else
-    ttl = dns_clip_ttl(ttl);
+    ttl = clip_dns_ttl(ttl);
 
   if (exitname) {
     /* XXXX fails to ever get attempts to get an exit address of
@@ -903,7 +902,7 @@ get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out)
   }
 
   if (ipv6)
-    tor_addr_from_ipv6_bytes(addr_out, (char*) bytes);
+    tor_addr_from_ipv6_bytes(addr_out, bytes);
   else
     tor_addr_from_ipv4n(addr_out, get_uint32(bytes));
 
diff --git a/src/feature/client/addressmap.h b/src/feature/client/addressmap.h
index 9179aef1d0..7f1024e09a 100644
--- a/src/feature/client/addressmap.h
+++ b/src/feature/client/addressmap.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file addressmap.h
+ * @brief Header for addressmap.c
+ **/
+
 #ifndef TOR_ADDRESSMAP_H
 #define TOR_ADDRESSMAP_H
 
@@ -62,4 +67,3 @@ STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf,
 #endif /* defined(ADDRESSMAP_PRIVATE) */
 
 #endif /* !defined(TOR_ADDRESSMAP_H) */
-
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
index 626c5efcae..66b04f3bc2 100644
--- a/src/feature/client/bridges.c
+++ b/src/feature/client/bridges.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -348,7 +348,7 @@ int
 node_is_a_configured_bridge(const node_t *node)
 {
   /* First, let's try searching for a bridge with matching identity. */
-  if (BUG(tor_digest_is_zero(node->identity)))
+  if (BUG(fast_mem_is_zero(node->identity, DIGEST_LEN)))
     return 0;
 
   if (find_bridge_by_digest(node->identity) != NULL)
diff --git a/src/feature/client/bridges.h b/src/feature/client/bridges.h
index 27b2750a45..174149cf97 100644
--- a/src/feature/client/bridges.h
+++ b/src/feature/client/bridges.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c
index 1743ab5a81..74260171fe 100644
--- a/src/feature/client/circpathbias.c
+++ b/src/feature/client/circpathbias.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -176,6 +176,7 @@ pathbias_get_scale_threshold(const or_options_t *options)
 static double
 pathbias_get_scale_ratio(const or_options_t *options)
 {
+  (void) options;
   /*
    * The scale factor is the denominator for our scaling
    * of circuit counts for our path bias window.
@@ -185,7 +186,8 @@ pathbias_get_scale_ratio(const or_options_t *options)
    */
   int denominator = networkstatus_get_param(NULL, "pb_scalefactor",
                               2, 2, INT32_MAX);
-  (void) options;
+  tor_assert(denominator > 0);
+
   /**
    * The mult factor is the numerator for our scaling
    * of circuit counts for our path bias window. It
@@ -301,7 +303,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
   return circ->cpath &&
          circ->cpath->next != circ->cpath &&
          circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS;
-#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */
+#else /* !defined(N2N_TAGGING_IS_POSSIBLE) */
   /* If tagging attacks are no longer possible, we probably want to
    * count bias from the first hop. However, one could argue that
    * timing-based tagging is still more useful than per-hop failure.
@@ -369,8 +371,9 @@ pathbias_should_count(origin_circuit_t *circ)
         !circ->build_state->onehop_tunnel) {
       if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) {
         log_info(LD_BUG,
-               "One-hop circuit has length %d. Path state is %s. "
+               "One-hop circuit %d has length %d. Path state is %s. "
                "Circuit is a %s currently %s.%s",
+               circ->global_identifier,
                circ->build_state->desired_path_len,
                pathbias_state_to_string(circ->path_state),
                circuit_purpose_to_string(circ->base_.purpose),
@@ -398,12 +401,13 @@ pathbias_should_count(origin_circuit_t *circ)
   /* Check to see if the shouldcount result has changed due to a
    * unexpected purpose change that would affect our results */
   if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) {
-      log_info(LD_BUG,
-              "Circuit %d is now being counted despite being ignored "
-              "in the past. Purpose is %s, path state is %s",
-              circ->global_identifier,
-              circuit_purpose_to_string(circ->base_.purpose),
-              pathbias_state_to_string(circ->path_state));
+    log_info(LD_CIRC,
+            "Circuit %d is not being counted by pathbias because it was "
+            "ignored in the past. Purpose is %s, path state is %s",
+            circ->global_identifier,
+            circuit_purpose_to_string(circ->base_.purpose),
+            pathbias_state_to_string(circ->path_state));
+    return 0;
   }
   circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED;
 
@@ -434,8 +438,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                                      approx_time()))) {
         log_info(LD_BUG,
-                "Opened circuit is in strange path state %s. "
+                "Opened circuit %d is in strange path state %s. "
                 "Circuit is a %s currently %s.%s",
+                circ->global_identifier,
                 pathbias_state_to_string(circ->path_state),
                 circuit_purpose_to_string(circ->base_.purpose),
                 circuit_state_to_string(circ->base_.state),
@@ -468,8 +473,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
           if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                   approx_time()))) {
             log_info(LD_BUG,
-                   "Unopened circuit has strange path state %s. "
+                   "Unopened circuit %d has strange path state %s. "
                    "Circuit is a %s currently %s.%s",
+                   circ->global_identifier,
                    pathbias_state_to_string(circ->path_state),
                    circuit_purpose_to_string(circ->base_.purpose),
                    circuit_state_to_string(circ->base_.state),
@@ -538,8 +544,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
         if ((rate_msg = rate_limit_log(&success_notice_limit,
                 approx_time()))) {
           log_info(LD_BUG,
-              "Succeeded circuit is in strange path state %s. "
+              "Succeeded circuit %d is in strange path state %s. "
               "Circuit is a %s currently %s.%s",
+              circ->global_identifier,
               pathbias_state_to_string(circ->path_state),
               circuit_purpose_to_string(circ->base_.purpose),
               circuit_state_to_string(circ->base_.state),
@@ -574,8 +581,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&success_notice_limit,
               approx_time()))) {
         log_info(LD_BUG,
-            "Opened circuit is in strange path state %s. "
+            "Opened circuit %d is in strange path state %s. "
             "Circuit is a %s currently %s.%s",
+            circ->global_identifier,
             pathbias_state_to_string(circ->path_state),
             circuit_purpose_to_string(circ->base_.purpose),
             circuit_state_to_string(circ->base_.state),
@@ -601,8 +609,9 @@ pathbias_count_use_attempt(origin_circuit_t *circ)
 
   if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) {
     log_notice(LD_BUG,
-        "Used circuit is in strange path state %s. "
+        "Used circuit %d is in strange path state %s. "
         "Circuit is a %s currently %s.",
+        circ->global_identifier,
         pathbias_state_to_string(circ->path_state),
         circuit_purpose_to_string(circ->base_.purpose),
         circuit_state_to_string(circ->base_.state));
@@ -817,6 +826,11 @@ pathbias_send_usable_probe(circuit_t *circ)
   ocirc->pathbias_probe_nonce &= 0x00ffffff;
   probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce);
 
+  if (!probe_nonce) {
+    log_err(LD_BUG, "Failed to generate nonce");
+    return -1;
+  }
+
   tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce);
   payload_len = (int)strlen(payload)+1;
 
diff --git a/src/feature/client/circpathbias.h b/src/feature/client/circpathbias.h
index a9a8d18df2..88cc982dd4 100644
--- a/src/feature/client/circpathbias.h
+++ b/src/feature/client/circpathbias.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c
index cf593cfb68..c1981ecde0 100644
--- a/src/feature/client/dnsserv.c
+++ b/src/feature/client/dnsserv.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,8 +26,9 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/policies.h"
 
 #include "feature/control/control_connection_st.h"
@@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse,
   edge_connection_t *conn;
   char *q_name;
 
+  /* Launching a request for a user counts as user activity. */
+  note_user_activity(approx_time());
+
   /* Make a new dummy AP connection, and attach the request to it. */
   entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
   entry_conn->entry_cfg.dns_request = 1;
@@ -234,7 +238,7 @@ dnsserv_launch_request(const char *name, int reverse,
     TO_CONN(conn)->port = control_conn->base_.port;
     TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
   }
-#else /* !(defined(AF_UNIX)) */
+#else /* !defined(AF_UNIX) */
   TO_CONN(conn)->port = control_conn->base_.port;
   TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
 #endif /* defined(AF_UNIX) */
diff --git a/src/feature/client/dnsserv.h b/src/feature/client/dnsserv.h
index fff1ed2adb..4011cb4e02 100644
--- a/src/feature/client/dnsserv.h
+++ b/src/feature/client/dnsserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
index 6c7f8057bd..70ef64cc86 100644
--- a/src/feature/client/entrynodes.c
+++ b/src/feature/client/entrynodes.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -47,8 +47,7 @@
  * As a persistent ordered list whose elements are taken from the
  * sampled set, we track a CONFIRMED GUARDS LIST.  A guard becomes
  * confirmed when we successfully build a circuit through it, and decide
- * to use that circuit.  We order the guards on this list by the order
- * in which they became confirmed.
+ * to use that circuit.
  *
  * And as a final group, we have an ordered list of PRIMARY GUARDS,
  * whose elements are taken from the filtered set. We prefer
@@ -59,7 +58,7 @@
  *
  * To build circuits, we take a primary guard if possible -- or a
  * reachable filtered confirmed guard if no primary guard is possible --
- * or a random reachable filtered guard otherwise.  If the guard is
+ * or the first (by sampled order) filtered guard otherwise.  If the guard is
  * primary, we can use the circuit immediately on success.  Otherwise,
  * the guard is now "pending" -- we won't use its circuit unless all
  * of the circuits we're trying to build through better guards have
@@ -92,14 +91,18 @@
  * [x] Whenever we remove a guard from the sample, remove it from the primary
  * and confirmed lists.
  *
- * [x] When we make a guard confirmed, update the primary list.
+ * [x] When we make a guard confirmed, update the primary list, and sort them
+ * by sampled order.
  *
  * [x] When we make a guard filtered or unfiltered, update the primary list.
  *
  * [x] When we are about to pick a guard, make sure that the primary list is
  * full.
  *
- * [x] Before calling sample_reachable_filtered_entry_guards(), make sure
+ * [x] When we update the confirmed list, or when we re-build the primary list
+ * and detect a change, we sort those lists by sampled_idx
+ *
+ * [x] Before calling first_reachable_filtered_entry_guard(), make sure
  * that the filtered, primary, and confirmed flags are up-to-date.
  *
  * [x] Call entry_guard_consider_retry every time we are about to check
@@ -114,7 +117,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "app/config/statefile.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
@@ -128,7 +131,7 @@
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
@@ -172,6 +175,7 @@ static entry_guard_t *get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
                                               const tor_addr_port_t *addrport);
 static int entry_guard_obeys_restriction(const entry_guard_t *guard,
                                          const entry_guard_restriction_t *rst);
+static int compare_guards_by_sampled_idx(const void **a_, const void **b_);
 
 /** Return 0 if we should apply guardfraction information found in the
  *  consensus. A specific consensus can be specified with the
@@ -890,6 +894,7 @@ entry_guard_add_to_sample_impl(guard_selection_t *gs,
   tor_free(guard->sampled_by_version);
   guard->sampled_by_version = tor_strdup(VERSION);
   guard->currently_listed = 1;
+  guard->sampled_idx = gs->next_sampled_idx++;
   guard->confirmed_idx = -1;
 
   /* non-persistent fields */
@@ -901,6 +906,11 @@ entry_guard_add_to_sample_impl(guard_selection_t *gs,
   guard->in_selection = gs;
   entry_guard_set_filtered_flags(get_options(), gs, guard);
   entry_guards_changed_for_guard_selection(gs);
+
+  /* Just added this guard to the sampled set and hence it might be used as a
+   * guard in the future: send GUARD NEW control event. */
+  control_event_guard(guard->nickname, guard->identity, "NEW");
+
   return guard;
 }
 
@@ -1038,7 +1048,7 @@ get_max_sample_size(guard_selection_t *gs,
  * Return a smartlist of the all the guards that are not currently
  * members of the sample (GUARDS - SAMPLED_GUARDS).  The elements of
  * this list are node_t pointers in the non-bridge case, and
- * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out/b>
+ * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out</b>
  * to the number of guards that we found in GUARDS, including those
  * that were already sampled.
  */
@@ -1383,7 +1393,7 @@ sampled_guards_prune_obsolete_entries(guard_selection_t *gs,
 
     if (rmv) {
       ++n_changes;
-      SMARTLIST_DEL_CURRENT(gs->sampled_entry_guards, guard);
+      SMARTLIST_DEL_CURRENT_KEEPORDER(gs->sampled_entry_guards, guard);
       remove_guard_from_confirmed_and_primary_lists(gs, guard);
       entry_guard_free(guard);
     }
@@ -1707,7 +1717,7 @@ entry_guards_update_filtered_sets(guard_selection_t *gs)
 }
 
 /**
- * Return a random guard from the reachable filtered sample guards
+ * Return the first sampled guard from the reachable filtered sample guards
  * in <b>gs</b>, subject to the exclusion rules listed in <b>flags</b>.
  * Return NULL if no such guard can be found.
  *
@@ -1718,7 +1728,7 @@ entry_guards_update_filtered_sets(guard_selection_t *gs)
  * violate it.
  **/
 STATIC entry_guard_t *
-sample_reachable_filtered_entry_guards(guard_selection_t *gs,
+first_reachable_filtered_entry_guard(guard_selection_t *gs,
                                        const entry_guard_restriction_t *rst,
                                        unsigned flags)
 {
@@ -1771,7 +1781,17 @@ sample_reachable_filtered_entry_guards(guard_selection_t *gs,
            flags, smartlist_len(reachable_filtered_sample));
 
   if (smartlist_len(reachable_filtered_sample)) {
-    result = smartlist_choose(reachable_filtered_sample);
+    /**
+     * Get the first guard of the filtered set builds from
+     * sampled_entry_guards. Proposal 310 suggests this design to overcome
+     * performance and security issues linked to the previous selection
+     * method. The guard selected here should be filtered out if this function
+     * is called again in the same context. I.e., if we filter guards to add
+     * them into some list X, then the guards from list X will be filtered out
+     * when this function is called again. Hence it requires setting exclude
+     * flags in a appropriate way (depending of the context of the caller).
+     */
+    result = smartlist_get(reachable_filtered_sample, 0);
     log_info(LD_GUARD, "  (Selected %s.)",
              result ? entry_guard_describe(result) : "<null>");
   }
@@ -1780,10 +1800,6 @@ sample_reachable_filtered_entry_guards(guard_selection_t *gs,
   return result;
 }
 
-/**
- * Helper: compare two entry_guard_t by their confirmed_idx values.
- * Used to sort the confirmed list.
- */
 static int
 compare_guards_by_confirmed_idx(const void **a_, const void **b_)
 {
@@ -1795,6 +1811,21 @@ compare_guards_by_confirmed_idx(const void **a_, const void **b_)
   else
     return 0;
 }
+/**
+ * Helper: compare two entry_guard_t by their sampled_idx values.
+ * Used to sort the sampled list
+ */
+static int
+compare_guards_by_sampled_idx(const void **a_, const void **b_)
+{
+  const entry_guard_t *a = *a_, *b = *b_;
+  if (a->sampled_idx < b->sampled_idx)
+    return -1;
+  else if (a->sampled_idx > b->sampled_idx)
+    return 1;
+  else
+    return 0;
+}
 
 /**
  * Find the confirmed guards from among the sampled guards in <b>gs</b>,
@@ -1811,7 +1842,7 @@ entry_guards_update_confirmed(guard_selection_t *gs)
   } SMARTLIST_FOREACH_END(guard);
 
   smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_confirmed_idx);
-
+  /** Needed to keep a dense array of confirmed_idx */
   int any_changed = 0;
   SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
     if (guard->confirmed_idx != guard_sl_idx) {
@@ -1821,6 +1852,8 @@ entry_guards_update_confirmed(guard_selection_t *gs)
   } SMARTLIST_FOREACH_END(guard);
 
   gs->next_confirmed_idx = smartlist_len(gs->confirmed_entry_guards);
+  // We need the confirmed list to always be give guards in sampled order
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_sampled_idx);
 
   if (any_changed) {
     entry_guards_changed_for_guard_selection(gs);
@@ -1849,6 +1882,9 @@ make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard)
 
   guard->confirmed_idx = gs->next_confirmed_idx++;
   smartlist_add(gs->confirmed_entry_guards, guard);
+  /** The confirmation ordering might not be the sample ording. We need to
+   * reorder */
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_sampled_idx);
 
   // This confirmed guard might kick something else out of the primary
   // guards.
@@ -1912,7 +1948,7 @@ entry_guards_update_primary(guard_selection_t *gs)
 
   /* Finally, fill out the list with sampled guards. */
   while (smartlist_len(new_primary_guards) < N_PRIMARY_GUARDS) {
-    entry_guard_t *guard = sample_reachable_filtered_entry_guards(gs, NULL,
+    entry_guard_t *guard = first_reachable_filtered_entry_guard(gs, NULL,
                                             SAMPLE_EXCLUDE_CONFIRMED|
                                             SAMPLE_EXCLUDE_PRIMARY|
                                             SAMPLE_NO_UPDATE_PRIMARY);
@@ -1943,6 +1979,7 @@ entry_guards_update_primary(guard_selection_t *gs)
                g->confirmed_idx >= 0 ? " (confirmed)" : "",
                g->is_filtered_guard ? "" : " (excluded by filter)");
     } SMARTLIST_FOREACH_END(g);
+    smartlist_sort(new_primary_guards, compare_guards_by_sampled_idx);
   }
 
   smartlist_free(old_primary_guards);
@@ -1974,10 +2011,12 @@ get_retry_schedule(time_t failing_since, time_t now,
   const struct {
     time_t maximum; int primary_delay; int nonprimary_delay;
   } delays[] = {
+    // clang-format off
     { SIX_HOURS,    10*60,  1*60*60 },
     { FOUR_DAYS,    90*60,  4*60*60 },
     { SEVEN_DAYS, 4*60*60, 18*60*60 },
     { TIME_MAX,   9*60*60, 36*60*60 }
+    // clang-format on
   };
 
   unsigned i;
@@ -2053,10 +2092,15 @@ select_primary_guard_for_circuit(guard_selection_t *gs,
 
   SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
     entry_guard_consider_retry(guard);
-    if (! entry_guard_obeys_restriction(guard, rst))
+    if (!entry_guard_obeys_restriction(guard, rst)) {
+      log_info(LD_GUARD, "Entry guard %s doesn't obey restriction, we test the"
+          " next one", entry_guard_describe(guard));
       continue;
+    }
     if (guard->is_reachable != GUARD_REACHABLE_NO) {
       if (need_descriptor && !guard_has_descriptor(guard)) {
+        log_info(LD_GUARD, "Guard %s does not have a descriptor",
+            entry_guard_describe(guard));
         continue;
       }
       *state_out = GUARD_CIRC_STATE_USABLE_ON_COMPLETION;
@@ -2069,9 +2113,11 @@ select_primary_guard_for_circuit(guard_selection_t *gs,
 
   if (smartlist_len(usable_primary_guards)) {
     chosen_guard = smartlist_choose(usable_primary_guards);
+    log_info(LD_GUARD,
+        "Selected primary guard %s for circuit from a list size of %d.",
+        entry_guard_describe(chosen_guard),
+        smartlist_len(usable_primary_guards));
     smartlist_free(usable_primary_guards);
-    log_info(LD_GUARD, "Selected primary guard %s for circuit.",
-             entry_guard_describe(chosen_guard));
   }
 
   smartlist_free(usable_primary_guards);
@@ -2116,10 +2162,10 @@ select_confirmed_guard_for_circuit(guard_selection_t *gs,
 }
 
 /**
- * For use with a circuit, pick a confirmed usable filtered guard
- * at random. Update the <b>last_tried_to_connect</b> time and the
- * <b>is_pending</b> fields of the guard as appropriate. Set <b>state_out</b>
- * to the new guard-state of the circuit.
+ * For use with a circuit, pick a usable filtered guard. Update the
+ * <b>last_tried_to_connect</b> time and the <b>is_pending</b> fields of the
+ * guard as appropriate. Set <b>state_out</b> to the new guard-state of the
+ * circuit.
  */
 static entry_guard_t *
 select_filtered_guard_for_circuit(guard_selection_t *gs,
@@ -2132,7 +2178,7 @@ select_filtered_guard_for_circuit(guard_selection_t *gs,
   unsigned flags = 0;
   if (need_descriptor)
     flags |= SAMPLE_EXCLUDE_NO_DESCRIPTOR;
-  chosen_guard = sample_reachable_filtered_entry_guards(gs,
+  chosen_guard = first_reachable_filtered_entry_guard(gs,
                                                  rst,
                                                  SAMPLE_EXCLUDE_CONFIRMED |
                                                  SAMPLE_EXCLUDE_PRIMARY |
@@ -2146,7 +2192,7 @@ select_filtered_guard_for_circuit(guard_selection_t *gs,
   chosen_guard->last_tried_to_connect = approx_time();
   *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
   log_info(LD_GUARD, "No primary or confirmed guards available. Selected "
-           "random guard %s for circuit. Will try other guards before "
+           "guard %s for circuit. Will try other guards before "
            "using this circuit.",
            entry_guard_describe(chosen_guard));
   return chosen_guard;
@@ -2187,8 +2233,8 @@ select_entry_guard_for_circuit(guard_selection_t *gs,
   if (chosen_guard)
     return chosen_guard;
 
-  /* "Otherwise, if there is no such entry, select a member at
-      random from {USABLE_FILTERED_GUARDS}." */
+  /* "Otherwise, if there is no such entry, select a member
+   * {USABLE_FILTERED_GUARDS} following the sample ordering" */
   chosen_guard = select_filtered_guard_for_circuit(gs, usage, rst, state_out);
 
   if (chosen_guard == NULL) {
@@ -2218,6 +2264,9 @@ entry_guards_note_guard_failure(guard_selection_t *gs,
   if (guard->failing_since == 0)
     guard->failing_since = approx_time();
 
+  /* This guard not reachable: send GUARD DOWN event */
+  control_event_guard(guard->nickname, guard->identity, "DOWN");
+
   log_info(LD_GUARD, "Recorded failure for %s%sguard %s",
            guard->is_primary?"primary ":"",
            guard->confirmed_idx>=0?"confirmed ":"",
@@ -2243,6 +2292,11 @@ entry_guards_note_guard_success(guard_selection_t *gs,
   const time_t last_time_on_internet = gs->last_time_on_internet;
   gs->last_time_on_internet = approx_time();
 
+  /* If guard was not already marked as reachable, send a GUARD UP signal */
+  if (guard->is_reachable != GUARD_REACHABLE_YES) {
+    control_event_guard(guard->nickname, guard->identity, "UP");
+  }
+
   guard->is_reachable = GUARD_REACHABLE_YES;
   guard->failing_since = 0;
   guard->is_pending = 0;
@@ -2771,10 +2825,12 @@ entry_guards_update_all(guard_selection_t *gs)
 
 /**
  * Return a newly allocated string for encoding the persistent parts of
- * <b>guard</b> to the state file.
+ * <b>guard</b> to the state file. <b>dense_sampled_idx</b> refers to the
+ * sampled_idx made dense for this <b>guard</b>. Encoding all guards should
+ * lead to a dense array of sampled_idx in the state file.
  */
 STATIC char *
-entry_guard_encode_for_state(entry_guard_t *guard)
+entry_guard_encode_for_state(entry_guard_t *guard, int dense_sampled_idx)
 {
   /*
    * The meta-format we use is K=V K=V K=V... where K can be any
@@ -2803,7 +2859,8 @@ entry_guard_encode_for_state(entry_guard_t *guard)
 
   format_iso_time_nospace(tbuf, guard->sampled_on_date);
   smartlist_add_asprintf(result, "sampled_on=%s", tbuf);
-
+  // Replacing the sampled_idx by dense array
+  smartlist_add_asprintf(result, "sampled_idx=%d", dense_sampled_idx);
   if (guard->sampled_by_version) {
     smartlist_add_asprintf(result, "sampled_by=%s",
                            guard->sampled_by_version);
@@ -2859,6 +2916,78 @@ entry_guard_encode_for_state(entry_guard_t *guard)
 }
 
 /**
+ * Extract key=val from the state string <b>s</b> and duplicate the value to
+ * some string target declared in entry_guard_parse_from_state
+ */
+static void
+parse_from_state_set_vals(const char *s, smartlist_t *entries, smartlist_t
+    *extra, strmap_t *vals)
+{
+    smartlist_split_string(entries, s, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
+      const char *eq = strchr(entry, '=');
+      if (!eq) {
+        smartlist_add(extra, entry);
+        continue;
+      }
+      char *key = tor_strndup(entry, eq-entry);
+      char **target = strmap_get(vals, key);
+      if (target == NULL || *target != NULL) {
+        /* unrecognized or already set */
+        smartlist_add(extra, entry);
+        tor_free(key);
+        continue;
+      }
+
+      *target = tor_strdup(eq+1);
+      tor_free(key);
+      tor_free(entry);
+    } SMARTLIST_FOREACH_END(entry);
+}
+
+/**
+ * Handle part of the parsing state file logic, focused on time related things
+ */
+static void
+parse_from_state_handle_time(entry_guard_t *guard, char *sampled_on, char
+    *unlisted_since, char *confirmed_on)
+{
+#define HANDLE_TIME(field) do {                                 \
+    if (field) {                                                \
+      int r = parse_iso_time_nospace(field, &field ## _time);   \
+      if (r < 0) {                                              \
+        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
+                 #field, escaped(field));                       \
+        field##_time = -1;                                      \
+      }                                                         \
+    }                                                           \
+  } while (0)
+
+  time_t sampled_on_time = 0;
+  time_t unlisted_since_time = 0;
+  time_t confirmed_on_time = 0;
+
+  HANDLE_TIME(sampled_on);
+  HANDLE_TIME(unlisted_since);
+  HANDLE_TIME(confirmed_on);
+
+  if (sampled_on_time <= 0)
+    sampled_on_time = approx_time();
+  if (unlisted_since_time < 0)
+    unlisted_since_time = 0;
+  if (confirmed_on_time < 0)
+    confirmed_on_time = 0;
+
+  #undef HANDLE_TIME
+
+  guard->sampled_on_date = sampled_on_time;
+  guard->unlisted_since_date = unlisted_since_time;
+  guard->confirmed_on_date = confirmed_on_time;
+}
+
+/**
  * Given a string generated by entry_guard_encode_for_state(), parse it
  * (if possible) and return an entry_guard_t object for it.  Return NULL
  * on complete failure.
@@ -2874,6 +3003,7 @@ entry_guard_parse_from_state(const char *s)
   char *rsa_id = NULL;
   char *nickname = NULL;
   char *sampled_on = NULL;
+  char *sampled_idx = NULL;
   char *sampled_by = NULL;
   char *unlisted_since = NULL;
   char *listed  = NULL;
@@ -2890,6 +3020,7 @@ entry_guard_parse_from_state(const char *s)
   char *pb_collapsed_circuits = NULL;
   char *pb_unusable_circuits = NULL;
   char *pb_timeouts = NULL;
+  int invalid_sampled_idx = get_max_sample_size_absolute();
 
   /* Split up the entries.  Put the ones we know about in strings and the
    * rest in "extra". */
@@ -2903,6 +3034,7 @@ entry_guard_parse_from_state(const char *s)
     FIELD(rsa_id);
     FIELD(nickname);
     FIELD(sampled_on);
+    FIELD(sampled_idx);
     FIELD(sampled_by);
     FIELD(unlisted_since);
     FIELD(listed);
@@ -2918,29 +3050,8 @@ entry_guard_parse_from_state(const char *s)
     FIELD(pb_unusable_circuits);
     FIELD(pb_timeouts);
 #undef FIELD
-
-    smartlist_split_string(entries, s, " ",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-
-    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
-      const char *eq = strchr(entry, '=');
-      if (!eq) {
-        smartlist_add(extra, entry);
-        continue;
-      }
-      char *key = tor_strndup(entry, eq-entry);
-      char **target = strmap_get(vals, key);
-      if (target == NULL || *target != NULL) {
-        /* unrecognized or already set */
-        smartlist_add(extra, entry);
-        tor_free(key);
-        continue;
-      }
-
-      *target = tor_strdup(eq+1);
-      tor_free(key);
-      tor_free(entry);
-    } SMARTLIST_FOREACH_END(entry);
+    /* Extract from s the key=val that we recognize, put the others in extra*/
+    parse_from_state_set_vals(s, entries, extra, vals);
 
     smartlist_free(entries);
     strmap_free(vals, NULL);
@@ -2988,43 +3099,12 @@ entry_guard_parse_from_state(const char *s)
   }
 
   /* Process the various time fields. */
-
-#define HANDLE_TIME(field) do {                                 \
-    if (field) {                                                \
-      int r = parse_iso_time_nospace(field, &field ## _time);   \
-      if (r < 0) {                                              \
-        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
-                 #field, escaped(field));                       \
-        field##_time = -1;                                      \
-      }                                                         \
-    }                                                           \
-  } while (0)
-
-  time_t sampled_on_time = 0;
-  time_t unlisted_since_time = 0;
-  time_t confirmed_on_time = 0;
-
-  HANDLE_TIME(sampled_on);
-  HANDLE_TIME(unlisted_since);
-  HANDLE_TIME(confirmed_on);
-
-  if (sampled_on_time <= 0)
-    sampled_on_time = approx_time();
-  if (unlisted_since_time < 0)
-    unlisted_since_time = 0;
-  if (confirmed_on_time < 0)
-    confirmed_on_time = 0;
-
-  #undef HANDLE_TIME
-
-  guard->sampled_on_date = sampled_on_time;
-  guard->unlisted_since_date = unlisted_since_time;
-  guard->confirmed_on_date = confirmed_on_time;
+  parse_from_state_handle_time(guard, sampled_on, unlisted_since,
+      confirmed_on);
 
   /* Take sampled_by_version verbatim. */
   guard->sampled_by_version = sampled_by;
   sampled_by = NULL; /* prevent free */
-
   /* Listed is a boolean */
   if (listed && strcmp(listed, "0"))
     guard->currently_listed = 1;
@@ -3042,6 +3122,29 @@ entry_guard_parse_from_state(const char *s)
     }
   }
 
+  if (sampled_idx) {
+    int ok = 1;
+    long idx = tor_parse_long(sampled_idx, 10, 0, INT_MAX, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_GUARD, "Guard has invalid sampled_idx %s",
+          escaped(sampled_idx));
+      /* set it to a idx higher than the max sample size */
+      guard->sampled_idx = invalid_sampled_idx++;
+    } else {
+      guard->sampled_idx = (int)idx;
+    }
+  } else if (confirmed_idx) {
+    /* This state has been written by an older Tor version which did not have
+     * sample ordering  */
+
+    guard->sampled_idx = guard->confirmed_idx;
+  } else {
+    log_info(LD_GUARD, "The state file seems to be into a status that could"
+             " yield to weird entry node selection: we're missing both a"
+             " sampled_idx and a confirmed_idx.");
+    guard->sampled_idx = invalid_sampled_idx++;
+  }
+
   /* Anything we didn't recognize gets crammed together */
   if (smartlist_len(extra) > 0) {
     guard->extra_state_fields = smartlist_join_strings(extra, " ", 0, NULL);
@@ -3096,6 +3199,7 @@ entry_guard_parse_from_state(const char *s)
   tor_free(listed);
   tor_free(confirmed_on);
   tor_free(confirmed_idx);
+  tor_free(sampled_idx);
   tor_free(bridge_addr);
   tor_free(pb_use_attempts);
   tor_free(pb_use_successes);
@@ -3125,13 +3229,15 @@ entry_guards_update_guards_in_state(or_state_t *state)
   config_line_t **nextline = &lines;
 
   SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    int i = 0;
     SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
       if (guard->is_persistent == 0)
         continue;
       *nextline = tor_malloc_zero(sizeof(config_line_t));
       (*nextline)->key = tor_strdup("Guard");
-      (*nextline)->value = entry_guard_encode_for_state(guard);
+      (*nextline)->value = entry_guard_encode_for_state(guard, i);
       nextline = &(*nextline)->next;
+      i++;
     } SMARTLIST_FOREACH_END(guard);
   } SMARTLIST_FOREACH_END(gs);
 
@@ -3184,6 +3290,14 @@ entry_guards_load_guards_from_state(or_state_t *state, int set)
       tor_assert(gs);
       smartlist_add(gs->sampled_entry_guards, guard);
       guard->in_selection = gs;
+      /* Recompute the next_sampled_id from the state. We do not assume that
+       * sampled guards appear in the correct order within the file, and we
+       * need to know what would be the next sampled idx to give to any
+       * new sampled guard (i.e., max of guard->sampled_idx + 1)*/
+      if (gs->next_sampled_idx <= guard->sampled_idx) {
+        gs->next_sampled_idx = guard->sampled_idx + 1;
+      }
+
     } else {
       entry_guard_free(guard);
     }
@@ -3191,6 +3305,10 @@ entry_guards_load_guards_from_state(or_state_t *state, int set)
 
   if (set) {
     SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      /** Guards should be in sample order within the file, but it is maybe
+       * better NOT to assume that. Let's order them before updating lists
+       */
+      smartlist_sort(gs->sampled_entry_guards, compare_guards_by_sampled_idx);
       entry_guards_update_all(gs);
     } SMARTLIST_FOREACH_END(gs);
   }
diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h
index 4e5eb4e960..4b236dc80c 100644
--- a/src/feature/client/entrynodes.h
+++ b/src/feature/client/entrynodes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 #include "lib/container/handles.h"
 
 /* Forward declare for guard_selection_t; entrynodes.c has the real struct */
-typedef struct guard_selection_s guard_selection_t;
+typedef struct guard_selection_t guard_selection_t;
 
 /* Forward declare for entry_guard_t; the real declaration is private. */
 typedef struct entry_guard_t entry_guard_t;
@@ -28,7 +28,7 @@ typedef struct circuit_guard_state_t circuit_guard_state_t;
    private. */
 typedef struct entry_guard_restriction_t entry_guard_restriction_t;
 
-/* Information about a guard's pathbias status.
+/** Information about a guard's pathbias status.
  * These fields are used in circpathbias.c to try to detect entry
  * nodes that are failing circuits at a suspicious frequency.
  */
@@ -117,6 +117,13 @@ struct entry_guard_t {
    * confirmed guard. */
   time_t confirmed_on_date; /* 0 if not confirmed */
   /**
+   * In what order was this guard sampled? Guards with
+   * lower indices appear earlier on the sampled list, the confirmed list and
+   * the primary list as a result of Prop 310
+   */
+  int sampled_idx;
+
+  /**
    * In what order was this guard confirmed? Guards with lower indices
    * appear earlier on the confirmed list.  If the confirmed list is compacted,
    * this field corresponds to the index of this guard on the confirmed list.
@@ -210,7 +217,7 @@ typedef enum guard_selection_type_t {
  * See the module documentation for entrynodes.c for more information
  * about guard selection algorithms.
  */
-struct guard_selection_s {
+struct guard_selection_t {
   /**
    * The name for this guard-selection object. (Must not contain spaces).
    */
@@ -242,8 +249,9 @@ struct guard_selection_s {
    * Ordered list (from highest to lowest priority) of guards that we
    * have successfully contacted and decided to use. Every member of
    * this list is a member of sampled_entry_guards. Every member should
-   * have confirmed_on_date set, and have confirmed_idx greater than
-   * any earlier member of the list.
+   * have confirmed_on_date set.
+   * The ordering of the list should be by sampled idx. The reasoning behind
+   * it is linked to Proposal 310.
    *
    * This list is persistent. It is a subset of the elements in
    * sampled_entry_guards, and its pointers point to elements of
@@ -271,6 +279,12 @@ struct guard_selection_s {
    * confirmed_entry_guards receive? */
   int next_confirmed_idx;
 
+  /** What sampled_idx value should the next-added member of
+   * sampled_entry_guards receive? This should follow the size of the sampled
+   * list until sampled relays get pruned for some reason
+   */
+  int next_sampled_idx;
+
 };
 
 struct entry_guard_handle_t;
@@ -515,7 +529,8 @@ MOCK_DECL(STATIC circuit_guard_state_t *,
 STATIC entry_guard_t *entry_guard_add_to_sample(guard_selection_t *gs,
                                                 const node_t *node);
 STATIC entry_guard_t *entry_guards_expand_sample(guard_selection_t *gs);
-STATIC char *entry_guard_encode_for_state(entry_guard_t *guard);
+STATIC char *entry_guard_encode_for_state(entry_guard_t *guard, int
+    dense_sampled_index);
 STATIC entry_guard_t *entry_guard_parse_from_state(const char *s);
 #define entry_guard_free(e) \
   FREE_AND_NULL(entry_guard_t, entry_guard_free_, (e))
@@ -523,7 +538,7 @@ STATIC void entry_guard_free_(entry_guard_t *e);
 STATIC void entry_guards_update_filtered_sets(guard_selection_t *gs);
 STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
 /**
- * @name Flags for sample_reachable_filtered_entry_guards()
+ * @name Flags for first_reachable_filtered_entry_guard()
  */
 /**@{*/
 #define SAMPLE_EXCLUDE_CONFIRMED   (1u<<0)
@@ -532,7 +547,7 @@ STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
 #define SAMPLE_NO_UPDATE_PRIMARY   (1u<<3)
 #define SAMPLE_EXCLUDE_NO_DESCRIPTOR (1u<<4)
 /**@}*/
-STATIC entry_guard_t *sample_reachable_filtered_entry_guards(
+STATIC entry_guard_t *first_reachable_filtered_entry_guard(
                                     guard_selection_t *gs,
                                     const entry_guard_restriction_t *rst,
                                     unsigned flags);
diff --git a/src/feature/client/feature_client.md b/src/feature/client/feature_client.md
new file mode 100644
index 0000000000..dd4bf78ec8
--- /dev/null
+++ b/src/feature/client/feature_client.md
@@ -0,0 +1,5 @@
+@dir /feature/client
+@brief feature/client: Client-specific code
+
+(There is also a bunch of client-specific code in other modules.)
+
diff --git a/src/feature/client/include.am b/src/feature/client/include.am
new file mode 100644
index 0000000000..53c9f047d4
--- /dev/null
+++ b/src/feature/client/include.am
@@ -0,0 +1,20 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/client/addressmap.c		\
+	src/feature/client/bridges.c		\
+	src/feature/client/circpathbias.c	\
+	src/feature/client/dnsserv.c		\
+	src/feature/client/entrynodes.c		\
+	src/feature/client/proxymode.c		\
+	src/feature/client/transports.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/client/addressmap.h			\
+	src/feature/client/bridges.h			\
+	src/feature/client/circpathbias.h		\
+	src/feature/client/dnsserv.h			\
+	src/feature/client/entrynodes.h			\
+	src/feature/client/proxymode.h			\
+	src/feature/client/transports.h
diff --git a/src/feature/client/proxymode.c b/src/feature/client/proxymode.c
new file mode 100644
index 0000000000..aa269ec7fb
--- /dev/null
+++ b/src/feature/client/proxymode.c
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file proxymode.c
+ * @brief Determine whether we are trying to be a proxy.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/client/proxymode.h"
+
+/** Return true iff we are trying to proxy client connections. */
+int
+proxy_mode(const or_options_t *options)
+{
+  (void)options;
+  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
+    if (p->type == CONN_TYPE_AP_LISTENER ||
+        p->type == CONN_TYPE_AP_TRANS_LISTENER ||
+        p->type == CONN_TYPE_AP_DNS_LISTENER ||
+        p->type == CONN_TYPE_AP_NATD_LISTENER)
+      return 1;
+  } SMARTLIST_FOREACH_END(p);
+  return 0;
+}
diff --git a/src/feature/client/proxymode.h b/src/feature/client/proxymode.h
new file mode 100644
index 0000000000..30be08ff78
--- /dev/null
+++ b/src/feature/client/proxymode.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file proxymode.h
+ * \brief Header file for proxymode.c.
+ **/
+
+#ifndef TOR_PROXYMODE_H
+#define TOR_PROXYMODE_H
+
+int proxy_mode(const or_options_t *options);
+
+#endif /* !defined(TOR_PROXYMODE_H) */
diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c
index 0d1342c87b..2bdc0ae151 100644
--- a/src/feature/client/transports.c
+++ b/src/feature/client/transports.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -97,15 +97,20 @@
 #include "core/or/circuitbuild.h"
 #include "feature/client/transports.h"
 #include "feature/relay/router.h"
+#include "feature/relay/relay_find_addr.h"
+/* 31851: split the server transport code out of the client module */
+#include "feature/relay/transport_config.h"
 #include "app/config/statefile.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
+#include "lib/process/process.h"
 #include "lib/process/env.h"
-#include "lib/process/subprocess.h"
 
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp);
 
 static inline int proxy_configuration_finished(const managed_proxy_t *mp);
@@ -127,6 +132,8 @@ static void parse_method_error(const char *line, int is_server_method);
 #define PROTO_SMETHODS_DONE "SMETHODS DONE"
 #define PROTO_PROXY_DONE "PROXY DONE"
 #define PROTO_PROXY_ERROR "PROXY-ERROR"
+#define PROTO_LOG "LOG"
+#define PROTO_STATUS "STATUS"
 
 /** The first and only supported - at the moment - configuration
     protocol version. */
@@ -490,8 +497,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
   tor_assert(mp->conf_state == PT_PROTO_COMPLETED);
 
   /* destroy the process handle and terminate the process. */
-  tor_process_handle_destroy(mp->process_handle, 1);
-  mp->process_handle = NULL;
+  process_set_data(mp->process, NULL);
+  process_terminate(mp->process);
 
   /* destroy all its registered transports, since we will no longer
      use them. */
@@ -520,34 +527,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
 static int
 launch_managed_proxy(managed_proxy_t *mp)
 {
-  int retval;
-
-  process_environment_t *env = create_managed_proxy_environment(mp);
-
-#ifdef _WIN32
-  /* Passing NULL as lpApplicationName makes Windows search for the .exe */
-  retval = tor_spawn_background(NULL,
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#else /* !(defined(_WIN32)) */
-  retval = tor_spawn_background(mp->argv[0],
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#endif /* defined(_WIN32) */
-
-  process_environment_free(env);
-
-  if (retval == PROCESS_STATUS_ERROR) {
-    log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.",
+  tor_assert(mp);
+
+  smartlist_t *env = create_managed_proxy_environment(mp);
+
+  /* Configure our process. */
+  process_set_data(mp->process, mp);
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback);
+  process_set_exit_callback(mp->process, managed_proxy_exit_callback);
+  process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE);
+  process_reset_environment(mp->process, env);
+
+  /* Cleanup our env. */
+  SMARTLIST_FOREACH(env, char *, x, tor_free(x));
+  smartlist_free(env);
+
+  /* Skip the argv[0] as we get that from process_new(argv[0]). */
+  for (int i = 1; mp->argv[i] != NULL; ++i)
+    process_append_argument(mp->process, mp->argv[i]);
+
+  if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.",
              mp->argv[0]);
     return -1;
   }
 
-  log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.",
-           mp->argv[0], tor_process_get_pid(mp->process_handle));
-
+  log_info(LD_CONFIG,
+           "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.",
+           mp->argv[0], process_get_pid(mp->process));
   mp->conf_state = PT_PROTO_LAUNCHED;
 
   return 0;
@@ -615,10 +623,6 @@ pt_configure_remaining_proxies(void)
 STATIC int
 configure_proxy(managed_proxy_t *mp)
 {
-  int configuration_finished = 0;
-  smartlist_t *proxy_output = NULL;
-  enum stream_status stream_status = 0;
-
   /* if we haven't launched the proxy yet, do it now */
   if (mp->conf_state == PT_PROTO_INFANT) {
     if (launch_managed_proxy(mp) < 0) { /* launch fail */
@@ -629,45 +633,8 @@ configure_proxy(managed_proxy_t *mp)
   }
 
   tor_assert(mp->conf_state != PT_PROTO_INFANT);
-  tor_assert(mp->process_handle);
-
-  proxy_output =
-    tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle),
-                              &stream_status);
-  if (!proxy_output) { /* failed to get input from proxy */
-    if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */
-      mp->conf_state = PT_PROTO_BROKEN;
-      log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' "
-               "is '%s'. Most probably the managed proxy stopped running. "
-               "This might be a bug of the managed proxy, a bug of Tor, or "
-               "a misconfiguration. Please enable logging on your managed "
-               "proxy and check the logs for errors.",
-               mp->argv[0], stream_status_to_string(stream_status));
-    }
-
-    goto done;
-  }
-
-  /* Handle lines. */
-  SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) {
-    handle_proxy_line(line, mp);
-    if (proxy_configuration_finished(mp))
-      goto done;
-  } SMARTLIST_FOREACH_END(line);
-
- done:
-  /* if the proxy finished configuring, exit the loop. */
-  if (proxy_configuration_finished(mp)) {
-    handle_finished_proxy(mp);
-    configuration_finished = 1;
-  }
-
-  if (proxy_output) {
-    SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp));
-    smartlist_free(proxy_output);
-  }
-
-  return configuration_finished;
+  tor_assert(mp->process);
+  return mp->conf_state == PT_PROTO_COMPLETED;
 }
 
 /** Register server managed proxy <b>mp</b> transports to state */
@@ -748,8 +715,14 @@ managed_proxy_destroy(managed_proxy_t *mp,
   /* free the outgoing proxy URI */
   tor_free(mp->proxy_uri);
 
-  tor_process_handle_destroy(mp->process_handle, also_terminate_process);
-  mp->process_handle = NULL;
+  /* do we want to terminate our process if it's still running? */
+  if (also_terminate_process && mp->process) {
+    /* Note that we do not call process_free(mp->process) here because we let
+     * the exit handler in managed_proxy_exit_callback() return `true` which
+     * makes the process subsystem deallocate the process_t. */
+    process_set_data(mp->process, NULL);
+    process_terminate(mp->process);
+  }
 
   tor_free(mp);
 }
@@ -763,6 +736,9 @@ get_pt_proxy_uri(void)
   const or_options_t *options = get_options();
   char *uri = NULL;
 
+  /* XXX: Currently TCPProxy is not supported in TOR_PT_PROXY because
+   * there isn't a standard URI scheme for some proxy protocols, such as
+   * haproxy. */
   if (options->Socks4Proxy || options->Socks5Proxy || options->HTTPSProxy) {
     char addr[TOR_ADDR_BUF_LEN+1];
 
@@ -945,21 +921,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp)
 
     parse_proxy_error(line);
     goto err;
-  } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) {
-    /* managed proxy launch failed: parse error message to learn why. */
-    int retval, child_state, saved_errno;
-    retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x",
-                        &child_state, &saved_errno);
-    if (retval == 2) {
-      log_warn(LD_GENERAL,
-               "Could not launch managed proxy executable at '%s' ('%s').",
-               mp->argv[0], strerror(saved_errno));
-    } else { /* failed to parse error message */
-      log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.",
-               mp->argv[0]);
-    }
 
-    mp->conf_state = PT_PROTO_FAILED_LAUNCH;
+    /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS
+     * string to make sure we can later extend this big if/else-if table with
+     * something that begins with "LOG" without having to get the order right.
+     * */
+  } else if (!strcmpstart(line, PROTO_LOG " ")) {
+    parse_log_line(line, mp);
+    return;
+  } else if (!strcmpstart(line, PROTO_STATUS " ")) {
+    parse_status_line(line, mp);
     return;
   }
 
@@ -1182,6 +1153,121 @@ parse_proxy_error(const char *line)
            line+strlen(PROTO_PROXY_ERROR)+1);
 }
 
+/** Parses a LOG <b>line</b> and emit log events accordingly. */
+STATIC void
+parse_log_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *log_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_LOG) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_LOG);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_LOG) + 1;
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s",
+             mp->argv[0], data);
+    goto done;
+  }
+
+  const config_line_t *severity = config_line_find(values, "SEVERITY");
+  const config_line_t *message = config_line_find(values, "MESSAGE");
+
+  /* Check if we got a message. */
+  if (! message) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "MESSAGE: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Check if severity is there and whether it's valid. */
+  if (! severity) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "SEVERITY: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  int log_severity = managed_proxy_severity_parse(severity->value);
+
+  if (log_severity == -1) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an "
+                    "invalid severity level: %s",
+                    mp->argv[0], severity->value);
+    goto done;
+  }
+
+  tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s",
+          mp->argv[0], message->value);
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  log_message = kvline_encode(values, KV_QUOTED);
+
+  /* Emit control port event. */
+  control_event_pt_log(log_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(log_message);
+}
+
+/** Parses a STATUS <b>line</b> and emit control events accordingly. */
+STATIC void
+parse_status_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *status_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_STATUS) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_STATUS);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_STATUS) + 1;
+
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid "
+             "STATUS message: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* We check if we received the TRANSPORT parameter, which is the only
+   * *required* value. */
+  const config_line_t *type = config_line_find(values, "TRANSPORT");
+
+  if (! type) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without "
+                    "TRANSPORT: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  status_message = kvline_encode(values, KV_QUOTED);
+
+  /* We have checked that TRANSPORT is there, we can now emit the STATUS event
+   * via the control port. */
+  control_event_pt_status(status_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(status_message);
+}
+
 /** Return a newly allocated string that tor should place in
  * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server
  * manged proxy in <b>mp</b>. Return NULL if no such options are found. */
@@ -1199,7 +1285,7 @@ get_transport_options_for_server_proxy(const managed_proxy_t *mp)
       string. */
   SMARTLIST_FOREACH_BEGIN(mp->transports_to_launch, const char *, transport) {
     smartlist_t *options_tmp_sl = NULL;
-    options_tmp_sl = get_options_for_server_transport(transport);
+    options_tmp_sl = pt_get_options_for_server_transport(transport);
     if (!options_tmp_sl)
       continue;
 
@@ -1257,7 +1343,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp)
 
 /** Return a newly allocated process_environment_t * for <b>mp</b>'s
  * process. */
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp)
 {
   const or_options_t *options = get_options();
@@ -1272,8 +1358,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
   /* The final environment to be passed to mp. */
   smartlist_t *merged_env_vars = get_current_process_environment_variables();
 
-  process_environment_t *env;
-
   {
     char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */
     smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp);
@@ -1337,8 +1421,10 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
         smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=%s",
                                ext_or_addrport_tmp);
       }
-      smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s",
-                             cookie_file_loc);
+      if (cookie_file_loc) {
+        smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s",
+                               cookie_file_loc);
+      }
 
       tor_free(ext_or_addrport_tmp);
       tor_free(cookie_file_loc);
@@ -1346,11 +1432,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     } else {
       smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=");
     }
-
-    /* All new versions of tor will keep stdin open, so PTs can use it
-     * as a reliable termination detection mechanism.
-     */
-    smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
   } else {
     /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the
      * TOR_PT_PROXY line.
@@ -1361,19 +1442,19 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     }
   }
 
+  /* All new versions of tor will keep stdin open, so PTs can use it
+   * as a reliable termination detection mechanism.
+   */
+  smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
+
   SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) {
     set_environment_variable_in_smartlist(merged_env_vars, env_var,
                                           tor_free_, 1);
   } SMARTLIST_FOREACH_END(env_var);
 
-  env = process_environment_make(merged_env_vars);
-
   smartlist_free(envs);
 
-  SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x));
-  smartlist_free(merged_env_vars);
-
-  return env;
+  return merged_env_vars;
 }
 
 /** Create and return a new managed proxy for <b>transport</b> using
@@ -1392,6 +1473,7 @@ managed_proxy_create(const smartlist_t *with_transport_list,
   mp->argv = proxy_argv;
   mp->transports = smartlist_new();
   mp->proxy_uri = get_pt_proxy_uri();
+  mp->process = process_new(proxy_argv[0]);
 
   mp->transports_to_launch = smartlist_new();
   SMARTLIST_FOREACH(with_transport_list, const char *, transport,
@@ -1736,3 +1818,95 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape)
 
   return new_string;
 }
+
+/** Callback function that is called when our PT process have data on its
+ * stdout. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stdout_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (mp == NULL)
+    return;
+
+  handle_proxy_line(line, mp);
+
+  if (proxy_configuration_finished(mp))
+    handle_finished_proxy(mp);
+}
+
+/** Callback function that is called when our PT process have data on its
+ * stderr. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stderr_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (BUG(mp == NULL))
+    return;
+
+  log_info(LD_PT,
+           "Managed proxy at '%s' reported via standard error: %s",
+           mp->argv[0], line);
+}
+
+/** Callback function that is called when our PT process terminates. The
+ * process exit code can be found in <b>exit_code</b> and our process can be
+ * found in <b>process</b>. Returns true iff we want the process subsystem to
+ * free our process_t handle for us. */
+STATIC bool
+managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_warn(LD_PT,
+          "Pluggable Transport process terminated with status code %" PRIu64,
+          exit_code);
+
+  /* Returning true here means that the process subsystem will take care of
+   * calling process_free() on our process_t. */
+  return true;
+}
+
+/** Returns a valid integer log severity level from <b>severity</b> that
+ * is compatible with Tor's logging functions. Returns <b>-1</b> on
+ * error. */
+STATIC int
+managed_proxy_severity_parse(const char *severity)
+{
+  tor_assert(severity);
+
+  /* Slightly different than log.c's parse_log_level :-( */
+  if (! strcmp(severity, "debug"))
+    return LOG_DEBUG;
+
+  if (! strcmp(severity, "info"))
+    return LOG_INFO;
+
+  if (! strcmp(severity, "notice"))
+    return LOG_NOTICE;
+
+  if (! strcmp(severity, "warning"))
+    return LOG_WARN;
+
+  if (! strcmp(severity, "error"))
+    return LOG_ERR;
+
+  return -1;
+}
diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h
index e2fa45828f..1ed942c175 100644
--- a/src/feature/client/transports.h
+++ b/src/feature/client/transports.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,8 @@
 #ifndef TOR_TRANSPORTS_H
 #define TOR_TRANSPORTS_H
 
+#include "lib/process/process.h"
+
 /** Represents a pluggable transport used by a bridge. */
 typedef struct transport_t {
   /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */
@@ -81,7 +83,7 @@ enum pt_proto_state {
   PT_PROTO_FAILED_LAUNCH /* failed while launching */
 };
 
-struct process_handle_t;
+struct process_t;
 
 /** Structure containing information of a managed proxy. */
 typedef struct {
@@ -94,10 +96,8 @@ typedef struct {
 
   int is_server; /* is it a server proxy? */
 
-  /* A pointer to the process handle of this managed proxy. */
-  struct process_handle_t *process_handle;
-
-  int pid; /* The Process ID this managed proxy is using. */
+  /* A pointer to the process of this managed proxy. */
+  struct process_t *process;
 
   /** Boolean: We are re-parsing our config, and we are going to
    * remove this managed proxy if we don't find it any transport
@@ -128,6 +128,8 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp);
 STATIC void parse_env_error(const char *line);
 STATIC void parse_proxy_error(const char *line);
 STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_log_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_status_line(const char *line, managed_proxy_t *mp);
 STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp);
 
 STATIC void managed_proxy_destroy(managed_proxy_t *mp,
@@ -142,6 +144,12 @@ STATIC char* get_pt_proxy_uri(void);
 
 STATIC void free_execve_args(char **arg);
 
+STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t);
+STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t);
+STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t);
+
+STATIC int managed_proxy_severity_parse(const char *);
+
 #endif /* defined(PT_PRIVATE) */
 
 #endif /* !defined(TOR_TRANSPORTS_H) */
diff --git a/src/feature/control/.may_include b/src/feature/control/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/control/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c
new file mode 100644
index 0000000000..405630ecd4
--- /dev/null
+++ b/src/feature/control/btrack.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack.c
+ * \brief Bootstrap trackers
+ *
+ * Initializes and shuts down the specific bootstrap trackers.  These
+ * trackers help the reporting of bootstrap progress by maintaining
+ * state information about various subsystems within tor.  When the
+ * correct state changes happen, these trackers emit controller
+ * events.
+ *
+ * These trackers avoid referring directly to the internals of state
+ * objects of other subsystems.
+ *
+ * btrack_circuit.c contains the tracker for origin circuits.
+ *
+ * btrack_orconn.c contains the tracker for OR connections.
+ *
+ * Eventually there will be a tracker for directory downloads as well.
+ **/
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_sys.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+static int
+btrack_init(void)
+{
+  if (btrack_orconn_init())
+    return -1;
+
+  return 0;
+}
+
+static void
+btrack_fini(void)
+{
+  btrack_orconn_fini();
+  btrack_circ_fini();
+}
+
+static int
+btrack_add_pubsub(pubsub_connector_t *connector)
+{
+  if (btrack_orconn_add_pubsub(connector))
+    return -1;
+  if (btrack_circ_add_pubsub(connector))
+    return -1;
+
+  return 0;
+}
+
+const subsys_fns_t sys_btrack = {
+  .name = "btrack",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = 55,
+  .initialize = btrack_init,
+  .shutdown = btrack_fini,
+  .add_pubsub = btrack_add_pubsub,
+};
diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c
new file mode 100644
index 0000000000..be51b51046
--- /dev/null
+++ b/src/feature/control/btrack_circuit.c
@@ -0,0 +1,166 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.c
+ * \brief Bootstrap tracker for origin circuits
+ *
+ * Track state changes of origin circuits, as published by the circuit
+ * subsystem.
+ **/
+
+#include "core/or/or.h"
+
+#include "core/or/ocirc_event.h"
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/control.h"
+#include "lib/log/log.h"
+
+/** Pair of a best origin circuit GID with its state or status */
+typedef struct btc_best_t {
+  uint32_t gid;
+  int val;
+} btc_best_t;
+
+/** GID and state of the best origin circuit we've seen so far */
+static btc_best_t best_any_state = { 0, -1 };
+/** GID and state of the best application circuit we've seen so far */
+static btc_best_t best_ap_state = { 0, -1 };
+/** GID and status of the best origin circuit we've seen so far */
+static btc_best_t best_any_evtype = { 0, -1 };
+/** GID and status of the best application circuit we've seen so far */
+static btc_best_t best_ap_evtype = { 0, -1 };
+
+/** Reset cached "best" values */
+static void
+btc_reset_bests(void)
+{
+  best_any_state.gid = best_ap_state.gid = 0;
+  best_any_state.val = best_ap_state.val = -1;
+  best_any_evtype.gid = best_ap_state.gid = 0;
+  best_any_evtype.val = best_ap_evtype.val = -1;
+}
+
+/** True if @a state is a "better" origin circuit state than @a best->val */
+static bool
+btc_state_better(int state, const btc_best_t *best)
+{
+  return state > best->val;
+}
+
+/**
+ * Definine an ordering on circuit status events
+ *
+ * The CIRC_EVENT_ constants aren't sorted in a useful order, so this
+ * array helps to decode them.  This approach depends on the statuses
+ * being nonnegative and dense.
+ **/
+static int circ_event_order[] = {
+  [CIRC_EVENT_FAILED] = -1,
+  [CIRC_EVENT_CLOSED] = -1,
+  [CIRC_EVENT_LAUNCHED] = 1,
+  [CIRC_EVENT_EXTENDED] = 2,
+  [CIRC_EVENT_BUILT] = 3,
+};
+#define N_CIRC_EVENT_ORDER \
+  (sizeof(circ_event_order) / sizeof(circ_event_order[0]))
+
+/** True if @a state is a "better" origin circuit event status than @a
+    best->val */
+static bool
+btc_evtype_better(int state, const btc_best_t *best)
+{
+  if (state < 0)
+    return false;
+  if (best->val < 0)
+    return true;
+
+  tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER);
+  tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER);
+  return circ_event_order[state] > circ_event_order[best->val];
+}
+
+static bool
+btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best,
+                 const char *type)
+{
+  if (btc_state_better(msg->state, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type,
+             best->val, msg->state, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->state;
+    return true;
+  }
+  return false;
+}
+
+static bool
+btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best,
+                  const char *type)
+{
+  if (btc_evtype_better(msg->evtype, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type,
+             best->val, msg->evtype, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->evtype;
+    return true;
+  }
+  return false;
+}
+
+DECLARE_SUBSCRIBE(ocirc_state, btc_state_rcvr);
+DECLARE_SUBSCRIBE(ocirc_cevent, btc_cevent_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, btc_chan_rcvr);
+
+static void
+btc_state_rcvr(const msg_t *msg, const ocirc_state_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d",
+            arg->gid, arg->state, arg->onehop);
+
+  btc_update_state(arg, &best_any_state, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_state(arg, &best_ap_state, "AP");
+}
+
+static void
+btc_cevent_rcvr(const msg_t *msg, const ocirc_cevent_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d",
+            arg->gid, arg->evtype, arg->reason, arg->onehop);
+
+  btc_update_evtype(arg, &best_any_evtype, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_evtype(arg, &best_ap_evtype, "AP");
+}
+
+static void
+btc_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d",
+            arg->gid, arg->chan, arg->onehop);
+}
+
+int
+btrack_circ_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_state))
+    return -1;
+  return 0;
+}
+
+void
+btrack_circ_fini(void)
+{
+  btc_reset_bests();
+}
diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h
new file mode 100644
index 0000000000..75699450c3
--- /dev/null
+++ b/src/feature/control/btrack_circuit.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.h
+ * \brief Header file for btrack_circuit.c
+ **/
+
+#ifndef TOR_BTRACK_CIRCUIT_H
+#define TOR_BTRACK_CIRCUIT_H
+
+#include "lib/pubsub/pubsub.h"
+
+int btrack_circ_init(void);
+void btrack_circ_fini(void);
+int btrack_circ_add_pubsub(pubsub_connector_t *);
+
+#endif /* !defined(TOR_BTRACK_CIRCUIT_H) */
diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c
new file mode 100644
index 0000000000..104c8af230
--- /dev/null
+++ b/src/feature/control/btrack_orconn.c
@@ -0,0 +1,206 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.c
+ * \brief Bootstrap tracker for OR connections
+ *
+ * Track state changes of OR connections, as published by the
+ * connection subsystem.  Also track circuit launch events, because
+ * they're one of the few ways to discover the association between a
+ * channel (and OR connection) and a circuit.
+ *
+ * We track all OR connections that we receive events for, whether or
+ * not they're carrying origin circuits.  (An OR connection might
+ * carry origin circuits only after we first find out about that
+ * connection.)
+ *
+ * All origin ORCONN events update the "any" state variables, while
+ * only application ORCONN events update the "ap" state variables (and
+ * also update the "any") variables.
+ *
+ * We do this because we want to report the first increments of
+ * connection progress as the earliest bootstrap phases.  This results
+ * in a better user experience because failures here translate into
+ * zero or very small amounts of displayed progress, instead of
+ * progress stuck near completion.  The first connection to a relay
+ * might be a one-hop circuit for directory lookups, or it might be a
+ * connection for an application circuit because we already have
+ * enough directory info to build an application circuit.
+ *
+ * We call functions in btrack_orconn_cevent.c to generate the actual
+ * controller events, because some of the state decoding we need to do
+ * is complicated.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+#include "lib/pubsub/pubsub.h"
+
+DECLARE_SUBSCRIBE(orconn_state, bto_state_rcvr);
+DECLARE_SUBSCRIBE(orconn_status, bto_status_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, bto_chan_rcvr);
+
+/** Pair of a best ORCONN GID and with its state */
+typedef struct bto_best_t {
+  uint64_t gid;
+  int state;
+} bto_best_t;
+
+/** GID and state of the best ORCONN we've seen so far */
+static bto_best_t best_any = { 0, -1 };
+/** GID and state of the best application circuit ORCONN we've seen so far */
+static bto_best_t best_ap = { 0, -1 };
+
+/**
+ * Update a cached state of a best ORCONN progress we've seen so far.
+ *
+ * Return true if the new state is better than the old.
+ **/
+static bool
+bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type)
+{
+  if (bto->state < best->state)
+    return false;
+  /* Update even if we won't change best->state, because it's more
+   * recent information that a particular connection transitioned to
+   * that state. */
+  best->gid = bto->gid;
+  if (bto->state > best->state) {
+    log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type,
+             best->state, bto->state, bto->gid);
+    best->state = bto->state;
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Update cached states of best ORCONN progress we've seen
+ *
+ * Only update the application ORCONN state if we know it's carrying
+ * an application circuit.
+ **/
+static void
+bto_update_bests(const bt_orconn_t *bto)
+{
+  tor_assert(bto->is_orig);
+
+  if (bto_update_best(bto, &best_any, "ANY"))
+    bto_cevent_anyconn(bto);
+  if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP"))
+    bto_cevent_apconn(bto);
+}
+
+/** Reset cached "best" values */
+static void
+bto_reset_bests(void)
+{
+  best_any.gid = best_ap.gid = 0;
+  best_any.state = best_ap.state = -1;
+}
+
+/**
+ * Update cached states of ORCONNs from the incoming message.  This
+ * message comes from code in connection_or.c.
+ **/
+static void
+bto_state_rcvr(const msg_t *msg, const orconn_state_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(arg->gid, arg->chan);
+  log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64
+            " proxy_type=%d state=%d",
+            arg->gid, arg->chan, arg->proxy_type, arg->state);
+  bto->proxy_type = arg->proxy_type;
+  bto->state = arg->state;
+  if (bto->is_orig)
+    bto_update_bests(bto);
+}
+
+/**
+ * Delete a cached ORCONN state if we get an incoming message saying
+ * the ORCONN is failed or closed.  This message comes from code in
+ * control.c.
+ **/
+static void
+bto_status_rcvr(const msg_t *msg, const orconn_status_msg_t *arg)
+{
+  (void)msg;
+  switch (arg->status) {
+  case OR_CONN_EVENT_FAILED:
+  case OR_CONN_EVENT_CLOSED:
+    log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d",
+             arg->gid, arg->status, arg->reason);
+    return bto_delete(arg->gid);
+  default:
+    break;
+  }
+}
+
+/**
+ * Create or update a cached ORCONN state for a newly launched
+ * connection, including whether it's launched by an origin circuit
+ * and whether it's a one-hop circuit.
+ **/
+static void
+bto_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(0, arg->chan);
+  if (!bto->is_orig || (bto->is_onehop && !arg->onehop)) {
+    log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d",
+              arg->chan, arg->onehop);
+  }
+  bto->is_orig = true;
+  if (!arg->onehop)
+    bto->is_onehop = false;
+  bto_update_bests(bto);
+}
+
+/**
+ * Initialize the hash maps and subscribe to ORCONN and origin
+ * circuit events.
+ **/
+int
+btrack_orconn_init(void)
+{
+  bto_init_maps();
+
+  return 0;
+}
+
+int
+btrack_orconn_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_state))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_status))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  return 0;
+}
+
+/** Clear the hash maps and reset the "best" states */
+void
+btrack_orconn_fini(void)
+{
+  bto_clear_maps();
+  bto_reset_bests();
+  bto_cevent_reset();
+}
diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h
new file mode 100644
index 0000000000..8b3d8be37d
--- /dev/null
+++ b/src/feature/control/btrack_orconn.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.h
+ * \brief Header file for btrack_orconn.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_H
+#define TOR_BTRACK_ORCONN_H
+
+#include "lib/pubsub/pubsub.h"
+
+#ifdef BTRACK_ORCONN_PRIVATE
+
+#include "ht.h"
+
+/**
+ * Structure for tracking OR connection states
+ *
+ * This gets linked into two hash maps: one with connection IDs, and
+ * another with channel IDs.
+ **/
+typedef struct bt_orconn_t {
+  HT_ENTRY(bt_orconn_t) node;   /**< Hash map entry indexed by gid */
+  HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */
+  uint64_t gid;                    /**< Global ID of this ORCONN */
+  uint64_t chan;                   /**< Channel ID, if known */
+  int proxy_type;                  /**< Proxy type */
+  uint8_t state;                   /**< State of this ORCONN */
+  bool is_orig;             /**< Does this carry an origin circuit? */
+  bool is_onehop;           /**< Is this for a one-hop circuit? */
+} bt_orconn_t;
+
+#endif /* defined(BTRACK_ORCONN_PRIVATE) */
+
+int btrack_orconn_init(void);
+int btrack_orconn_add_pubsub(pubsub_connector_t *);
+void btrack_orconn_fini(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_H) */
diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c
new file mode 100644
index 0000000000..d11be59280
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.c
@@ -0,0 +1,161 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.c
+ * \brief Emit bootstrap status events for OR connections
+ *
+ * We do some decoding of the raw OR_CONN_STATE_* values.  For
+ * example, OR_CONN_STATE_CONNECTING means the first TCP connect()
+ * completing, regardless of whether it's directly to a relay instead
+ * of a proxy or a PT.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/control_events.h"
+
+/**
+ * Have we completed our first OR connection?
+ *
+ * Block display of application circuit progress until we do, to avoid
+ * some misleading behavior of jumping to high progress.
+ **/
+static bool bto_first_orconn = false;
+
+/** Is the ORCONN using a pluggable transport? */
+static bool
+using_pt(const bt_orconn_t *bto)
+{
+  return bto->proxy_type == PROXY_PLUGGABLE;
+}
+
+/** Is the ORCONN using a non-PT proxy? */
+static bool
+using_proxy(const bt_orconn_t *bto)
+{
+  switch (bto->proxy_type) {
+  case PROXY_CONNECT:
+  case PROXY_SOCKS4:
+  case PROXY_SOCKS5:
+  case PROXY_HAPROXY:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_anyconn(const bt_orconn_t *bto)
+{
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0);
+    /* Unblock directory progress display */
+    control_event_boot_first_orconn();
+    /* Unblock apconn progress display */
+    bto_first_orconn = true;
+    break;
+  default:
+    break;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any application circuit OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_apconn(const bt_orconn_t *bto)
+{
+  if (!bto_first_orconn)
+    return;
+
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0);
+    break;
+  default:
+    break;
+  }
+}
+
+/** Forget that we completed our first OR connection */
+void
+bto_cevent_reset(void)
+{
+  bto_first_orconn = false;
+}
diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h
new file mode 100644
index 0000000000..8b2207721e
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.h
+ * \brief Header file for btrack_orconn_cevent.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_CEVENT_H
+#define TOR_BTRACK_ORCONN_CEVENT_H
+
+#include "feature/control/btrack_orconn.h"
+
+void bto_cevent_anyconn(const bt_orconn_t *);
+void bto_cevent_apconn(const bt_orconn_t *);
+void bto_cevent_reset(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_CEVENT_H) */
diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c
new file mode 100644
index 0000000000..a60dffb8c4
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.c
@@ -0,0 +1,224 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.c
+ * \brief Hash map implementation for btrack_orconn.c
+ *
+ * These functions manipulate the hash maps that contain bt_orconn
+ * objects.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+
+static inline unsigned int
+bto_gid_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid));
+}
+
+static inline int
+bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->gid == b->gid;
+}
+
+static inline unsigned int
+bto_chan_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan));
+}
+
+static inline int
+bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->chan == b->chan;
+}
+
+HT_HEAD(bto_gid_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_);
+HT_GENERATE2(bto_gid_ht, bt_orconn_t, node,
+             bto_gid_hash_, bto_gid_eq_, 0.6,
+             tor_reallocarray_, tor_free_);
+static struct bto_gid_ht *bto_gid_map;
+
+HT_HEAD(bto_chan_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_,
+             bto_chan_eq_);
+HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node,
+             bto_chan_hash_, bto_chan_eq_, 0.6,
+             tor_reallocarray_, tor_free_);
+static struct bto_chan_ht *bto_chan_map;
+
+/** Clear the GID hash map, freeing any bt_orconn_t objects that become
+ * unreferenced */
+static void
+bto_gid_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_gid_ht, bto_gid_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt);
+
+    c->gid = 0;
+    /* Don't delete if chan ID isn't zero: it's still in the chan hash map */
+    if (!c->chan)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_gid_ht, bto_gid_map);
+  tor_free(bto_gid_map);
+}
+
+/** Clear the chan ID hash map, freeing any bt_orconn_t objects that
+ * become unreferenced */
+static void
+bto_chan_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_chan_ht, bto_chan_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt);
+
+    c->chan = 0;
+    /* Don't delete if GID isn't zero, it's still in the GID hash map */
+    if (!c->gid)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_chan_ht, bto_chan_map);
+  tor_free(bto_chan_map);
+}
+
+/** Delete a bt_orconn from the hash maps by GID */
+void
+bto_delete(uint64_t gid)
+{
+  bt_orconn_t key, *bto;
+
+  key.gid = gid;
+  key.chan = 0;
+  bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto) {
+    /* The orconn might be unregistered because it's an EXT_OR_CONN? */
+    log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64,
+              gid);
+    return;
+  }
+  HT_REMOVE(bto_gid_ht, bto_gid_map, &key);
+  if (bto->chan) {
+    key.chan = bto->chan;
+    HT_REMOVE(bto_chan_ht, bto_chan_map, &key);
+  }
+  tor_free(bto);
+}
+
+/**
+ * Helper for bto_find_or_new().
+ *
+ * Update GID and chan ID of an existing bt_orconn object if needed,
+ * given a search key previously used within bto_find_or_new().
+ **/
+static bt_orconn_t *
+bto_update(bt_orconn_t *bto, const bt_orconn_t *key)
+{
+  /* ORCONN GIDs shouldn't change once assigned */
+  tor_assert(!bto->gid || !key->gid || bto->gid == key->gid);
+  if (!bto->gid && key->gid) {
+    /* Got a gid when we didn't already have one; insert into gid map */
+    log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan,
+              key->gid);
+    bto->gid = key->gid;
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  }
+  /* association of ORCONN with channel shouldn't change */
+  tor_assert(!bto->chan || !key->chan || bto->chan == key->chan);
+  if (!bto->chan && key->chan) {
+    /* Got a chan when we didn't already have one; insert into chan map */
+    log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64,
+              bto->gid, key->chan);
+    bto->chan = key->chan;
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+  }
+  return bto;
+}
+
+/** Helper for bto_find_or_new() */
+static bt_orconn_t *
+bto_new(const bt_orconn_t *key)
+{
+  struct bt_orconn_t *bto = tor_malloc(sizeof(*bto));
+
+  bto->gid = key->gid;
+  bto->chan = key->chan;
+  bto->state = 0;
+  bto->proxy_type = 0;
+  bto->is_orig = false;
+  bto->is_onehop = true;
+
+  if (bto->gid)
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  if (bto->chan)
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+
+  return bto;
+}
+
+/**
+ * Insert a new bt_orconn with the given GID and chan ID, or update
+ * the GID and chan ID if one already exists.
+ *
+ * Return the found or allocated bt_orconn.
+ **/
+bt_orconn_t *
+bto_find_or_new(uint64_t gid, uint64_t chan)
+{
+  bt_orconn_t key, *bto = NULL;
+
+  tor_assert(gid || chan);
+  key.gid = gid;
+  key.chan = chan;
+  if (key.gid)
+    bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto && key.chan) {
+    /* Not found by GID; look up by chan ID */
+    bto = HT_FIND(bto_chan_ht, bto_chan_map, &key);
+  }
+  if (bto)
+    return bto_update(bto, &key);
+  else
+    return bto_new(&key);
+}
+
+/** Initialize the hash maps  */
+void
+bto_init_maps(void)
+{
+  bto_gid_map = tor_malloc(sizeof(*bto_gid_map));
+  HT_INIT(bto_gid_ht, bto_gid_map);
+  bto_chan_map = tor_malloc(sizeof(*bto_chan_map));
+  HT_INIT(bto_chan_ht, bto_chan_map);
+}
+
+/** Clear the hash maps, freeing all associated storage */
+void
+bto_clear_maps(void)
+{
+  bto_gid_clear_map();
+  bto_chan_clear_map();
+}
diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h
new file mode 100644
index 0000000000..c83b22b1e8
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.h
+ * \brief Header file for btrack_orconn_maps.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_MAPS_H
+#define TOR_BTRACK_ORCONN_MAPS_H
+
+void bto_delete(uint64_t);
+bt_orconn_t *bto_find_or_new(uint64_t, uint64_t);
+
+void bto_init_maps(void);
+void bto_clear_maps(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_MAPS_H) */
diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h
new file mode 100644
index 0000000000..5a157b7b54
--- /dev/null
+++ b/src/feature/control/btrack_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_sys.h
+ * \brief Declare subsystem object for the bootstrap tracker susbystem.
+ **/
+
+#ifndef TOR_BTRACK_SYS_H
+#define TOR_BTRACK_SYS_H
+
+extern const struct subsys_fns_t sys_btrack;
+
+#endif /* !defined(TOR_BTRACK_SYS_H) */
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
index 26ac12d307..ee1026359d 100644
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@ -1,6 +1,5 @@
-
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,83 +32,27 @@
  * stack.
  **/
 
+#define CONTROL_MODULE_PRIVATE
 #define CONTROL_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
 #include "app/main/main.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
-#include "core/or/channel.h"
-#include "core/or/channeltls.h"
-#include "core/or/circuitbuild.h"
-#include "core/or/circuitlist.h"
-#include "core/or/circuitstats.h"
-#include "core/or/circuituse.h"
-#include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/policies.h"
-#include "core/or/reasons.h"
-#include "core/or/versions.h"
 #include "core/proto/proto_control0.h"
 #include "core/proto/proto_http.h"
-#include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/dnsserv.h"
-#include "feature/client/entrynodes.h"
 #include "feature/control/control.h"
-#include "feature/control/fmt_serverstatus.h"
-#include "feature/control/getinfo_geoip.h"
-#include "feature/dircache/dirserv.h"
-#include "feature/dirclient/dirclient.h"
-#include "feature/dirclient/dlstatus.h"
-#include "feature/dircommon/directory.h"
-#include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
-#include "feature/hs/hs_common.h"
-#include "feature/hs/hs_control.h"
-#include "feature/hs_common/shared_random_client.h"
-#include "feature/nodelist/authcert.h"
-#include "feature/nodelist/dirlist.h"
-#include "feature/nodelist/microdesc.h"
-#include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
-#include "feature/nodelist/routerlist.h"
-#include "feature/relay/router.h"
-#include "feature/relay/routermode.h"
-#include "feature/relay/selftest.h"
-#include "feature/rend/rendclient.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_proto.h"
 #include "feature/rend/rendcommon.h"
-#include "feature/rend/rendparse.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
-#include "feature/stats/predict_ports.h"
-#include "lib/container/buffers.h"
-#include "lib/crypt_ops/crypto_rand.h"
-#include "lib/crypt_ops/crypto_util.h"
-#include "lib/encoding/confline.h"
-#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/procmon.h"
 
-#include "feature/dircache/cached_dir_st.h"
 #include "feature/control/control_connection_st.h"
-#include "core/or/cpath_build_state_st.h"
-#include "core/or/entry_connection_st.h"
-#include "feature/nodelist/extrainfo_st.h"
-#include "feature/nodelist/networkstatus_st.h"
-#include "feature/nodelist/node_st.h"
-#include "core/or/or_connection_st.h"
-#include "core/or/or_circuit_st.h"
-#include "core/or/origin_circuit_st.h"
-#include "feature/nodelist/microdesc_st.h"
-#include "feature/rend/rend_authorized_client_st.h"
-#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
-#include "feature/rend/rend_service_descriptor_st.h"
-#include "feature/nodelist/routerinfo_st.h"
-#include "feature/nodelist/routerlist_st.h"
-#include "core/or/socks_request_st.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -118,152 +61,6 @@
 #include <sys/stat.h>
 #endif
 
-#ifndef _WIN32
-#include <pwd.h>
-#include <sys/resource.h>
-#endif
-
-#include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/evloop/procmon.h"
-#include "lib/evloop/compat_libevent.h"
-
-/** Yield true iff <b>s</b> is the state of a control_connection_t that has
- * finished authentication and is accepting commands. */
-#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
-
-/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
- * connection is interested in events of type <b>e</b>.  We use this
- * so that we can decide to skip generating event messages that nobody
- * has interest in without having to walk over the global connection
- * list to find out.
- **/
-typedef uint64_t event_mask_t;
-
-/** An event mask of all the events that any controller is interested in
- * receiving. */
-static event_mask_t global_event_mask = 0;
-
-/** True iff we have disabled log messages from being sent to the controller */
-static int disable_log_messages = 0;
-
-/** Macro: true if any control connection is interested in events of type
- * <b>e</b>. */
-#define EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & EVENT_MASK_(e)))
-
-/** Macro: true if any event from the bitfield 'e' is interesting. */
-#define ANY_EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & (e)))
-
-/** If we're using cookie-type authentication, how long should our cookies be?
- */
-#define AUTHENTICATION_COOKIE_LEN 32
-
-/** If true, we've set authentication_cookie to a secret code and
- * stored it to disk. */
-static int authentication_cookie_is_set = 0;
-/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
- * and which we're using to authenticate controllers.  (If the controller can
- * read it off disk, it has permission to connect.) */
-static uint8_t *authentication_cookie = NULL;
-
-#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
-  "Tor safe cookie authentication server-to-controller hash"
-#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
-  "Tor safe cookie authentication controller-to-server hash"
-#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
-
-/** The list of onion services that have been added via ADD_ONION that do not
- * belong to any particular control connection.
- */
-static smartlist_t *detached_onion_services = NULL;
-
-/** A sufficiently large size to record the last bootstrap phase string. */
-#define BOOTSTRAP_MSG_LEN 1024
-
-/** What was the last bootstrap phase message we sent? We keep track
- * of this so we can respond to getinfo status/bootstrap-phase queries. */
-static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
-
-static void connection_printf_to_buf(control_connection_t *conn,
-                                     const char *format, ...)
-  CHECK_PRINTF(2,3);
-static void send_control_event_impl(uint16_t event,
-                                    const char *format, va_list ap)
-  CHECK_PRINTF(2,0);
-static int control_event_status(int type, int severity, const char *format,
-                                va_list args)
-  CHECK_PRINTF(3,0);
-
-static void send_control_done(control_connection_t *conn);
-static void send_control_event(uint16_t event,
-                               const char *format, ...)
-  CHECK_PRINTF(2,3);
-static int handle_control_setconf(control_connection_t *conn, uint32_t len,
-                                  char *body);
-static int handle_control_resetconf(control_connection_t *conn, uint32_t len,
-                                    char *body);
-static int handle_control_getconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_setevents(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_authenticate(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_signal(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                                     const char *body);
-static char *list_getinfo_options(void);
-static int handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_extendcircuit(control_connection_t *conn,
-                                        uint32_t len,
-                                        const char *body);
-static int handle_control_setcircuitpurpose(control_connection_t *conn,
-                                            uint32_t len, const char *body);
-static int handle_control_attachstream(control_connection_t *conn,
-                                       uint32_t len,
-                                        const char *body);
-static int handle_control_postdescriptor(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_redirectstream(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_closestream(control_connection_t *conn, uint32_t len,
-                                      const char *body);
-static int handle_control_closecircuit(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_resolve(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_usefeature(control_connection_t *conn,
-                                     uint32_t len,
-                                     const char *body);
-static int handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_hspost(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_add_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_del_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
-                                      size_t len);
-static void orconn_target_get_name(char *buf, size_t len,
-                                   or_connection_t *conn);
-
-static int get_cached_network_liveness(void);
-static void set_cached_network_liveness(int liveness);
-
-static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
-
-static char * download_status_to_string(const download_status_t *dl);
-static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
-
 /** Convert a connection_t* to an control_connection_t*; assert if the cast is
  * invalid. */
 control_connection_t *
@@ -273,410 +70,6 @@ TO_CONTROL_CONN(connection_t *c)
   return DOWNCAST(control_connection_t, c);
 }
 
-/** Given a control event code for a message event, return the corresponding
- * log severity. */
-static inline int
-event_to_log_severity(int event)
-{
-  switch (event) {
-    case EVENT_DEBUG_MSG: return LOG_DEBUG;
-    case EVENT_INFO_MSG: return LOG_INFO;
-    case EVENT_NOTICE_MSG: return LOG_NOTICE;
-    case EVENT_WARN_MSG: return LOG_WARN;
-    case EVENT_ERR_MSG: return LOG_ERR;
-    default: return -1;
-  }
-}
-
-/** Given a log severity, return the corresponding control event code. */
-static inline int
-log_severity_to_event(int severity)
-{
-  switch (severity) {
-    case LOG_DEBUG: return EVENT_DEBUG_MSG;
-    case LOG_INFO: return EVENT_INFO_MSG;
-    case LOG_NOTICE: return EVENT_NOTICE_MSG;
-    case LOG_WARN: return EVENT_WARN_MSG;
-    case LOG_ERR: return EVENT_ERR_MSG;
-    default: return -1;
-  }
-}
-
-/** Helper: clear bandwidth counters of all origin circuits. */
-static void
-clear_circ_bw_fields(void)
-{
-  origin_circuit_t *ocirc;
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-    ocirc = TO_ORIGIN_CIRCUIT(circ);
-    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-  }
-  SMARTLIST_FOREACH_END(circ);
-}
-
-/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
- * connection's event_mask field. */
-void
-control_update_global_event_mask(void)
-{
-  smartlist_t *conns = get_connection_array();
-  event_mask_t old_mask, new_mask;
-  old_mask = global_event_mask;
-  int any_old_per_sec_events = control_any_per_second_event_enabled();
-
-  global_event_mask = 0;
-  SMARTLIST_FOREACH(conns, connection_t *, _conn,
-  {
-    if (_conn->type == CONN_TYPE_CONTROL &&
-        STATE_IS_OPEN(_conn->state)) {
-      control_connection_t *conn = TO_CONTROL_CONN(_conn);
-      global_event_mask |= conn->event_mask;
-    }
-  });
-
-  new_mask = global_event_mask;
-
-  /* Handle the aftermath.  Set up the log callback to tell us only what
-   * we want to hear...*/
-  control_adjust_event_log_severity();
-
-  /* Macro: true if ev was false before and is true now. */
-#define NEWLY_ENABLED(ev) \
-  (! (old_mask & (ev)) && (new_mask & (ev)))
-
-  /* ...then, if we've started logging stream or circ bw, clear the
-   * appropriate fields. */
-  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
-    SMARTLIST_FOREACH(conns, connection_t *, conn,
-    {
-      if (conn->type == CONN_TYPE_AP) {
-        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-        edge_conn->n_written = edge_conn->n_read = 0;
-      }
-    });
-  }
-  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
-    clear_circ_bw_fields();
-  }
-  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
-    uint64_t r, w;
-    control_get_bytes_rw_last_sec(&r, &w);
-  }
-  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
-    reschedule_per_second_timer();
-  }
-
-#undef NEWLY_ENABLED
-}
-
-/** Adjust the log severities that result in control_event_logmsg being called
- * to match the severity of log messages that any controllers are interested
- * in. */
-void
-control_adjust_event_log_severity(void)
-{
-  int i;
-  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
-
-  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      min_log_event = i;
-      break;
-    }
-  }
-  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      max_log_event = i;
-      break;
-    }
-  }
-  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
-    if (min_log_event > EVENT_NOTICE_MSG)
-      min_log_event = EVENT_NOTICE_MSG;
-    if (max_log_event < EVENT_ERR_MSG)
-      max_log_event = EVENT_ERR_MSG;
-  }
-  if (min_log_event <= max_log_event)
-    change_callback_log_severity(event_to_log_severity(min_log_event),
-                                 event_to_log_severity(max_log_event),
-                                 control_event_logmsg);
-  else
-    change_callback_log_severity(LOG_ERR, LOG_ERR,
-                                 control_event_logmsg);
-}
-
-/** Return true iff the event with code <b>c</b> is being sent to any current
- * control connection.  This is useful if the amount of work needed to prepare
- * to call the appropriate control_event_...() function is high.
- */
-int
-control_event_is_interesting(int event)
-{
-  return EVENT_IS_INTERESTING(event);
-}
-
-/** Return true if any event that needs to fire once a second is enabled. */
-int
-control_any_per_second_event_enabled(void)
-{
-  return ANY_EVENT_IS_INTERESTING(
-      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CELL_STATS) |
-      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CONN_BW) |
-      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
-  );
-}
-
-/* The value of 'get_bytes_read()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_read = 0;
-/* The value of 'get_bytes_written()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_written = 0;
-
-/**
- * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
- * and written by Tor since the last call to this function.
- *
- * Call this only from the main thread.
- */
-static void
-control_get_bytes_rw_last_sec(uint64_t *n_read,
-                              uint64_t *n_written)
-{
-  const uint64_t stats_n_bytes_read = get_bytes_read();
-  const uint64_t stats_n_bytes_written = get_bytes_written();
-
-  *n_read = stats_n_bytes_read - stats_prev_n_read;
-  *n_written = stats_n_bytes_written - stats_prev_n_written;
-  stats_prev_n_read = stats_n_bytes_read;
-  stats_prev_n_written = stats_n_bytes_written;
-}
-
-/**
- * Run all the controller events (if any) that are scheduled to trigger once
- * per second.
- */
-void
-control_per_second_events(void)
-{
-  if (!control_any_per_second_event_enabled())
-    return;
-
-  uint64_t bytes_read, bytes_written;
-  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
-  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
-
-  control_event_stream_bandwidth_used();
-  control_event_conn_bandwidth_used();
-  control_event_circ_bandwidth_used();
-  control_event_circuit_cell_stats();
-}
-
-/** Append a NUL-terminated string <b>s</b> to the end of
- * <b>conn</b>-\>outbuf.
- */
-static inline void
-connection_write_str_to_buf(const char *s, control_connection_t *conn)
-{
-  size_t len = strlen(s);
-  connection_buf_add(s, len, TO_CONN(conn));
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
- * contents of <b>data</b> into *<b>out</b>, adding a period before any period
- * that appears at the start of a line, and adding a period-CRLF line at
- * the end. Replace all LF characters sequences with CRLF.  Return the number
- * of bytes in *<b>out</b>.
- */
-STATIC size_t
-write_escaped_data(const char *data, size_t len, char **out)
-{
-  tor_assert(len < SIZE_MAX - 9);
-  size_t sz_out = len+8+1;
-  char *outp;
-  const char *start = data, *end;
-  size_t i;
-  int start_of_line;
-  for (i=0; i < len; ++i) {
-    if (data[i] == '\n') {
-      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
-      if (sz_out >= SIZE_T_CEILING) {
-        log_warn(LD_BUG, "Input to write_escaped_data was too long");
-        *out = tor_strdup(".\r\n");
-        return 3;
-      }
-    }
-  }
-  *out = outp = tor_malloc(sz_out);
-  end = data+len;
-  start_of_line = 1;
-  while (data < end) {
-    if (*data == '\n') {
-      if (data > start && data[-1] != '\r')
-        *outp++ = '\r';
-      start_of_line = 1;
-    } else if (*data == '.') {
-      if (start_of_line) {
-        start_of_line = 0;
-        *outp++ = '.';
-      }
-    } else {
-      start_of_line = 0;
-    }
-    *outp++ = *data++;
-  }
-  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
-    *outp++ = '\r';
-    *outp++ = '\n';
-  }
-  *outp++ = '.';
-  *outp++ = '\r';
-  *outp++ = '\n';
-  *outp = '\0'; /* NUL-terminate just in case. */
-  tor_assert(outp >= *out);
-  tor_assert((size_t)(outp - *out) <= sz_out);
-  return outp - *out;
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
- * the contents of <b>data</b> into *<b>out</b>, removing any period
- * that appears at the start of a line, and replacing all CRLF sequences
- * with LF.   Return the number of
- * bytes in *<b>out</b>. */
-STATIC size_t
-read_escaped_data(const char *data, size_t len, char **out)
-{
-  char *outp;
-  const char *next;
-  const char *end;
-
-  *out = outp = tor_malloc(len+1);
-
-  end = data+len;
-
-  while (data < end) {
-    /* we're at the start of a line. */
-    if (*data == '.')
-      ++data;
-    next = memchr(data, '\n', end-data);
-    if (next) {
-      size_t n_to_copy = next-data;
-      /* Don't copy a CR that precedes this LF. */
-      if (n_to_copy && *(next-1) == '\r')
-        --n_to_copy;
-      memcpy(outp, data, n_to_copy);
-      outp += n_to_copy;
-      data = next+1; /* This will point at the start of the next line,
-                      * or the end of the string, or a period. */
-    } else {
-      memcpy(outp, data, end-data);
-      outp += (end-data);
-      *outp = '\0';
-      return outp - *out;
-    }
-    *outp++ = '\n';
-  }
-
-  *outp = '\0';
-  return outp - *out;
-}
-
-/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
- * double-quoted string with escaped characters, return the length of that
- * string (as encoded, including quotes).  Otherwise return -1. */
-static inline int
-get_escaped_string_length(const char *start, size_t in_len_max,
-                          int *chars_out)
-{
-  const char *cp, *end;
-  int chars = 0;
-
-  if (*start != '\"')
-    return -1;
-
-  cp = start+1;
-  end = start+in_len_max;
-
-  /* Calculate length. */
-  while (1) {
-    if (cp >= end) {
-      return -1; /* Too long. */
-    } else if (*cp == '\\') {
-      if (++cp == end)
-        return -1; /* Can't escape EOS. */
-      ++cp;
-      ++chars;
-    } else if (*cp == '\"') {
-      break;
-    } else {
-      ++cp;
-      ++chars;
-    }
-  }
-  if (chars_out)
-    *chars_out = chars;
-  return (int)(cp - start+1);
-}
-
-/** As decode_escaped_string, but does not decode the string: copies the
- * entire thing, including quotation marks. */
-static const char *
-extract_escaped_string(const char *start, size_t in_len_max,
-                       char **out, size_t *out_len)
-{
-  int length = get_escaped_string_length(start, in_len_max, NULL);
-  if (length<0)
-    return NULL;
-  *out_len = length;
-  *out = tor_strndup(start, *out_len);
-  return start+length;
-}
-
-/** Given a pointer to a string starting at <b>start</b> containing
- * <b>in_len_max</b> characters, decode a string beginning with one double
- * quote, containing any number of non-quote characters or characters escaped
- * with a backslash, and ending with a final double quote.  Place the resulting
- * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
- * store its length in <b>out_len</b>.  On success, return a pointer to the
- * character immediately following the escaped string.  On failure, return
- * NULL. */
-static const char *
-decode_escaped_string(const char *start, size_t in_len_max,
-                   char **out, size_t *out_len)
-{
-  const char *cp, *end;
-  char *outp;
-  int len, n_chars = 0;
-
-  len = get_escaped_string_length(start, in_len_max, &n_chars);
-  if (len<0)
-    return NULL;
-
-  end = start+len-1; /* Index of last quote. */
-  tor_assert(*end == '\"');
-  outp = *out = tor_malloc(len+1);
-  *out_len = n_chars;
-
-  cp = start+1;
-  while (cp < end) {
-    if (*cp == '\\')
-      ++cp;
-    *outp++ = *cp++;
-  }
-  *outp = '\0';
-  tor_assert((outp - *out) == (int)*out_len);
-
-  return end+1;
-}
-
 /** Create and add a new controller connection on <b>sock</b>.  If
  * <b>CC_LOCAL_FD_IS_OWNER</b> is set in <b>flags</b>, this Tor process should
  * exit when the connection closes.  If <b>CC_LOCAL_FD_IS_AUTHENTICATED</b>
@@ -720,29 +113,6 @@ control_connection_add_local_fd(tor_socket_t sock, unsigned flags)
   return 0;
 }
 
-/** Acts like sprintf, but writes its formatted string to the end of
- * <b>conn</b>-\>outbuf. */
-static void
-connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
-{
-  va_list ap;
-  char *buf = NULL;
-  int len;
-
-  va_start(ap,format);
-  len = tor_vasprintf(&buf, format, ap);
-  va_end(ap);
-
-  if (len < 0) {
-    log_err(LD_BUG, "Unable to format string for controller.");
-    tor_assert(0);
-  }
-
-  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
-
-  tor_free(buf);
-}
-
 /** Write all of the open control ports to ControlPortWriteToFile */
 void
 control_ports_write_to_file(void)
@@ -787,886 +157,11 @@ control_ports_write_to_file(void)
   smartlist_free(lines);
 }
 
-/** Send a "DONE" message down the control connection <b>conn</b>. */
-static void
-send_control_done(control_connection_t *conn)
-{
-  connection_write_str_to_buf("250 OK\r\n", conn);
-}
-
-/** Represents an event that's queued to be sent to one or more
- * controllers. */
-typedef struct queued_event_s {
-  uint16_t event;
-  char *msg;
-} queued_event_t;
-
-/** Pointer to int. If this is greater than 0, we don't allow new events to be
- * queued. */
-static tor_threadlocal_t block_event_queue_flag;
-
-/** Holds a smartlist of queued_event_t objects that may need to be sent
- * to one or more controllers */
-static smartlist_t *queued_control_events = NULL;
-
-/** True if the flush_queued_events_event is pending. */
-static int flush_queued_event_pending = 0;
-
-/** Lock to protect the above fields. */
-static tor_mutex_t *queued_control_events_lock = NULL;
-
-/** An event that should fire in order to flush the contents of
- * queued_control_events. */
-static mainloop_event_t *flush_queued_events_event = NULL;
-
-void
-control_initialize_event_queue(void)
-{
-  if (queued_control_events == NULL) {
-    queued_control_events = smartlist_new();
-  }
-
-  if (flush_queued_events_event == NULL) {
-    struct event_base *b = tor_libevent_get_base();
-    if (b) {
-      flush_queued_events_event =
-        mainloop_event_new(flush_queued_events_cb, NULL);
-      tor_assert(flush_queued_events_event);
-    }
-  }
-
-  if (queued_control_events_lock == NULL) {
-    queued_control_events_lock = tor_mutex_new();
-    tor_threadlocal_init(&block_event_queue_flag);
-  }
-}
-
-static int *
-get_block_event_queue(void)
-{
-  int *val = tor_threadlocal_get(&block_event_queue_flag);
-  if (PREDICT_UNLIKELY(val == NULL)) {
-    val = tor_malloc_zero(sizeof(int));
-    tor_threadlocal_set(&block_event_queue_flag, val);
-  }
-  return val;
-}
-
-/** Helper: inserts an event on the list of events queued to be sent to
- * one or more controllers, and schedules the events to be flushed if needed.
- *
- * This function takes ownership of <b>msg</b>, and may free it.
- *
- * We queue these events rather than send them immediately in order to break
- * the dependency in our callgraph from code that generates events for the
- * controller, and the network layer at large.  Otherwise, nearly every
- * interesting part of Tor would potentially call every other interesting part
- * of Tor.
- */
-MOCK_IMPL(STATIC void,
-queue_control_event_string,(uint16_t event, char *msg))
-{
-  /* This is redundant with checks done elsewhere, but it's a last-ditch
-   * attempt to avoid queueing something we shouldn't have to queue. */
-  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
-    tor_free(msg);
-    return;
-  }
-
-  int *block_event_queue = get_block_event_queue();
-  if (*block_event_queue) {
-    tor_free(msg);
-    return;
-  }
-
-  queued_event_t *ev = tor_malloc(sizeof(*ev));
-  ev->event = event;
-  ev->msg = msg;
-
-  /* No queueing an event while queueing an event */
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  tor_assert(queued_control_events);
-  smartlist_add(queued_control_events, ev);
-
-  int activate_event = 0;
-  if (! flush_queued_event_pending && in_main_thread()) {
-    activate_event = 1;
-    flush_queued_event_pending = 1;
-  }
-
-  tor_mutex_release(queued_control_events_lock);
-
-  --*block_event_queue;
-
-  /* We just put an event on the queue; mark the queue to be
-   * flushed.  We only do this from the main thread for now; otherwise,
-   * we'd need to incur locking overhead in Libevent or use a socket.
-   */
-  if (activate_event) {
-    tor_assert(flush_queued_events_event);
-    mainloop_event_activate(flush_queued_events_event);
-  }
-}
-
-#define queued_event_free(ev) \
-  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
-
-/** Release all storage held by <b>ev</b>. */
-static void
-queued_event_free_(queued_event_t *ev)
-{
-  if (ev == NULL)
-    return;
-
-  tor_free(ev->msg);
-  tor_free(ev);
-}
-
-/** Send every queued event to every controller that's interested in it,
- * and remove the events from the queue.  If <b>force</b> is true,
- * then make all controllers send their data out immediately, since we
- * may be about to shut down. */
-static void
-queued_events_flush_all(int force)
-{
-  /* Make sure that we get all the pending log events, if there are any. */
-  flush_pending_log_callbacks();
-
-  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
-    return;
-  }
-  smartlist_t *all_conns = get_connection_array();
-  smartlist_t *controllers = smartlist_new();
-  smartlist_t *queued_events;
-
-  int *block_event_queue = get_block_event_queue();
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  /* No queueing an event while flushing events. */
-  flush_queued_event_pending = 0;
-  queued_events = queued_control_events;
-  queued_control_events = smartlist_new();
-  tor_mutex_release(queued_control_events_lock);
-
-  /* Gather all the controllers that will care... */
-  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
-    if (conn->type == CONN_TYPE_CONTROL &&
-        !conn->marked_for_close &&
-        conn->state == CONTROL_CONN_STATE_OPEN) {
-      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
-
-      smartlist_add(controllers, control_conn);
-    }
-  } SMARTLIST_FOREACH_END(conn);
-
-  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
-    const event_mask_t bit = ((event_mask_t)1) << ev->event;
-    const size_t msg_len = strlen(ev->msg);
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      if (control_conn->event_mask & bit) {
-        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
-      }
-    } SMARTLIST_FOREACH_END(control_conn);
-
-    queued_event_free(ev);
-  } SMARTLIST_FOREACH_END(ev);
-
-  if (force) {
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      connection_flush(TO_CONN(control_conn));
-    } SMARTLIST_FOREACH_END(control_conn);
-  }
-
-  smartlist_free(queued_events);
-  smartlist_free(controllers);
-
-  --*block_event_queue;
-}
-
-/** Libevent callback: Flushes pending events to controllers that are
- * interested in them. */
-static void
-flush_queued_events_cb(mainloop_event_t *event, void *arg)
-{
-  (void) event;
-  (void) arg;
-  queued_events_flush_all(0);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is given by <b>msg</b>.
- *
- * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
- * respect to the EXTENDED_EVENTS feature. */
-MOCK_IMPL(STATIC void,
-send_control_event_string,(uint16_t event,
-                           const char *msg))
-{
-  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
-  queue_control_event_string(event, tor_strdup(msg));
-}
-
-/** Helper for send_control_event and control_event_status:
- * Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event_impl(uint16_t event,
-                        const char *format, va_list ap)
-{
-  char *buf = NULL;
-  int len;
-
-  len = tor_vasprintf(&buf, format, ap);
-  if (len < 0) {
-    log_warn(LD_BUG, "Unable to format event for controller.");
-    return;
-  }
-
-  queue_control_event_string(event, buf);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event(uint16_t event,
-                   const char *format, ...)
-{
-  va_list ap;
-  va_start(ap, format);
-  send_control_event_impl(event, format, ap);
-  va_end(ap);
-}
-
-/** Given a text circuit <b>id</b>, return the corresponding circuit. */
-static origin_circuit_t *
-get_circ(const char *id)
-{
-  uint32_t n_id;
-  int ok;
-  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  return circuit_get_by_global_id(n_id);
-}
-
-/** Given a text stream <b>id</b>, return the corresponding AP connection. */
-static entry_connection_t *
-get_stream(const char *id)
-{
-  uint64_t n_id;
-  int ok;
-  connection_t *conn;
-  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  conn = connection_get_by_global_id(n_id);
-  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
-    return NULL;
-  return TO_ENTRY_CONN(conn);
-}
-
-/** Helper for setconf and resetconf. Acts like setconf, except
- * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
- * contents of body.
- */
-static int
-control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
-                       int use_defaults)
-{
-  setopt_err_t opt_err;
-  config_line_t *lines=NULL;
-  char *start = body;
-  char *errstring = NULL;
-  const unsigned flags =
-    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
-
-  char *config;
-  smartlist_t *entries = smartlist_new();
-
-  /* We have a string, "body", of the format '(key(=val|="val")?)' entries
-   * separated by space.  break it into a list of configuration entries. */
-  while (*body) {
-    char *eq = body;
-    char *key;
-    char *entry;
-    while (!TOR_ISSPACE(*eq) && *eq != '=')
-      ++eq;
-    key = tor_strndup(body, eq-body);
-    body = eq+1;
-    if (*eq == '=') {
-      char *val=NULL;
-      size_t val_len=0;
-      if (*body != '\"') {
-        char *val_start = body;
-        while (!TOR_ISSPACE(*body))
-          body++;
-        val = tor_strndup(val_start, body-val_start);
-        val_len = strlen(val);
-      } else {
-        body = (char*)extract_escaped_string(body, (len - (body-start)),
-                                             &val, &val_len);
-        if (!body) {
-          connection_write_str_to_buf("551 Couldn't parse string\r\n", conn);
-          SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-          smartlist_free(entries);
-          tor_free(key);
-          return 0;
-        }
-      }
-      tor_asprintf(&entry, "%s %s", key, val);
-      tor_free(key);
-      tor_free(val);
-    } else {
-      entry = key;
-    }
-    smartlist_add(entries, entry);
-    while (TOR_ISSPACE(*body))
-      ++body;
-  }
-
-  smartlist_add_strdup(entries, "");
-  config = smartlist_join_strings(entries, "\n", 0, NULL);
-  SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-  smartlist_free(entries);
-
-  if (config_get_lines(config, &lines, 0) < 0) {
-    log_warn(LD_CONTROL,"Controller gave us config lines we can't parse.");
-    connection_write_str_to_buf("551 Couldn't parse configuration\r\n",
-                                conn);
-    tor_free(config);
-    return 0;
-  }
-  tor_free(config);
-
-  opt_err = options_trial_assign(lines, flags, &errstring);
-  {
-    const char *msg;
-    switch (opt_err) {
-      case SETOPT_ERR_MISC:
-        msg = "552 Unrecognized option";
-        break;
-      case SETOPT_ERR_PARSE:
-        msg = "513 Unacceptable option value";
-        break;
-      case SETOPT_ERR_TRANSITION:
-        msg = "553 Transition not allowed";
-        break;
-      case SETOPT_ERR_SETTING:
-      default:
-        msg = "553 Unable to set option";
-        break;
-      case SETOPT_OK:
-        config_free_lines(lines);
-        send_control_done(conn);
-        return 0;
-    }
-    log_warn(LD_CONTROL,
-             "Controller gave us config lines that didn't validate: %s",
-             errstring);
-    connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    config_free_lines(lines);
-    tor_free(errstring);
-    return 0;
-  }
-}
-
-/** Called when we receive a SETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body.*/
-static int
-handle_control_setconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 0);
-}
-
-/** Called when we receive a RESETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body. */
-static int
-handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 1);
-}
-
-/** Called when we receive a GETCONF message.  Parse the request, and
- * reply with a CONFVALUE or an ERROR message */
-static int
-handle_control_getconf(control_connection_t *conn, uint32_t body_len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *msg = NULL;
-  size_t msg_len;
-  const or_options_t *options = get_options();
-  int i, len;
-
-  (void) body_len; /* body is NUL-terminated; so we can ignore len. */
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    if (!option_is_recognized(q)) {
-      smartlist_add(unrecognized, (char*) q);
-    } else {
-      config_line_t *answer = option_get_assignment(options,q);
-      if (!answer) {
-        const char *name = option_get_canonical_name(q);
-        smartlist_add_asprintf(answers, "250-%s\r\n", name);
-      }
-
-      while (answer) {
-        config_line_t *next;
-        smartlist_add_asprintf(answers, "250-%s=%s\r\n",
-                     answer->key, answer->value);
-
-        next = answer->next;
-        tor_free(answer->key);
-        tor_free(answer->value);
-        tor_free(answer);
-        answer = next;
-      }
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if ((len = smartlist_len(unrecognized))) {
-    for (i=0; i < len-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-Unrecognized configuration key \"%s\"\r\n",
-                               (char*)smartlist_get(unrecognized, i));
-    connection_printf_to_buf(conn,
-                             "552 Unrecognized configuration key \"%s\"\r\n",
-                             (char*)smartlist_get(unrecognized, len-1));
-  } else if ((len = smartlist_len(answers))) {
-    char *tmp = smartlist_get(answers, len-1);
-    tor_assert(strlen(tmp)>4);
-    tmp[3] = ' ';
-    msg = smartlist_join_strings(answers, "", 0, &msg_len);
-    connection_buf_add(msg, msg_len, TO_CONN(conn));
-  } else {
-    connection_write_str_to_buf("250 OK\r\n", conn);
-  }
-
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  smartlist_free(unrecognized);
-
-  tor_free(msg);
-
-  return 0;
-}
-
-/** Called when we get a +LOADCONF message. */
-static int
-handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  setopt_err_t retval;
-  char *errstring = NULL;
-  const char *msg = NULL;
-  (void) len;
-
-  retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring);
-
-  if (retval != SETOPT_OK)
-    log_warn(LD_CONTROL,
-             "Controller gave us config file that didn't validate: %s",
-             errstring);
-
-  switch (retval) {
-  case SETOPT_ERR_PARSE:
-    msg = "552 Invalid config file";
-    break;
-  case SETOPT_ERR_TRANSITION:
-    msg = "553 Transition not allowed";
-    break;
-  case SETOPT_ERR_SETTING:
-    msg = "553 Unable to set option";
-    break;
-  case SETOPT_ERR_MISC:
-  default:
-    msg = "550 Unable to load config";
-    break;
-  case SETOPT_OK:
-    break;
-  }
-  if (msg) {
-    if (errstring)
-      connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    else
-      connection_printf_to_buf(conn, "%s\r\n", msg);
-  } else {
-    send_control_done(conn);
-  }
-  tor_free(errstring);
-  return 0;
-}
-
-/** Helper structure: maps event values to their names. */
-struct control_event_t {
-  uint16_t event_code;
-  const char *event_name;
-};
-/** Table mapping event values to their names.  Used to implement SETEVENTS
- * and GETINFO events/names, and to keep they in sync. */
-static const struct control_event_t control_event_table[] = {
-  { EVENT_CIRCUIT_STATUS, "CIRC" },
-  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
-  { EVENT_STREAM_STATUS, "STREAM" },
-  { EVENT_OR_CONN_STATUS, "ORCONN" },
-  { EVENT_BANDWIDTH_USED, "BW" },
-  { EVENT_DEBUG_MSG, "DEBUG" },
-  { EVENT_INFO_MSG, "INFO" },
-  { EVENT_NOTICE_MSG, "NOTICE" },
-  { EVENT_WARN_MSG, "WARN" },
-  { EVENT_ERR_MSG, "ERR" },
-  { EVENT_NEW_DESC, "NEWDESC" },
-  { EVENT_ADDRMAP, "ADDRMAP" },
-  { EVENT_DESCCHANGED, "DESCCHANGED" },
-  { EVENT_NS, "NS" },
-  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
-  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
-  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
-  { EVENT_GUARD, "GUARD" },
-  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
-  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
-  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
-  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
-  { EVENT_GOT_SIGNAL, "SIGNAL" },
-  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
-  { EVENT_CONN_BW, "CONN_BW" },
-  { EVENT_CELL_STATS, "CELL_STATS" },
-  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
-  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
-  { EVENT_HS_DESC, "HS_DESC" },
-  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
-  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
-  { 0, NULL },
-};
-
-/** Called when we get a SETEVENTS message: update conn->event_mask,
- * and reply with DONE or ERROR. */
-static int
-handle_control_setevents(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  int event_code;
-  event_mask_t event_mask = 0;
-  smartlist_t *events = smartlist_new();
-
-  (void) len;
-
-  smartlist_split_string(events, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
-    {
-      if (!strcasecmp(ev, "EXTENDED") ||
-          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
-        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
-                 "supported.", ev);
-        continue;
-      } else {
-        int i;
-        event_code = -1;
-
-        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-          if (!strcasecmp(ev, control_event_table[i].event_name)) {
-            event_code = control_event_table[i].event_code;
-            break;
-          }
-        }
-
-        if (event_code == -1) {
-          connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n",
-                                   ev);
-          SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-          smartlist_free(events);
-          return 0;
-        }
-      }
-      event_mask |= (((event_mask_t)1) << event_code);
-    }
-  SMARTLIST_FOREACH_END(ev);
-  SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-  smartlist_free(events);
-
-  conn->event_mask = event_mask;
-
-  control_update_global_event_mask();
-  send_control_done(conn);
-  return 0;
-}
-
-/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
- * Return a smartlist of acceptable passwords (unterminated strings of
- * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
- * failure.
- */
-smartlist_t *
-decode_hashed_passwords(config_line_t *passwords)
-{
-  char decoded[64];
-  config_line_t *cl;
-  smartlist_t *sl = smartlist_new();
-
-  tor_assert(passwords);
-
-  for (cl = passwords; cl; cl = cl->next) {
-    const char *hashed = cl->value;
-
-    if (!strcmpstart(hashed, "16:")) {
-      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
-                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
-          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
-        goto err;
-      }
-    } else {
-        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
-            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
-          goto err;
-        }
-    }
-    smartlist_add(sl,
-                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
-  }
-
-  return sl;
-
- err:
-  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
-  smartlist_free(sl);
-  return NULL;
-}
-
-/** Called when we get an AUTHENTICATE message.  Check whether the
- * authentication is valid, and if so, update the connection's state to
- * OPEN.  Reply with DONE or ERROR.
- */
-static int
-handle_control_authenticate(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  int used_quoted_string = 0;
-  const or_options_t *options = get_options();
-  const char *errstr = "Unknown error";
-  char *password;
-  size_t password_len;
-  const char *cp;
-  int i;
-  int bad_cookie=0, bad_password=0;
-  smartlist_t *sl = NULL;
-
-  if (!len) {
-    password = tor_strdup("");
-    password_len = 0;
-  } else if (TOR_ISXDIGIT(body[0])) {
-    cp = body;
-    while (TOR_ISXDIGIT(*cp))
-      ++cp;
-    i = (int)(cp - body);
-    tor_assert(i>0);
-    password_len = i/2;
-    password = tor_malloc(password_len + 1);
-    if (base16_decode(password, password_len+1, body, i)
-                      != (int) password_len) {
-      connection_write_str_to_buf(
-            "551 Invalid hexadecimal encoding.  Maybe you tried a plain text "
-            "password?  If so, the standard requires that you put it in "
-            "double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(password);
-      return 0;
-    }
-  } else {
-    if (!decode_escaped_string(body, len, &password, &password_len)) {
-      connection_write_str_to_buf("551 Invalid quoted string.  You need "
-            "to put the password in double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return 0;
-    }
-    used_quoted_string = 1;
-  }
-
-  if (conn->safecookie_client_hash != NULL) {
-    /* The controller has chosen safe cookie authentication; the only
-     * acceptable authentication value is the controller-to-server
-     * response. */
-
-    tor_assert(authentication_cookie_is_set);
-
-    if (password_len != DIGEST256_LEN) {
-      log_warn(LD_CONTROL,
-               "Got safe cookie authentication response with wrong length "
-               "(%d)", (int)password_len);
-      errstr = "Wrong length for safe cookie response.";
-      goto err;
-    }
-
-    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
-      log_warn(LD_CONTROL,
-               "Got incorrect safe cookie authentication response");
-      errstr = "Safe cookie response did not match expected value.";
-      goto err;
-    }
-
-    tor_free(conn->safecookie_client_hash);
-    goto ok;
-  }
-
-  if (!options->CookieAuthentication && !options->HashedControlPassword &&
-      !options->HashedControlSessionPassword) {
-    /* if Tor doesn't demand any stronger authentication, then
-     * the controller can get in with anything. */
-    goto ok;
-  }
-
-  if (options->CookieAuthentication) {
-    int also_password = options->HashedControlPassword != NULL ||
-      options->HashedControlSessionPassword != NULL;
-    if (password_len != AUTHENTICATION_COOKIE_LEN) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
-                 "(%d)", (int)password_len);
-        errstr = "Wrong length on authentication cookie.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else if (tor_memneq(authentication_cookie, password, password_len)) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
-        errstr = "Authentication cookie did not match expected value.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else {
-      goto ok;
-    }
-  }
-
-  if (options->HashedControlPassword ||
-      options->HashedControlSessionPassword) {
-    int bad = 0;
-    smartlist_t *sl_tmp;
-    char received[DIGEST_LEN];
-    int also_cookie = options->CookieAuthentication;
-    sl = smartlist_new();
-    if (options->HashedControlPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (options->HashedControlSessionPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (bad) {
-      if (!also_cookie) {
-        log_warn(LD_BUG,
-                 "Couldn't decode HashedControlPassword: invalid base16");
-        errstr="Couldn't decode HashedControlPassword value in configuration.";
-        goto err;
-      }
-      bad_password = 1;
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-    } else {
-      SMARTLIST_FOREACH(sl, char *, expected,
-      {
-        secret_to_key_rfc2440(received,DIGEST_LEN,
-                              password,password_len,expected);
-        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
-                      received, DIGEST_LEN))
-          goto ok;
-      });
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-
-      if (used_quoted_string)
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration";
-      else
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration. Maybe you tried a plain text password? "
-          "If so, the standard requires that you put it in double quotes.";
-      bad_password = 1;
-      if (!also_cookie)
-        goto err;
-    }
-  }
-
-  /** We only get here if both kinds of authentication failed. */
-  tor_assert(bad_password && bad_cookie);
-  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
-  errstr = "Password did not match HashedControlPassword *or* authentication "
-    "cookie.";
-
- err:
-  tor_free(password);
-  connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr);
-  connection_mark_for_close(TO_CONN(conn));
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
- ok:
-  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
-           ")", conn->base_.s);
-  send_control_done(conn);
-  conn->base_.state = CONTROL_CONN_STATE_OPEN;
-  tor_free(password);
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
-}
-
-/** Called when we get a SAVECONF command. Try to flush the current options to
- * disk, and report success or failure. */
-static int
-handle_control_saveconf(control_connection_t *conn, uint32_t len,
-                        const char *body)
-{
-  (void) len;
-
-  int force = !strcmpstart(body, "FORCE");
-  const or_options_t *options = get_options();
-  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
-    connection_write_str_to_buf(
-      "551 Unable to write configuration to disk.\r\n", conn);
-  } else {
-    send_control_done(conn);
-  }
-  return 0;
-}
-
-struct signal_t {
-  int sig;
-  const char *signal_name;
-};
-
-static const struct signal_t signal_table[] = {
+const struct signal_name_t signal_table[] = {
+  /* NOTE: this table is used for handling SIGNAL commands and generating
+   * SIGNAL events.  Order is significant: if there are two entries for the
+   * same numeric signal, the first one is the canonical name generated
+   * for the events. */
   { SIGHUP, "RELOAD" },
   { SIGHUP, "HUP" },
   { SIGINT, "SHUTDOWN" },
@@ -1680,3576 +175,11 @@ static const struct signal_t signal_table[] = {
   { SIGNEWNYM, "NEWNYM" },
   { SIGCLEARDNSCACHE, "CLEARDNSCACHE"},
   { SIGHEARTBEAT, "HEARTBEAT"},
+  { SIGACTIVE, "ACTIVE" },
+  { SIGDORMANT, "DORMANT" },
   { 0, NULL },
 };
 
-/** Called when we get a SIGNAL command. React to the provided signal, and
- * report success or failure. (If the signal results in a shutdown, success
- * may not be reported.) */
-static int
-handle_control_signal(control_connection_t *conn, uint32_t len,
-                      const char *body)
-{
-  int sig = -1;
-  int i;
-  int n = 0;
-  char *s;
-
-  (void) len;
-
-  while (body[n] && ! TOR_ISSPACE(body[n]))
-    ++n;
-  s = tor_strndup(body, n);
-
-  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
-    if (!strcasecmp(s, signal_table[i].signal_name)) {
-      sig = signal_table[i].sig;
-      break;
-    }
-  }
-
-  if (sig < 0)
-    connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n",
-                             s);
-  tor_free(s);
-  if (sig < 0)
-    return 0;
-
-  send_control_done(conn);
-  /* Flush the "done" first if the signal might make us shut down. */
-  if (sig == SIGTERM || sig == SIGINT)
-    connection_flush(TO_CONN(conn));
-
-  activate_signal(sig);
-
-  return 0;
-}
-
-/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
- * as an owning connection, so that we will exit if the connection
- * closes. */
-static int
-handle_control_takeownership(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  (void)len;
-  (void)body;
-
-  conn->is_owning_control_connection = 1;
-
-  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
-           "Tor instance.",
-           (int)(conn->base_.s));
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
- * containing funny characters. */
-static int
-address_is_invalid_mapaddress_target(const char *addr)
-{
-  if (!strcmpstart(addr, "*."))
-    return address_is_invalid_destination(addr+2, 1);
-  else
-    return address_is_invalid_destination(addr, 1);
-}
-
-/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
- * and report success or failure. */
-static int
-handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                          const char *body)
-{
-  smartlist_t *elts;
-  smartlist_t *lines;
-  smartlist_t *reply;
-  char *r;
-  size_t sz;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  lines = smartlist_new();
-  elts = smartlist_new();
-  reply = smartlist_new();
-  smartlist_split_string(lines, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(lines, char *, line) {
-    tor_strlower(line);
-    smartlist_split_string(elts, line, "=", 0, 2);
-    if (smartlist_len(elts) == 2) {
-      const char *from = smartlist_get(elts,0);
-      const char *to = smartlist_get(elts,1);
-      if (address_is_invalid_mapaddress_target(to)) {
-        smartlist_add_asprintf(reply,
-                     "512-syntax error: invalid address '%s'", to);
-        log_warn(LD_CONTROL,
-                 "Skipping invalid argument '%s' in MapAddress msg", to);
-      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
-                 !strcmp(from, "::")) {
-        const char type =
-          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
-          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
-        const char *address = addressmap_register_virtual_address(
-                                                     type, tor_strdup(to));
-        if (!address) {
-          smartlist_add_asprintf(reply,
-                       "451-resource exhausted: skipping '%s'", line);
-          log_warn(LD_CONTROL,
-                   "Unable to allocate address for '%s' in MapAddress msg",
-                   safe_str_client(line));
-        } else {
-          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
-        }
-      } else {
-        const char *msg;
-        if (addressmap_register_auto(from, to, 1,
-                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
-          smartlist_add_asprintf(reply,
-                                 "512-syntax error: invalid address mapping "
-                                 " '%s': %s", line, msg);
-          log_warn(LD_CONTROL,
-                   "Skipping invalid argument '%s' in MapAddress msg: %s",
-                   line, msg);
-        } else {
-          smartlist_add_asprintf(reply, "250-%s", line);
-        }
-      }
-    } else {
-      smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is "
-                   "not of expected form 'foo=bar'.", line);
-      log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong "
-                           "number of items.",
-                           safe_str_client(line));
-    }
-    SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
-    smartlist_clear(elts);
-  } SMARTLIST_FOREACH_END(line);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  smartlist_free(elts);
-
-  if (smartlist_len(reply)) {
-    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
-    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
-    connection_buf_add(r, sz, TO_CONN(conn));
-    tor_free(r);
-  } else {
-    const char *response =
-      "512 syntax error: not enough arguments to mapaddress.\r\n";
-    connection_buf_add(response, strlen(response), TO_CONN(conn));
-  }
-
-  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
-  smartlist_free(reply);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for various
- * trivial-to-implement questions. */
-static int
-getinfo_helper_misc(control_connection_t *conn, const char *question,
-                    char **answer, const char **errmsg)
-{
-  (void) conn;
-  if (!strcmp(question, "version")) {
-    *answer = tor_strdup(get_version());
-  } else if (!strcmp(question, "bw-event-cache")) {
-    *answer = get_bw_samples();
-  } else if (!strcmp(question, "config-file")) {
-    const char *a = get_torrc_fname(0);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-defaults-file")) {
-    const char *a = get_torrc_fname(1);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-text")) {
-    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
-  } else if (!strcmp(question, "config-can-saveconf")) {
-    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
-  } else if (!strcmp(question, "info/names")) {
-    *answer = list_getinfo_options();
-  } else if (!strcmp(question, "dormant")) {
-    int dormant = rep_hist_circbuilding_dormant(time(NULL));
-    *answer = tor_strdup(dormant ? "1" : "0");
-  } else if (!strcmp(question, "events/names")) {
-    int i;
-    smartlist_t *event_names = smartlist_new();
-
-    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-      smartlist_add(event_names, (char *)control_event_table[i].event_name);
-    }
-
-    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
-
-    smartlist_free(event_names);
-  } else if (!strcmp(question, "signal/names")) {
-    smartlist_t *signal_names = smartlist_new();
-    int j;
-    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
-      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
-    }
-
-    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
-
-    smartlist_free(signal_names);
-  } else if (!strcmp(question, "features/names")) {
-    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
-  } else if (!strcmp(question, "address")) {
-    uint32_t addr;
-    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
-      *errmsg = "Address unknown";
-      return -1;
-    }
-    *answer = tor_dup_ip(addr);
-  } else if (!strcmp(question, "traffic/read")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
-  } else if (!strcmp(question, "traffic/written")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
-  } else if (!strcmp(question, "uptime")) {
-    long uptime_secs = get_uptime();
-    tor_asprintf(answer, "%ld", uptime_secs);
-  } else if (!strcmp(question, "process/pid")) {
-    int myPid = -1;
-
-#ifdef _WIN32
-      myPid = _getpid();
-#else
-      myPid = getpid();
-#endif
-
-    tor_asprintf(answer, "%d", myPid);
-  } else if (!strcmp(question, "process/uid")) {
-#ifdef _WIN32
-      *answer = tor_strdup("-1");
-#else
-      int myUid = geteuid();
-      tor_asprintf(answer, "%d", myUid);
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/user")) {
-#ifdef _WIN32
-      *answer = tor_strdup("");
-#else
-      int myUid = geteuid();
-      const struct passwd *myPwEntry = tor_getpwuid(myUid);
-
-      if (myPwEntry) {
-        *answer = tor_strdup(myPwEntry->pw_name);
-      } else {
-        *answer = tor_strdup("");
-      }
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/descriptor-limit")) {
-    int max_fds = get_max_sockets();
-    tor_asprintf(answer, "%d", max_fds);
-  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
-    tor_asprintf(answer, "%"PRIu64,
-                 (get_options()->MaxMemInQueues));
-  } else if (!strcmp(question, "fingerprint")) {
-    crypto_pk_t *server_key;
-    if (!server_mode(get_options())) {
-      *errmsg = "Not running in server mode";
-      return -1;
-    }
-    server_key = get_server_identity_key();
-    *answer = tor_malloc(HEX_DIGEST_LEN+1);
-    crypto_pk_get_fingerprint(server_key, *answer, 0);
-  }
-  return 0;
-}
-
-/** Awful hack: return a newly allocated string based on a routerinfo and
- * (possibly) an extrainfo, sticking the read-history and write-history from
- * <b>ei</b> into the resulting string.  The thing you get back won't
- * necessarily have a valid signature.
- *
- * New code should never use this; it's for backward compatibility.
- *
- * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
- * not be NUL-terminated. */
-static char *
-munge_extrainfo_into_routerinfo(const char *ri_body,
-                                const signed_descriptor_t *ri,
-                                const signed_descriptor_t *ei)
-{
-  char *out = NULL, *outp;
-  int i;
-  const char *router_sig;
-  const char *ei_body = signed_descriptor_get_body(ei);
-  size_t ri_len = ri->signed_descriptor_len;
-  size_t ei_len = ei->signed_descriptor_len;
-  if (!ei_body)
-    goto bail;
-
-  outp = out = tor_malloc(ri_len+ei_len+1);
-  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
-    goto bail;
-  ++router_sig;
-  memcpy(out, ri_body, router_sig-ri_body);
-  outp += router_sig-ri_body;
-
-  for (i=0; i < 2; ++i) {
-    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
-    const char *cp, *eol;
-    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
-      continue;
-    ++cp;
-    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
-      continue;
-    memcpy(outp, cp, eol-cp+1);
-    outp += eol-cp+1;
-  }
-  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
-  *outp++ = '\0';
-  tor_assert(outp-out < (int)(ri_len+ei_len+1));
-
-  return out;
- bail:
-  tor_free(out);
-  return tor_strndup(ri_body, ri->signed_descriptor_len);
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * which ports are bound. */
-static int
-getinfo_helper_listeners(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  int type;
-  smartlist_t *res;
-
-  (void)control_conn;
-  (void)errmsg;
-
-  if (!strcmp(question, "net/listeners/or"))
-    type = CONN_TYPE_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/extor"))
-    type = CONN_TYPE_EXT_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/dir"))
-    type = CONN_TYPE_DIR_LISTENER;
-  else if (!strcmp(question, "net/listeners/socks"))
-    type = CONN_TYPE_AP_LISTENER;
-  else if (!strcmp(question, "net/listeners/trans"))
-    type = CONN_TYPE_AP_TRANS_LISTENER;
-  else if (!strcmp(question, "net/listeners/natd"))
-    type = CONN_TYPE_AP_NATD_LISTENER;
-  else if (!strcmp(question, "net/listeners/httptunnel"))
-    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
-  else if (!strcmp(question, "net/listeners/dns"))
-    type = CONN_TYPE_AP_DNS_LISTENER;
-  else if (!strcmp(question, "net/listeners/control"))
-    type = CONN_TYPE_CONTROL_LISTENER;
-  else
-    return 0; /* unknown key */
-
-  res = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
-    struct sockaddr_storage ss;
-    socklen_t ss_len = sizeof(ss);
-
-    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
-      continue;
-
-    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
-      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
-    } else {
-      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
-      smartlist_add(res, esc_for_log(tmp));
-      tor_free(tmp);
-    }
-
-  } SMARTLIST_FOREACH_END(conn);
-
-  *answer = smartlist_join_strings(res, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
-  smartlist_free(res);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * the current time in both local and UTC forms. */
-STATIC int
-getinfo_helper_current_time(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-
-  struct timeval now;
-  tor_gettimeofday(&now);
-  char timebuf[ISO_TIME_LEN+1];
-
-  if (!strcmp(question, "current-time/local"))
-    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else if (!strcmp(question, "current-time/utc"))
-    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else
-    return 0;
-
-  *answer = tor_strdup(timebuf);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * directory information. */
-STATIC int
-getinfo_helper_dir(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  (void) control_conn;
-  if (!strcmpstart(question, "desc/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/id/* "
-                "instead of desc/id/*.";
-      return 0;
-    }
-  } else if (!strcmpstart(question, "desc/name/")) {
-    const routerinfo_t *ri = NULL;
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node =
-      node_get_by_nickname(question+strlen("desc/name/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/name/* "
-                "instead of desc/name/*.";
-      return 0;
-    }
-  } else if (!strcmp(question, "desc/download-enabled")) {
-    int r = we_fetch_router_descriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmp(question, "desc/all-recent")) {
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
-      {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        if (body)
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-      });
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
-    /* XXXX Remove this once Torstat asks for extrainfos. */
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
-                                     ri->cache_info.extra_info_digest);
-        if (ei && body) {
-          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
-                                                        &ri->cache_info, ei));
-        } else if (body) {
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-        }
-      } SMARTLIST_FOREACH_END(ri);
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/client/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      rend_cache_entry_t *e = NULL;
-      if (!rend_cache_lookup_entry(question, -1, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      const char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_cache_lookup_encoded_as_client(&service_pk);
-      if (desc) {
-        *answer = tor_strdup(desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/service/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-    rend_cache_entry_t *e = NULL;
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_service_lookup_current_desc(&service_pk);
-      if (desc) {
-        /* Newly allocated string, we have ownership. */
-        *answer = desc;
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "md/all")) {
-    const smartlist_t *nodes = nodelist_get_list();
-    tor_assert(nodes);
-
-    if (smartlist_len(nodes) == 0) {
-      *answer = tor_strdup("");
-      return 0;
-    }
-
-    smartlist_t *microdescs = smartlist_new();
-
-    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
-      if (n->md && n->md->body) {
-        char *copy = tor_strndup(n->md->body, n->md->bodylen);
-        smartlist_add(microdescs, copy);
-      }
-    } SMARTLIST_FOREACH_END(n);
-
-    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
-    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
-    smartlist_free(microdescs);
-  } else if (!strcmpstart(question, "md/id/")) {
-    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmpstart(question, "md/name/")) {
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
-    /* XXXX duplicated code */
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmp(question, "md/download-enabled")) {
-    int r = we_fetch_microdescriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmpstart(question, "desc-annotations/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node =
-      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *annotations =
-        signed_descriptor_get_annotations(&ri->cache_info);
-      if (annotations)
-        *answer = tor_strndup(annotations,
-                              ri->cache_info.annotations_len);
-    }
-  } else if (!strcmpstart(question, "dir/server/")) {
-    size_t answer_len = 0;
-    char *url = NULL;
-    smartlist_t *descs = smartlist_new();
-    const char *msg;
-    int res;
-    char *cp;
-    tor_asprintf(&url, "/tor/%s", question+4);
-    res = dirserv_get_routerdescs(descs, url, &msg);
-    if (res) {
-      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
-      smartlist_free(descs);
-      tor_free(url);
-      *errmsg = msg;
-      return -1;
-    }
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      answer_len += sd->signed_descriptor_len);
-    cp = *answer = tor_malloc(answer_len+1);
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      {
-                        memcpy(cp, signed_descriptor_get_body(sd),
-                               sd->signed_descriptor_len);
-                        cp += sd->signed_descriptor_len;
-                      });
-    *cp = '\0';
-    tor_free(url);
-    smartlist_free(descs);
-  } else if (!strcmpstart(question, "dir/status/")) {
-    *answer = tor_strdup("");
-  } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
-    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
-      const cached_dir_t *consensus = dirserv_get_consensus("ns");
-      if (consensus)
-        *answer = tor_strdup(consensus->dir);
-    }
-    if (!*answer) { /* try loading it from disk */
-      *answer = networkstatus_read_cached_consensus("ns");
-      if (!*answer) { /* generate an error */
-        *errmsg = "Could not open cached consensus. "
-          "Make sure FetchUselessDescriptors is set to 1.";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "network-status")) { /* v1 */
-    static int network_status_warned = 0;
-    if (!network_status_warned) {
-      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
-               "go away in a future version of Tor.");
-      network_status_warned = 1;
-    }
-    routerlist_t *routerlist = router_get_routerlist();
-    if (!routerlist || !routerlist->routers ||
-        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
-      return -1;
-    }
-  } else if (!strcmpstart(question, "extra-info/digest/")) {
-    question += strlen("extra-info/digest/");
-    if (strlen(question) == HEX_DIGEST_LEN) {
-      char d[DIGEST_LEN];
-      signed_descriptor_t *sd = NULL;
-      if (base16_decode(d, sizeof(d), question, strlen(question))
-                        == sizeof(d)) {
-        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
-         * but I don't want to risk affecting other parts of the code,
-         * especially since the rules for using our own extrainfo (including
-         * when it might be freed) are different from those for using one
-         * we have downloaded. */
-        if (router_extrainfo_digest_is_me(d))
-          sd = &(router_get_my_extrainfo()->cache_info);
-        else
-          sd = extrainfo_get_by_descriptor_digest(d);
-      }
-      if (sd) {
-        const char *body = signed_descriptor_get_body(sd);
-        if (body)
-          *answer = tor_strndup(body, sd->signed_descriptor_len);
-      }
-    }
-  }
-
-  return 0;
-}
-
-/** Given a smartlist of 20-byte digests, return a newly allocated string
- * containing each of those digests in order, formatted in HEX, and terminated
- * with a newline. */
-static char *
-digest_list_to_string(const smartlist_t *sl)
-{
-  int len;
-  char *result, *s;
-
-  /* Allow for newlines, and a \0 at the end */
-  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
-  result = tor_malloc_zero(len);
-
-  s = result;
-  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
-    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
-    s[HEX_DIGEST_LEN] = '\n';
-    s += HEX_DIGEST_LEN + 1;
-  } SMARTLIST_FOREACH_END(digest);
-  *s = '\0';
-
-  return result;
-}
-
-/** Turn a download_status_t into a human-readable description in a newly
- * allocated string.  The format is specified in control-spec.txt, under
- * the documentation for "GETINFO download/..." .  */
-static char *
-download_status_to_string(const download_status_t *dl)
-{
-  char *rv = NULL;
-  char tbuf[ISO_TIME_LEN+1];
-  const char *schedule_str, *want_authority_str;
-  const char *increment_on_str, *backoff_str;
-
-  if (dl) {
-    /* Get some substrings of the eventual output ready */
-    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
-
-    switch (dl->schedule) {
-      case DL_SCHED_GENERIC:
-        schedule_str = "DL_SCHED_GENERIC";
-        break;
-      case DL_SCHED_CONSENSUS:
-        schedule_str = "DL_SCHED_CONSENSUS";
-        break;
-      case DL_SCHED_BRIDGE:
-        schedule_str = "DL_SCHED_BRIDGE";
-        break;
-      default:
-        schedule_str = "unknown";
-        break;
-    }
-
-    switch (dl->want_authority) {
-      case DL_WANT_ANY_DIRSERVER:
-        want_authority_str = "DL_WANT_ANY_DIRSERVER";
-        break;
-      case DL_WANT_AUTHORITY:
-        want_authority_str = "DL_WANT_AUTHORITY";
-        break;
-      default:
-        want_authority_str = "unknown";
-        break;
-    }
-
-    switch (dl->increment_on) {
-      case DL_SCHED_INCREMENT_FAILURE:
-        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
-        break;
-      case DL_SCHED_INCREMENT_ATTEMPT:
-        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
-        break;
-      default:
-        increment_on_str = "unknown";
-        break;
-    }
-
-    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
-
-    /* Now assemble them */
-    tor_asprintf(&rv,
-                 "next-attempt-at %s\n"
-                 "n-download-failures %u\n"
-                 "n-download-attempts %u\n"
-                 "schedule %s\n"
-                 "want-authority %s\n"
-                 "increment-on %s\n"
-                 "backoff %s\n"
-                 "last-backoff-position %u\n"
-                 "last-delay-used %d\n",
-                 tbuf,
-                 dl->n_download_failures,
-                 dl->n_download_attempts,
-                 schedule_str,
-                 want_authority_str,
-                 increment_on_str,
-                 backoff_str,
-                 dl->last_backoff_position,
-                 dl->last_delay_used);
-  }
-
-  return rv;
-}
-
-/** Handle the consensus download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_networkstatus(const char *flavor,
-                                       download_status_t **dl_to_emit,
-                                       const char **errmsg)
-{
-  /*
-   * We get the one for the current bootstrapped status by default, or
-   * take an extra /bootstrap or /running suffix
-   */
-  if (strcmp(flavor, "ns") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
-  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
-  } else if (strcmp(flavor, "ns/running") == 0 ) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
-  } else if (strcmp(flavor, "microdesc") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/running") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
-  } else {
-    *errmsg = "Unknown flavor";
-  }
-}
-
-/** Handle the cert download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_cert(const char *fp_sk_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  const char *sk_req;
-  char id_digest[DIGEST_LEN];
-  char sk_digest[DIGEST_LEN];
-
-  /*
-   * We have to handle four cases; fp_sk_req is the request with
-   * a prefix of "downloads/cert/" snipped off.
-   *
-   * Case 1: fp_sk_req = "fps"
-   *  - We should emit a digest_list with a list of all the identity
-   *    fingerprints that can be queried for certificate download status;
-   *    get it by calling list_authority_ids_with_downloads().
-   *
-   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
-   *  - We want the default certificate for this identity fingerprint's
-   *    download status; this is the download we get from URLs starting
-   *    in /fp/ on the directory server.  We can get it with
-   *    id_only_download_status_for_authority_id().
-   *
-   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
-   *  - We want a list of all signing key digests for this identity
-   *    fingerprint which can be queried for certificate download status.
-   *    Get it with list_sk_digests_for_authority_id().
-   *
-   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
-   *         signing key digest sk
-   *   - We want the download status for the certificate for this specific
-   *     signing key and fingerprint.  These correspond to the ones we get
-   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
-   *     list_sk_digests_for_authority_id().
-   */
-
-  if (strcmp(fp_sk_req, "fps") == 0) {
-    *digest_list = list_authority_ids_with_downloads();
-    if (!(*digest_list)) {
-      *errmsg = "Failed to get list of authority identity digests (!)";
-    }
-  } else if (!strcmpstart(fp_sk_req, "fp/")) {
-    fp_sk_req += strlen("fp/");
-    /* Okay, look for another / to tell the fp from fp-sk cases */
-    sk_req = strchr(fp_sk_req, '/');
-    if (sk_req) {
-      /* okay, split it here and try to parse <fp> */
-      if (base16_decode(id_digest, DIGEST_LEN,
-                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
-        /* Skip past the '/' */
-        ++sk_req;
-        if (strcmp(sk_req, "sks") == 0) {
-          /* We're asking for the list of signing key fingerprints */
-          *digest_list = list_sk_digests_for_authority_id(id_digest);
-          if (!(*digest_list)) {
-            *errmsg = "Failed to get list of signing key digests for this "
-                      "authority identity digest";
-          }
-        } else {
-          /* We've got a signing key digest */
-          if (base16_decode(sk_digest, DIGEST_LEN,
-                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
-            *dl_to_emit =
-              download_status_for_authority_id_and_sk(id_digest, sk_digest);
-            if (!(*dl_to_emit)) {
-              *errmsg = "Failed to get download status for this identity/"
-                        "signing key digest pair";
-            }
-          } else {
-            *errmsg = "That didn't look like a signing key digest";
-          }
-        }
-      } else {
-        *errmsg = "That didn't look like an identity digest";
-      }
-    } else {
-      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
-      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
-        if (base16_decode(id_digest, DIGEST_LEN,
-                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
-          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
-          if (!(*dl_to_emit)) {
-            *errmsg = "Failed to get download status for this authority "
-                      "identity digest";
-          }
-        } else {
-          *errmsg = "That didn't look like a digest";
-        }
-      } else {
-        *errmsg = "That didn't look like a digest";
-      }
-    }
-  } else {
-    *errmsg = "Unknown certificate download status query";
-  }
-}
-
-/** Handle the routerdesc download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_desc(const char *desc_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  char desc_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: desc_req = "descs"
-   *   - Emit a list of all router descriptor digests, which we get by
-   *     calling router_get_descriptor_digests(); this can return NULL
-   *     if we have no current ns-flavor consensus.
-   *
-   * Case 2: desc_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using router_get_dl_status_by_descriptor_digest().
-   */
-
-  if (strcmp(desc_req, "descs") == 0) {
-    *digest_list = router_get_descriptor_digests();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
-    }
-    /*
-     * Microdescs don't use the download_status_t mechanism, so we don't
-     * answer queries about their downloads here; see microdesc.c.
-     */
-  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(desc_digest, DIGEST_LEN,
-                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such descriptor digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown router descriptor download status query";
-  }
-}
-
-/** Handle the bridge download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_bridge(const char *bridge_req,
-                                download_status_t **dl_to_emit,
-                                smartlist_t **digest_list,
-                                const char **errmsg)
-{
-  char bridge_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: bridge_req = "bridges"
-   *   - Emit a list of all bridge identity digests, which we get by
-   *     calling list_bridge_identities(); this can return NULL if we are
-   *     not using bridges.
-   *
-   * Case 2: bridge_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using get_bridge_dl_status_by_id().
-   */
-
-  if (strcmp(bridge_req, "bridges") == 0) {
-    *digest_list = list_bridge_identities();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to be using bridges";
-    }
-  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(bridge_digest, DIGEST_LEN,
-                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such bridge identity digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown bridge descriptor download status query";
-  }
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * download status information. */
-STATIC int
-getinfo_helper_downloads(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  download_status_t *dl_to_emit = NULL;
-  smartlist_t *digest_list = NULL;
-
-  /* Assert args are sane */
-  tor_assert(control_conn != NULL);
-  tor_assert(question != NULL);
-  tor_assert(answer != NULL);
-  tor_assert(errmsg != NULL);
-
-  /* We check for this later to see if we should supply a default */
-  *errmsg = NULL;
-
-  /* Are we after networkstatus downloads? */
-  if (!strcmpstart(question, "downloads/networkstatus/")) {
-    getinfo_helper_downloads_networkstatus(
-        question + strlen("downloads/networkstatus/"),
-        &dl_to_emit, errmsg);
-  /* Certificates? */
-  } else if (!strcmpstart(question, "downloads/cert/")) {
-    getinfo_helper_downloads_cert(
-        question + strlen("downloads/cert/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Router descriptors? */
-  } else if (!strcmpstart(question, "downloads/desc/")) {
-    getinfo_helper_downloads_desc(
-        question + strlen("downloads/desc/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Bridge descriptors? */
-  } else if (!strcmpstart(question, "downloads/bridge/")) {
-    getinfo_helper_downloads_bridge(
-        question + strlen("downloads/bridge/"),
-        &dl_to_emit, &digest_list, errmsg);
-  } else {
-    *errmsg = "Unknown download status query";
-  }
-
-  if (dl_to_emit) {
-    *answer = download_status_to_string(dl_to_emit);
-
-    return 0;
-  } else if (digest_list) {
-    *answer = digest_list_to_string(digest_list);
-    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
-    smartlist_free(digest_list);
-
-    return 0;
-  } else {
-    if (!(*errmsg)) {
-      *errmsg = "Unknown error";
-    }
-
-    return -1;
-  }
-}
-
-/** Allocate and return a description of <b>circ</b>'s current status,
- * including its path (if any). */
-static char *
-circuit_describe_status_for_controller(origin_circuit_t *circ)
-{
-  char *rv;
-  smartlist_t *descparts = smartlist_new();
-
-  {
-    char *vpath = circuit_list_path_for_controller(circ);
-    if (*vpath) {
-      smartlist_add(descparts, vpath);
-    } else {
-      tor_free(vpath); /* empty path; don't put an extra space in the result */
-    }
-  }
-
-  {
-    cpath_build_state_t *build_state = circ->build_state;
-    smartlist_t *flaglist = smartlist_new();
-    char *flaglist_joined;
-
-    if (build_state->onehop_tunnel)
-      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
-    if (build_state->is_internal)
-      smartlist_add(flaglist, (void *)"IS_INTERNAL");
-    if (build_state->need_capacity)
-      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
-    if (build_state->need_uptime)
-      smartlist_add(flaglist, (void *)"NEED_UPTIME");
-
-    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
-    if (smartlist_len(flaglist)) {
-      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
-
-      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
-
-      tor_free(flaglist_joined);
-    }
-
-    smartlist_free(flaglist);
-  }
-
-  smartlist_add_asprintf(descparts, "PURPOSE=%s",
-                    circuit_purpose_to_controller_string(circ->base_.purpose));
-
-  {
-    const char *hs_state =
-      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
-
-    if (hs_state != NULL) {
-      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
-    }
-  }
-
-  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
-    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
-    const char *onion_address;
-    if (circ->rend_data) {
-      onion_address = rend_data_get_address(circ->rend_data);
-    } else {
-      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
-      onion_address = addr;
-    }
-    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
-  }
-
-  {
-    char tbuf[ISO_TIME_USEC_LEN+1];
-    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
-
-    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
-  }
-
-  // Show username and/or password if available.
-  if (circ->socks_username_len > 0) {
-    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
-                                     (size_t) circ->socks_username_len);
-    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
-                           socks_username_escaped);
-    tor_free(socks_username_escaped);
-  }
-  if (circ->socks_password_len > 0) {
-    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
-                                     (size_t) circ->socks_password_len);
-    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
-                           socks_password_escaped);
-    tor_free(socks_password_escaped);
-  }
-
-  rv = smartlist_join_strings(descparts, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
-  smartlist_free(descparts);
-
-  return rv;
-}
-
-/** Implementation helper for GETINFO: knows how to generate summaries of the
- * current states of things we send events about. */
-static int
-getinfo_helper_events(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  const or_options_t *options = get_options();
-  (void) control_conn;
-  if (!strcmp(question, "circuit-status")) {
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
-      origin_circuit_t *circ;
-      char *circdesc;
-      const char *state;
-      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
-        continue;
-      circ = TO_ORIGIN_CIRCUIT(circ_);
-
-      if (circ->base_.state == CIRCUIT_STATE_OPEN)
-        state = "BUILT";
-      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
-        state = "GUARD_WAIT";
-      else if (circ->cpath)
-        state = "EXTENDED";
-      else
-        state = "LAUNCHED";
-
-      circdesc = circuit_describe_status_for_controller(circ);
-
-      smartlist_add_asprintf(status, "%lu %s%s%s",
-                   (unsigned long)circ->global_identifier,
-                   state, *circdesc ? " " : "", circdesc);
-      tor_free(circdesc);
-    }
-    SMARTLIST_FOREACH_END(circ_);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "stream-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    char buf[256];
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      entry_connection_t *conn;
-      circuit_t *circ;
-      origin_circuit_t *origin_circ = NULL;
-      if (base_conn->type != CONN_TYPE_AP ||
-          base_conn->marked_for_close ||
-          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
-          base_conn->state == AP_CONN_STATE_NATD_WAIT)
-        continue;
-      conn = TO_ENTRY_CONN(base_conn);
-      switch (base_conn->state)
-        {
-        case AP_CONN_STATE_CONTROLLER_WAIT:
-        case AP_CONN_STATE_CIRCUIT_WAIT:
-          if (conn->socks_request &&
-              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
-            state = "NEWRESOLVE";
-          else
-            state = "NEW";
-          break;
-        case AP_CONN_STATE_RENDDESC_WAIT:
-        case AP_CONN_STATE_CONNECT_WAIT:
-          state = "SENTCONNECT"; break;
-        case AP_CONN_STATE_RESOLVE_WAIT:
-          state = "SENTRESOLVE"; break;
-        case AP_CONN_STATE_OPEN:
-          state = "SUCCEEDED"; break;
-        default:
-          log_warn(LD_BUG, "Asked for stream in unknown state %d",
-                   base_conn->state);
-          continue;
-        }
-      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-      if (circ && CIRCUIT_IS_ORIGIN(circ))
-        origin_circ = TO_ORIGIN_CIRCUIT(circ);
-      write_stream_target_to_buf(conn, buf, sizeof(buf));
-      smartlist_add_asprintf(status, "%lu %s %lu %s",
-                   (unsigned long) base_conn->global_identifier,state,
-                   origin_circ?
-                         (unsigned long)origin_circ->global_identifier : 0ul,
-                   buf);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "orconn-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      char name[128];
-      or_connection_t *conn;
-      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
-        continue;
-      conn = TO_OR_CONN(base_conn);
-      if (conn->base_.state == OR_CONN_STATE_OPEN)
-        state = "CONNECTED";
-      else if (conn->nickname)
-        state = "LAUNCHED";
-      else
-        state = "NEW";
-      orconn_target_get_name(name, sizeof(name), conn);
-      smartlist_add_asprintf(status, "%s %s", name, state);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmpstart(question, "address-mappings/")) {
-    time_t min_e, max_e;
-    smartlist_t *mappings;
-    question += strlen("address-mappings/");
-    if (!strcmp(question, "all")) {
-      min_e = 0; max_e = TIME_MAX;
-    } else if (!strcmp(question, "cache")) {
-      min_e = 2; max_e = TIME_MAX;
-    } else if (!strcmp(question, "config")) {
-      min_e = 0; max_e = 0;
-    } else if (!strcmp(question, "control")) {
-      min_e = 1; max_e = 1;
-    } else {
-      return 0;
-    }
-    mappings = smartlist_new();
-    addressmap_get_mappings(mappings, min_e, max_e, 1);
-    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
-    smartlist_free(mappings);
-  } else if (!strcmpstart(question, "status/")) {
-    /* Note that status/ is not a catch-all for events; there's only supposed
-     * to be a status GETINFO if there's a corresponding STATUS event. */
-    if (!strcmp(question, "status/circuit-established")) {
-      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
-    } else if (!strcmp(question, "status/enough-dir-info")) {
-      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
-    } else if (!strcmp(question, "status/good-server-descriptor") ||
-               !strcmp(question, "status/accepted-server-descriptor")) {
-      /* They're equivalent for now, until we can figure out how to make
-       * good-server-descriptor be what we want. See comment in
-       * control-spec.txt. */
-      *answer = tor_strdup(directories_have_accepted_server_descriptor()
-                           ? "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
-      *answer = tor_strdup(check_whether_orport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
-      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded")) {
-      tor_asprintf(answer, "OR=%d DIR=%d",
-                   check_whether_orport_reachable(options) ? 1 : 0,
-                   check_whether_dirport_reachable(options) ? 1 : 0);
-    } else if (!strcmp(question, "status/bootstrap-phase")) {
-      *answer = tor_strdup(last_sent_bootstrap_message);
-    } else if (!strcmpstart(question, "status/version/")) {
-      int is_server = server_mode(options);
-      networkstatus_t *c = networkstatus_get_latest_consensus();
-      version_status_t status;
-      const char *recommended;
-      if (c) {
-        recommended = is_server ? c->server_versions : c->client_versions;
-        status = tor_version_is_obsolete(VERSION, recommended);
-      } else {
-        recommended = "?";
-        status = VS_UNKNOWN;
-      }
-
-      if (!strcmp(question, "status/version/recommended")) {
-        *answer = tor_strdup(recommended);
-        return 0;
-      }
-      if (!strcmp(question, "status/version/current")) {
-        switch (status)
-          {
-          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
-          case VS_OLD: *answer = tor_strdup("obsolete"); break;
-          case VS_NEW: *answer = tor_strdup("new"); break;
-          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
-          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
-          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
-          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
-          default: tor_fragile_assert();
-          }
-      } else if (!strcmp(question, "status/version/num-versioning") ||
-                 !strcmp(question, "status/version/num-concurring")) {
-        tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO));
-        log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful "
-                 "information", question);
-      }
-    } else if (!strcmp(question, "status/clients-seen")) {
-      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
-      if (!bridge_stats) {
-        *errmsg = "No bridge-client stats available";
-        return -1;
-      }
-      *answer = bridge_stats;
-    } else if (!strcmp(question, "status/fresh-relay-descs")) {
-      if (!server_mode(options)) {
-        *errmsg = "Only relays have descriptors";
-        return -1;
-      }
-      routerinfo_t *r;
-      extrainfo_t *e;
-      if (router_build_fresh_descriptor(&r, &e) < 0) {
-        *errmsg = "Error generating descriptor";
-        return -1;
-      }
-      size_t size = r->cache_info.signed_descriptor_len + 1;
-      if (e) {
-        size += e->cache_info.signed_descriptor_len + 1;
-      }
-      tor_assert(r->cache_info.signed_descriptor_len);
-      char *descs = tor_malloc(size);
-      char *cp = descs;
-      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
-             r->cache_info.signed_descriptor_len);
-      cp += r->cache_info.signed_descriptor_len - 1;
-      if (e) {
-        if (cp[0] == '\0') {
-          cp[0] = '\n';
-        } else if (cp[0] != '\n') {
-          cp[1] = '\n';
-          cp++;
-        }
-        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
-               e->cache_info.signed_descriptor_len);
-        cp += e->cache_info.signed_descriptor_len - 1;
-      }
-      if (cp[0] == '\n') {
-        cp[0] = '\0';
-      } else if (cp[0] != '\0') {
-        cp[1] = '\0';
-      }
-      *answer = descs;
-      routerinfo_free(r);
-      extrainfo_free(e);
-    } else {
-      return 0;
-    }
-  }
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows how to enumerate hidden services
- * created via the control port. */
-STATIC int
-getinfo_helper_onions(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  smartlist_t *onion_list = NULL;
-  (void) errmsg;  /* no errors from this method */
-
-  if (control_conn && !strcmp(question, "onions/current")) {
-    onion_list = control_conn->ephemeral_onion_services;
-  } else if (!strcmp(question, "onions/detached")) {
-    onion_list = detached_onion_services;
-  } else {
-    return 0;
-  }
-  if (!onion_list || smartlist_len(onion_list) == 0) {
-    if (answer) {
-      *answer = tor_strdup("");
-    }
-  } else {
-    if (answer) {
-      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about network
- * liveness. */
-static int
-getinfo_helper_liveness(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-  if (strcmp(question, "network-liveness") == 0) {
-    if (get_cached_network_liveness()) {
-      *answer = tor_strdup("up");
-    } else {
-      *answer = tor_strdup("down");
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about shared random
- * value. */
-static int
-getinfo_helper_sr(control_connection_t *control_conn,
-                  const char *question, char **answer,
-                  const char **errmsg)
-{
-  (void) control_conn;
-  (void) errmsg;
-
-  if (!strcmp(question, "sr/current")) {
-    *answer = sr_get_current_for_control();
-  } else if (!strcmp(question, "sr/previous")) {
-    *answer = sr_get_previous_for_control();
-  }
-  /* Else statement here is unrecognized key so do nothing. */
-
-  return 0;
-}
-
-/** Callback function for GETINFO: on a given control connection, try to
- * answer the question <b>q</b> and store the newly-allocated answer in
- * *<b>a</b>. If an internal error occurs, return -1 and optionally set
- * *<b>error_out</b> to point to an error message to be delivered to the
- * controller. On success, _or if the key is not recognized_, return 0. Do not
- * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
- * to improve the error message.
- */
-typedef int (*getinfo_helper_t)(control_connection_t *,
-                                const char *q, char **a,
-                                const char **error_out);
-
-/** A single item for the GETINFO question-to-answer-function table. */
-typedef struct getinfo_item_t {
-  const char *varname; /**< The value (or prefix) of the question. */
-  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
-                        * this entry is documentation-only. */
-  const char *desc; /**< Description of the variable. */
-  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
-} getinfo_item_t;
-
-#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
-#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
-#define DOC(name, desc) { name, NULL, desc, 0 }
-
-/** Table mapping questions accepted by GETINFO to the functions that know how
- * to answer them. */
-static const getinfo_item_t getinfo_items[] = {
-  ITEM("version", misc, "The current version of Tor."),
-  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
-  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
-  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
-  ITEM("config-text", misc,
-       "Return the string that would be written by a saveconf command."),
-  ITEM("config-can-saveconf", misc,
-       "Is it possible to save the configuration to the \"torrc\" file?"),
-  ITEM("accounting/bytes", accounting,
-       "Number of bytes read/written so far in the accounting interval."),
-  ITEM("accounting/bytes-left", accounting,
-      "Number of bytes left to write/read so far in the accounting interval."),
-  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
-  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
-  ITEM("accounting/interval-start", accounting,
-       "Time when the accounting period starts."),
-  ITEM("accounting/interval-end", accounting,
-       "Time when the accounting period ends."),
-  ITEM("accounting/interval-wake", accounting,
-       "Time to wake up in this accounting period."),
-  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
-  ITEM("entry-guards", entry_guards,
-       "Which nodes are we using as entry guards?"),
-  ITEM("fingerprint", misc, NULL),
-  PREFIX("config/", config, "Current configuration values."),
-  DOC("config/names",
-      "List of configuration options, types, and documentation."),
-  DOC("config/defaults",
-      "List of default values for configuration options. "
-      "See also config/names"),
-  PREFIX("current-time/", current_time, "Current time."),
-  DOC("current-time/local", "Current time on the local system."),
-  DOC("current-time/utc", "Current UTC time."),
-  PREFIX("downloads/networkstatus/", downloads,
-         "Download statuses for networkstatus objects"),
-  DOC("downloads/networkstatus/ns",
-      "Download status for current-mode networkstatus download"),
-  DOC("downloads/networkstatus/ns/bootstrap",
-      "Download status for bootstrap-time networkstatus download"),
-  DOC("downloads/networkstatus/ns/running",
-      "Download status for run-time networkstatus download"),
-  DOC("downloads/networkstatus/microdesc",
-      "Download status for current-mode microdesc download"),
-  DOC("downloads/networkstatus/microdesc/bootstrap",
-      "Download status for bootstrap-time microdesc download"),
-  DOC("downloads/networkstatus/microdesc/running",
-      "Download status for run-time microdesc download"),
-  PREFIX("downloads/cert/", downloads,
-         "Download statuses for certificates, by id fingerprint and "
-         "signing key"),
-  DOC("downloads/cert/fps",
-      "List of authority fingerprints for which any download statuses "
-      "exist"),
-  DOC("downloads/cert/fp/<fp>",
-      "Download status for <fp> with the default signing key; corresponds "
-      "to /fp/ URLs on directory server."),
-  DOC("downloads/cert/fp/<fp>/sks",
-      "List of signing keys for which specific download statuses are "
-      "available for this id fingerprint"),
-  DOC("downloads/cert/fp/<fp>/<sk>",
-      "Download status for <fp> with signing key <sk>; corresponds "
-      "to /fp-sk/ URLs on directory server."),
-  PREFIX("downloads/desc/", downloads,
-         "Download statuses for router descriptors, by descriptor digest"),
-  DOC("downloads/desc/descs",
-      "Return a list of known router descriptor digests"),
-  DOC("downloads/desc/<desc>",
-      "Return a download status for a given descriptor digest"),
-  PREFIX("downloads/bridge/", downloads,
-         "Download statuses for bridge descriptors, by bridge identity "
-         "digest"),
-  DOC("downloads/bridge/bridges",
-      "Return a list of configured bridge identity digests with download "
-      "statuses"),
-  DOC("downloads/bridge/<desc>",
-      "Return a download status for a given bridge identity digest"),
-  ITEM("info/names", misc,
-       "List of GETINFO options, types, and documentation."),
-  ITEM("events/names", misc,
-       "Events that the controller can ask for with SETEVENTS."),
-  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
-  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
-  PREFIX("desc/id/", dir, "Router descriptors by ID."),
-  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
-  ITEM("desc/all-recent", dir,
-       "All non-expired, non-superseded router descriptors."),
-  ITEM("desc/download-enabled", dir,
-       "Do we try to download router descriptors?"),
-  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
-  ITEM("md/all", dir, "All known microdescriptors."),
-  PREFIX("md/id/", dir, "Microdescriptors by ID"),
-  PREFIX("md/name/", dir, "Microdescriptors by name"),
-  ITEM("md/download-enabled", dir,
-       "Do we try to download microdescriptors?"),
-  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
-  PREFIX("hs/client/desc/id", dir,
-         "Hidden Service descriptor in client's cache by onion."),
-  PREFIX("hs/service/desc/id/", dir,
-         "Hidden Service descriptor in services's cache by onion."),
-  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
-  ITEM("ns/all", networkstatus,
-       "Brief summary of router status (v2 directory format)"),
-  PREFIX("ns/id/", networkstatus,
-         "Brief summary of router status by ID (v2 directory format)."),
-  PREFIX("ns/name/", networkstatus,
-         "Brief summary of router status by nickname (v2 directory format)."),
-  PREFIX("ns/purpose/", networkstatus,
-         "Brief summary of router status by purpose (v2 directory format)."),
-  PREFIX("consensus/", networkstatus,
-         "Information about and from the ns consensus."),
-  ITEM("network-status", dir,
-       "Brief summary of router status (v1 directory format)"),
-  ITEM("network-liveness", liveness,
-       "Current opinion on whether the network is live"),
-  ITEM("circuit-status", events, "List of current circuits originating here."),
-  ITEM("stream-status", events,"List of current streams."),
-  ITEM("orconn-status", events, "A list of current OR connections."),
-  ITEM("dormant", misc,
-       "Is Tor dormant (not building circuits because it's idle)?"),
-  PREFIX("address-mappings/", events, NULL),
-  DOC("address-mappings/all", "Current address mappings."),
-  DOC("address-mappings/cache", "Current cached DNS replies."),
-  DOC("address-mappings/config",
-      "Current address mappings from configuration."),
-  DOC("address-mappings/control", "Current address mappings from controller."),
-  PREFIX("status/", events, NULL),
-  DOC("status/circuit-established",
-      "Whether we think client functionality is working."),
-  DOC("status/enough-dir-info",
-      "Whether we have enough up-to-date directory information to build "
-      "circuits."),
-  DOC("status/bootstrap-phase",
-      "The last bootstrap phase status event that Tor sent."),
-  DOC("status/clients-seen",
-      "Breakdown of client countries seen by a bridge."),
-  DOC("status/fresh-relay-descs",
-      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
-  DOC("status/version/recommended", "List of currently recommended versions."),
-  DOC("status/version/current", "Status of the current version."),
-  DOC("status/version/num-versioning", "Number of versioning authorities."),
-  DOC("status/version/num-concurring",
-      "Number of versioning authorities agreeing on the status of the "
-      "current version"),
-  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
-  ITEM("traffic/read", misc,"Bytes read since the process was started."),
-  ITEM("traffic/written", misc,
-       "Bytes written since the process was started."),
-  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
-  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
-  ITEM("process/uid", misc, "User id running the tor process."),
-  ITEM("process/user", misc,
-       "Username under which the tor process is running."),
-  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
-  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
-  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
-  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
-  PREFIX("dir/status/", dir,
-         "v2 networkstatus docs as retrieved from a DirPort."),
-  ITEM("dir/status-vote/current/consensus", dir,
-       "v3 Networkstatus consensus as retrieved from a DirPort."),
-  ITEM("exit-policy/default", policies,
-       "The default value appended to the configured exit policy."),
-  ITEM("exit-policy/reject-private/default", policies,
-       "The default rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate."),
-  ITEM("exit-policy/reject-private/relay", policies,
-       "The relay-specific rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
-  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
-  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
-  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
-  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
-  ITEM("onions/current", onions,
-       "Onion services owned by the current control connection."),
-  ITEM("onions/detached", onions,
-       "Onion services detached from the control connection."),
-  ITEM("sr/current", sr, "Get current shared random value."),
-  ITEM("sr/previous", sr, "Get previous shared random value."),
-  { NULL, NULL, NULL, 0 }
-};
-
-/** Allocate and return a list of recognized GETINFO options. */
-static char *
-list_getinfo_options(void)
-{
-  int i;
-  smartlist_t *lines = smartlist_new();
-  char *ans;
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    if (!getinfo_items[i].desc)
-      continue;
-
-    smartlist_add_asprintf(lines, "%s%s -- %s\n",
-                 getinfo_items[i].varname,
-                 getinfo_items[i].is_prefix ? "*" : "",
-                 getinfo_items[i].desc);
-  }
-  smartlist_sort_strings(lines);
-
-  ans = smartlist_join_strings(lines, "", 0, NULL);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-
-  return ans;
-}
-
-/** Lookup the 'getinfo' entry <b>question</b>, and return
- * the answer in <b>*answer</b> (or NULL if key not recognized).
- * Return 0 if success or unrecognized, or -1 if recognized but
- * internal error. */
-static int
-handle_getinfo_helper(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **err_out)
-{
-  int i;
-  *answer = NULL; /* unrecognized key by default */
-
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    int match;
-    if (getinfo_items[i].is_prefix)
-      match = !strcmpstart(question, getinfo_items[i].varname);
-    else
-      match = !strcmp(question, getinfo_items[i].varname);
-    if (match) {
-      tor_assert(getinfo_items[i].fn);
-      return getinfo_items[i].fn(control_conn, question, answer, err_out);
-    }
-  }
-
-  return 0; /* unrecognized */
-}
-
-/** Called when we receive a GETINFO command.  Try to fetch all requested
- * information, and reply with information or error message. */
-static int
-handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *ans = NULL;
-  int i;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    const char *errmsg = NULL;
-
-    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
-      if (!errmsg)
-        errmsg = "Internal error";
-      connection_printf_to_buf(conn, "551 %s\r\n", errmsg);
-      goto done;
-    }
-    if (!ans) {
-      if (errmsg) /* use provided error message */
-        smartlist_add_strdup(unrecognized, errmsg);
-      else /* use default error message */
-        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
-    } else {
-      smartlist_add_strdup(answers, q);
-      smartlist_add(answers, ans);
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if (smartlist_len(unrecognized)) {
-    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
-    for (i=0; i < smartlist_len(unrecognized)-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-%s\r\n",
-                               (char *)smartlist_get(unrecognized, i));
-
-    connection_printf_to_buf(conn,
-                             "552 %s\r\n",
-                             (char *)smartlist_get(unrecognized, i));
-    goto done;
-  }
-
-  for (i = 0; i < smartlist_len(answers); i += 2) {
-    char *k = smartlist_get(answers, i);
-    char *v = smartlist_get(answers, i+1);
-    if (!strchr(v, '\n') && !strchr(v, '\r')) {
-      connection_printf_to_buf(conn, "250-%s=", k);
-      connection_write_str_to_buf(v, conn);
-      connection_write_str_to_buf("\r\n", conn);
-    } else {
-      char *esc = NULL;
-      size_t esc_len;
-      esc_len = write_escaped_data(v, strlen(v), &esc);
-      connection_printf_to_buf(conn, "250+%s=\r\n", k);
-      connection_buf_add(esc, esc_len, TO_CONN(conn));
-      tor_free(esc);
-    }
-  }
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
-  smartlist_free(unrecognized);
-
-  return 0;
-}
-
-/** Given a string, convert it to a circuit purpose. */
-static uint8_t
-circuit_purpose_from_string(const char *string)
-{
-  if (!strcasecmpstart(string, "purpose="))
-    string += strlen("purpose=");
-
-  if (!strcasecmp(string, "general"))
-    return CIRCUIT_PURPOSE_C_GENERAL;
-  else if (!strcasecmp(string, "controller"))
-    return CIRCUIT_PURPOSE_CONTROLLER;
-  else
-    return CIRCUIT_PURPOSE_UNKNOWN;
-}
-
-/** Return a newly allocated smartlist containing the arguments to the command
- * waiting in <b>body</b>. If there are fewer than <b>min_args</b> arguments,
- * or if <b>max_args</b> is nonnegative and there are more than
- * <b>max_args</b> arguments, send a 512 error to the controller, using
- * <b>command</b> as the command name in the error message. */
-static smartlist_t *
-getargs_helper(const char *command, control_connection_t *conn,
-               const char *body, int min_args, int max_args)
-{
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(args) < min_args) {
-    connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command);
-    goto err;
-  } else if (max_args >= 0 && smartlist_len(args) > max_args) {
-    connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command);
-    goto err;
-  }
-  return args;
- err:
-  SMARTLIST_FOREACH(args, char *, s, tor_free(s));
-  smartlist_free(args);
-  return NULL;
-}
-
-/** Helper.  Return the first element of <b>sl</b> at index <b>start_at</b> or
- * higher that starts with <b>prefix</b>, case-insensitive.  Return NULL if no
- * such element exists. */
-static const char *
-find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix)
-{
-  int i;
-  for (i = start_at; i < smartlist_len(sl); ++i) {
-    const char *elt = smartlist_get(sl, i);
-    if (!strcasecmpstart(elt, prefix))
-      return elt;
-  }
-  return NULL;
-}
-
-/** Helper.  Return true iff s is an argument that we should treat as a
- * key-value pair. */
-static int
-is_keyval_pair(const char *s)
-{
-  /* An argument is a key-value pair if it has an =, and it isn't of the form
-   * $fingeprint=name */
-  return strchr(s, '=') && s[0] != '$';
-}
-
-/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
- * circuit, and report success or failure. */
-static int
-handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  smartlist_t *router_nicknames=NULL, *nodes=NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
-  smartlist_t *args;
-  (void) len;
-
-  router_nicknames = smartlist_new();
-
-  args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1);
-  if (!args)
-    goto done;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,0));
-
-  if (zero_circ) {
-    const char *purp = find_element_starting_with(args, 1, "PURPOSE=");
-
-    if (purp) {
-      intended_purpose = circuit_purpose_from_string(purp);
-      if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-        SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-        smartlist_free(args);
-        goto done;
-      }
-    }
-
-    if ((smartlist_len(args) == 1) ||
-        (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) {
-      // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar"
-      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
-      if (!circ) {
-        connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      } else {
-        connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                  (unsigned long)circ->global_identifier);
-      }
-      SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-      smartlist_free(args);
-      goto done;
-    }
-    // "EXTENDCIRCUIT 0 router1,router2" ||
-    // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo"
-  }
-
-  if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  if (smartlist_len(args) < 2) {
-    connection_printf_to_buf(conn,
-                             "512 syntax error: not enough arguments.\r\n");
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0);
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-
-  nodes = smartlist_new();
-  int first_node = zero_circ;
-  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
-    const node_t *node = node_get_by_nickname(n, 0);
-    if (!node) {
-      connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n);
-      goto done;
-    }
-    if (!node_has_preferred_descriptor(node, first_node)) {
-      connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n);
-      goto done;
-    }
-    smartlist_add(nodes, (void*)node);
-    first_node = 0;
-  } SMARTLIST_FOREACH_END(n);
-  if (!smartlist_len(nodes)) {
-    connection_write_str_to_buf("512 No router names provided\r\n", conn);
-    goto done;
-  }
-
-  if (zero_circ) {
-    /* start a new circuit */
-    circ = origin_circuit_init(intended_purpose, 0);
-  }
-
-  /* now circ refers to something that is ready to be extended */
-  first_node = zero_circ;
-  SMARTLIST_FOREACH(nodes, const node_t *, node,
-  {
-    extend_info_t *info = extend_info_from_node(node, first_node);
-    if (!info) {
-      tor_assert_nonfatal(first_node);
-      log_warn(LD_CONTROL,
-               "controller tried to connect to a node that lacks a suitable "
-               "descriptor, or which doesn't have any "
-               "addresses that are allowed by the firewall configuration; "
-               "circuit marked for closing.");
-      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-    circuit_append_new_exit(circ, info);
-    if (circ->build_state->desired_path_len > 1) {
-      circ->build_state->onehop_tunnel = 0;
-    }
-    extend_info_free(info);
-    first_node = 0;
-  });
-
-  /* now that we've populated the cpath, start extending */
-  if (zero_circ) {
-    int err_reason = 0;
-    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
-      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-  } else {
-    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
-        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
-      int err_reason = 0;
-      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
-      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
-        log_info(LD_CONTROL,
-                 "send_next_onion_skin failed; circuit marked for closing.");
-        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-        connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn);
-        goto done;
-      }
-    }
-  }
-
-  connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                             (unsigned long)circ->global_identifier);
-  if (zero_circ) /* send a 'launched' event, for completeness */
-    control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
- done:
-  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
-  smartlist_free(router_nicknames);
-  smartlist_free(nodes);
-  return 0;
-}
-
-/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
- * circuit and it's a valid purpose, change it. */
-static int
-handle_control_setcircuitpurpose(control_connection_t *conn,
-                                 uint32_t len, const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  uint8_t new_purpose;
-  smartlist_t *args;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1);
-  if (!args)
-    goto done;
-
-  if (!(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    goto done;
-  }
-
-  {
-    const char *purp = find_element_starting_with(args,1,"PURPOSE=");
-    if (!purp) {
-      connection_write_str_to_buf("552 No purpose given\r\n", conn);
-      goto done;
-    }
-    new_purpose = circuit_purpose_from_string(purp);
-    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-      connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-      goto done;
-    }
-  }
-
-  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  if (args) {
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-  }
-  return 0;
-}
-
-/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
- * stream, and report success or failure. */
-static int
-handle_control_attachstream(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  smartlist_t *args;
-  crypt_path_t *cpath=NULL;
-  int hop=0, hop_line_ok=1;
-  (void) len;
-
-  args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,1));
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 1));
-  } else if (circ) {
-    const char *hopstring = find_element_starting_with(args,2,"HOP=");
-    if (hopstring) {
-      hopstring += strlen("HOP=");
-      hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX,
-                                  &hop_line_ok, NULL);
-      if (!hop_line_ok) { /* broken hop line */
-        connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring);
-      }
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok)
-    return 0;
-
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
-    connection_write_str_to_buf(
-                    "555 Connection is not managed by controller.\r\n",
-                    conn);
-    return 0;
-  }
-
-  /* Do we need to detach it first? */
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
-    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
-    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
-    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
-    /* Un-mark it as ending, since we're going to reuse it. */
-    edge_conn->edge_has_sent_end = 0;
-    edge_conn->end_reason = 0;
-    if (tmpcirc)
-      circuit_detach_stream(tmpcirc, edge_conn);
-    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
-    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
-  }
-
-  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
-    connection_write_str_to_buf(
-                    "551 Can't attach stream to non-open origin circuit\r\n",
-                    conn);
-    return 0;
-  }
-  /* Is this a single hop circuit? */
-  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
-    connection_write_str_to_buf(
-               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
-    return 0;
-  }
-
-  if (circ && hop>0) {
-    /* find this hop in the circuit, and set cpath */
-    cpath = circuit_get_cpath_hop(circ, hop);
-    if (!cpath) {
-      connection_printf_to_buf(conn,
-                               "551 Circuit doesn't have %d hops.\r\n", hop);
-      return 0;
-    }
-  }
-  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
-    connection_write_str_to_buf("551 Unable to attach stream\r\n", conn);
-    return 0;
-  }
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
- * descriptor, and report success or failure. */
-static int
-handle_control_postdescriptor(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  char *desc;
-  const char *msg=NULL;
-  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
-  int cache = 0; /* eventually, we may switch this to 1 */
-
-  const char *cp = memchr(body, '\n', len);
-
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  ++cp;
-
-  char *cmdline = tor_memdup_nulterm(body, cp-body);
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, cmdline, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, char *, option) {
-    if (!strcasecmpstart(option, "purpose=")) {
-      option += strlen("purpose=");
-      purpose = router_purpose_from_string(option);
-      if (purpose == ROUTER_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else if (!strcasecmpstart(option, "cache=")) {
-      option += strlen("cache=");
-      if (!strcasecmp(option, "no"))
-        cache = 0;
-      else if (!strcasecmp(option, "yes"))
-        cache = 1;
-      else {
-        connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else { /* unrecognized argument? */
-      connection_printf_to_buf(conn,
-        "512 Unexpected argument \"%s\" to postdescriptor\r\n", option);
-      goto done;
-    }
-  } SMARTLIST_FOREACH_END(option);
-
-  read_escaped_data(cp, len-(cp-body), &desc);
-
-  switch (router_load_single_router(desc, purpose, cache, &msg)) {
-  case -1:
-    if (!msg) msg = "Could not parse descriptor";
-    connection_printf_to_buf(conn, "554 %s\r\n", msg);
-    break;
-  case 0:
-    if (!msg) msg = "Descriptor not added";
-    connection_printf_to_buf(conn, "251 %s\r\n",msg);
-    break;
-  case 1:
-    send_control_done(conn);
-    break;
-  }
-
-  tor_free(desc);
- done:
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  tor_free(cmdline);
-  return 0;
-}
-
-/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
- * address of the named AP stream, and report success or failure. */
-static int
-handle_control_redirectstream(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  char *new_addr = NULL;
-  uint16_t new_port = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
-           || !ap_conn->socks_request) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else {
-    int ok = 1;
-    if (smartlist_len(args) > 2) { /* they included a port too */
-      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
-                                            10, 1, 65535, &ok, NULL);
-    }
-    if (!ok) {
-      connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n",
-                               (char*)smartlist_get(args, 2));
-    } else {
-      new_addr = tor_strdup(smartlist_get(args, 1));
-    }
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!new_addr)
-    return 0;
-
-  strlcpy(ap_conn->socks_request->address, new_addr,
-          sizeof(ap_conn->socks_request->address));
-  if (new_port)
-    ap_conn->socks_request->port = new_port;
-  tor_free(new_addr);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSESTREAM command; try to close the named stream
- * and report success or failure. */
-static int
-handle_control_closestream(control_connection_t *conn, uint32_t len,
-                           const char *body)
-{
-  entry_connection_t *ap_conn=NULL;
-  uint8_t reason=0;
-  smartlist_t *args;
-  int ok;
-  (void) len;
-
-  args = getargs_helper("CLOSESTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  else if (!(ap_conn = get_stream(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
-                                       &ok, NULL);
-    if (!ok) {
-      connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n",
-                               (char*)smartlist_get(args, 1));
-      ap_conn = NULL;
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn)
-    return 0;
-
-  connection_mark_unattached_ap(ap_conn, reason);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
- * and report success or failure. */
-static int
-handle_control_closecircuit(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  int safe = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1);
-  if (!args)
-    return 0;
-
-  if (!(circ=get_circ(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    int i;
-    for (i=1; i < smartlist_len(args); ++i) {
-      if (!strcasecmp(smartlist_get(args, i), "IfUnused"))
-        safe = 1;
-      else
-        log_info(LD_CONTROL, "Skipping unknown option %s",
-                 (char*)smartlist_get(args,i));
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!circ)
-    return 0;
-
-  if (!safe || !circ->p_streams) {
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
-  }
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a RESOLVE command: start trying to resolve
- * the listed addresses. */
-static int
-handle_control_resolve(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *args, *failed;
-  int is_reverse = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-
-  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
-    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
-             "isn't listening for ADDRMAP events.  It probably won't see "
-             "the answer.");
-  }
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  {
-    const char *modearg = find_element_starting_with(args, 0, "mode=");
-    if (modearg && !strcasecmp(modearg, "mode=reverse"))
-      is_reverse = 1;
-  }
-  failed = smartlist_new();
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      if (!is_keyval_pair(arg)) {
-          if (dnsserv_launch_request(arg, is_reverse, conn)<0)
-            smartlist_add(failed, (char*)arg);
-      }
-  });
-
-  send_control_done(conn);
-  SMARTLIST_FOREACH(failed, const char *, arg, {
-      control_event_address_mapped(arg, arg, time(NULL),
-                                   "internal", 0);
-  });
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  smartlist_free(failed);
-  return 0;
-}
-
-/** Called when we get a PROTOCOLINFO command: send back a reply. */
-static int
-handle_control_protocolinfo(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  const char *bad_arg = NULL;
-  smartlist_t *args;
-  (void)len;
-
-  conn->have_sent_protocolinfo = 1;
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      int ok;
-      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
-      if (!ok) {
-        bad_arg = arg;
-        break;
-      }
-    });
-  if (bad_arg) {
-    connection_printf_to_buf(conn, "513 No such version %s\r\n",
-                             escaped(bad_arg));
-    /* Don't tolerate bad arguments when not authenticated. */
-    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
-      connection_mark_for_close(TO_CONN(conn));
-    goto done;
-  } else {
-    const or_options_t *options = get_options();
-    int cookies = options->CookieAuthentication;
-    char *cfile = get_controller_cookie_file_name();
-    char *abs_cfile;
-    char *esc_cfile;
-    char *methods;
-    abs_cfile = make_path_absolute(cfile);
-    esc_cfile = esc_for_log(abs_cfile);
-    {
-      int passwd = (options->HashedControlPassword != NULL ||
-                    options->HashedControlSessionPassword != NULL);
-      smartlist_t *mlist = smartlist_new();
-      if (cookies) {
-        smartlist_add(mlist, (char*)"COOKIE");
-        smartlist_add(mlist, (char*)"SAFECOOKIE");
-      }
-      if (passwd)
-        smartlist_add(mlist, (char*)"HASHEDPASSWORD");
-      if (!cookies && !passwd)
-        smartlist_add(mlist, (char*)"NULL");
-      methods = smartlist_join_strings(mlist, ",", 0, NULL);
-      smartlist_free(mlist);
-    }
-
-    connection_printf_to_buf(conn,
-                             "250-PROTOCOLINFO 1\r\n"
-                             "250-AUTH METHODS=%s%s%s\r\n"
-                             "250-VERSION Tor=%s\r\n"
-                             "250 OK\r\n",
-                             methods,
-                             cookies?" COOKIEFILE=":"",
-                             cookies?esc_cfile:"",
-                             escaped(VERSION));
-    tor_free(methods);
-    tor_free(cfile);
-    tor_free(abs_cfile);
-    tor_free(esc_cfile);
-  }
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Called when we get an AUTHCHALLENGE command. */
-static int
-handle_control_authchallenge(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  const char *cp = body;
-  char *client_nonce;
-  size_t client_nonce_len;
-  char server_hash[DIGEST256_LEN];
-  char server_hash_encoded[HEX_DIGEST256_LEN+1];
-  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
-  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
-
-  cp += strspn(cp, " \t\n\r");
-  if (!strcasecmpstart(cp, "SAFECOOKIE")) {
-    cp += strlen("SAFECOOKIE");
-  } else {
-    connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE "
-                                "authentication\r\n", conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  if (!authentication_cookie_is_set) {
-    connection_write_str_to_buf("515 Cookie authentication is disabled\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp == '"') {
-    const char *newcp =
-      decode_escaped_string(cp, len - (cp - body),
-                            &client_nonce, &client_nonce_len);
-    if (newcp == NULL) {
-      connection_write_str_to_buf("513 Invalid quoted client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return -1;
-    }
-    cp = newcp;
-  } else {
-    size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef");
-
-    client_nonce_len = client_nonce_encoded_len / 2;
-    client_nonce = tor_malloc_zero(client_nonce_len);
-
-    if (base16_decode(client_nonce, client_nonce_len,
-                      cp, client_nonce_encoded_len)
-                      != (int) client_nonce_len) {
-      connection_write_str_to_buf("513 Invalid base16 client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(client_nonce);
-      return -1;
-    }
-
-    cp += client_nonce_encoded_len;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp != '\0' ||
-      cp != body + len) {
-    connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    tor_free(client_nonce);
-    return -1;
-  }
-  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-  /* Now compute and send the server-to-controller response, and the
-   * server's nonce. */
-  tor_assert(authentication_cookie != NULL);
-
-  {
-    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
-                      client_nonce_len +
-                      SAFECOOKIE_SERVER_NONCE_LEN);
-    char *tmp = tor_malloc_zero(tmp_len);
-    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
-    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
-           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-    crypto_hmac_sha256(server_hash,
-                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
-                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    crypto_hmac_sha256(client_hash,
-                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
-                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    conn->safecookie_client_hash = client_hash;
-
-    tor_free(tmp);
-  }
-
-  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
-                server_hash, sizeof(server_hash));
-  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
-                server_nonce, sizeof(server_nonce));
-
-  connection_printf_to_buf(conn,
-                           "250 AUTHCHALLENGE SERVERHASH=%s "
-                           "SERVERNONCE=%s\r\n",
-                           server_hash_encoded,
-                           server_nonce_encoded);
-
-  tor_free(client_nonce);
-  return 0;
-}
-
-/** Called when we get a USEFEATURE command: parse the feature list, and
- * set up the control_connection's options properly. */
-static int
-handle_control_usefeature(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  int bad = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmp(arg, "VERBOSE_NAMES"))
-        ;
-      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
-        ;
-      else {
-        connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n",
-                                 arg);
-        bad = 1;
-        break;
-      }
-  } SMARTLIST_FOREACH_END(arg);
-
-  if (!bad) {
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the DROPGUARDS command. */
-static int
-handle_control_dropguards(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-
-  static int have_warned = 0;
-  if (! have_warned) {
-    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
-             "the risks before using it. It may be removed in a future "
-             "version of Tor.");
-    have_warned = 1;
-  }
-
-  if (smartlist_len(args)) {
-    connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
-  } else {
-    remove_all_entry_guards();
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the HSFETCH command. */
-static int
-handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  int i;
-  char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL;
-  smartlist_t *args = NULL, *hsdirs = NULL;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  static const char *hsfetch_command = "HSFETCH";
-  static const char *v2_str = "v2-";
-  const size_t v2_str_len = strlen(v2_str);
-  rend_data_t *rend_query = NULL;
-
-  /* Make sure we have at least one argument, the HSAddress. */
-  args = getargs_helper(hsfetch_command, conn, body, 1, -1);
-  if (!args) {
-    goto exit;
-  }
-
-  /* Extract the first argument (either HSAddress or DescID). */
-  arg1 = smartlist_get(args, 0);
-  /* Test if it's an HS address without the .onion part. */
-  if (rend_valid_v2_service_id(arg1)) {
-    hsaddress = arg1;
-  } else if (strcmpstart(arg1, v2_str) == 0 &&
-             rend_valid_descriptor_id(arg1 + v2_str_len) &&
-             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
-                           REND_DESC_ID_V2_LEN_BASE32) == 0) {
-    /* We have a well formed version 2 descriptor ID. Keep the decoded value
-     * of the id. */
-    desc_id = digest;
-  } else {
-    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
-                             arg1);
-    goto done;
-  }
-
-  static const char *opt_server = "SERVER=";
-
-  /* Skip first argument because it's the HSAddress or DescID. */
-  for (i = 1; i < smartlist_len(args); ++i) {
-    const char *arg = smartlist_get(args, i);
-    const node_t *node;
-
-    if (!strcasecmpstart(arg, opt_server)) {
-      const char *server;
-
-      server = arg + strlen(opt_server);
-      node = node_get_by_hex_id(server, 0);
-      if (!node) {
-        connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                 server);
-        goto done;
-      }
-      if (!hsdirs) {
-        /* Stores routerstatus_t object for each specified server. */
-        hsdirs = smartlist_new();
-      }
-      /* Valid server, add it to our local list. */
-      smartlist_add(hsdirs, node->rs);
-    } else {
-      connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n",
-                               arg);
-      goto done;
-    }
-  }
-
-  rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
-                                       REND_NO_AUTH);
-  if (rend_query == NULL) {
-    connection_printf_to_buf(conn, "551 Error creating the HS query\r\n");
-    goto done;
-  }
-
-  /* Using a descriptor ID, we force the user to provide at least one
-   * hsdir server using the SERVER= option. */
-  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
-      connection_printf_to_buf(conn, "512 %s option is required\r\n",
-                               opt_server);
-      goto done;
-  }
-
-  /* We are about to trigger HSDir fetch so send the OK now because after
-   * that 650 event(s) are possible so better to have the 250 OK before them
-   * to avoid out of order replies. */
-  send_control_done(conn);
-
-  /* Trigger the fetch using the built rend query and possibly a list of HS
-   * directory to use. This function ignores the client cache thus this will
-   * always send a fetch command. */
-  rend_client_fetch_v2_desc(rend_query, hsdirs);
-
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  /* Contains data pointer that we don't own thus no cleanup. */
-  smartlist_free(hsdirs);
-  rend_data_free(rend_query);
- exit:
-  return 0;
-}
-
-/** Implementation for the HSPOST command. */
-static int
-handle_control_hspost(control_connection_t *conn,
-                      uint32_t len,
-                      const char *body)
-{
-  static const char *opt_server = "SERVER=";
-  static const char *opt_hsaddress = "HSADDRESS=";
-  smartlist_t *hs_dirs = NULL;
-  const char *encoded_desc = body;
-  size_t encoded_desc_len = len;
-  const char *onion_address = NULL;
-
-  char *cp = memchr(body, '\n', len);
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  char *argline = tor_strndup(body, cp-body);
-
-  smartlist_t *args = smartlist_new();
-
-  /* If any SERVER= or HSADDRESS= options were specified, try to parse
-   * the options line. */
-  if (!strcasecmpstart(argline, opt_server) ||
-      !strcasecmpstart(argline, opt_hsaddress)) {
-    /* encoded_desc begins after a newline character */
-    cp = cp + 1;
-    encoded_desc = cp;
-    encoded_desc_len = len-(cp-body);
-
-    smartlist_split_string(args, argline, " ",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmpstart(arg, opt_server)) {
-        const char *server = arg + strlen(opt_server);
-        const node_t *node = node_get_by_hex_id(server, 0);
-
-        if (!node || !node->rs) {
-          connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                   server);
-          goto done;
-        }
-        /* Valid server, add it to our local list. */
-        if (!hs_dirs)
-          hs_dirs = smartlist_new();
-        smartlist_add(hs_dirs, node->rs);
-      } else if (!strcasecmpstart(arg, opt_hsaddress)) {
-        const char *address = arg + strlen(opt_hsaddress);
-        if (!hs_address_is_valid(address)) {
-          connection_printf_to_buf(conn, "512 Malformed onion address\r\n");
-          goto done;
-        }
-        onion_address = address;
-      } else {
-        connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n",
-                                 arg);
-        goto done;
-      }
-    } SMARTLIST_FOREACH_END(arg);
-  }
-
-  /* Handle the v3 case. */
-  if (onion_address) {
-    char *desc_str = NULL;
-    read_escaped_data(encoded_desc, encoded_desc_len, &desc_str);
-    if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) {
-      connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-    } else {
-      send_control_done(conn);
-    }
-    tor_free(desc_str);
-    goto done;
-  }
-
-  /* From this point on, it is only v2. */
-
-  /* Read the dot encoded descriptor, and parse it. */
-  rend_encoded_v2_service_descriptor_t *desc =
-      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
-  read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str);
-
-  rend_service_descriptor_t *parsed = NULL;
-  char *intro_content = NULL;
-  size_t intro_size;
-  size_t encoded_size;
-  const char *next_desc;
-  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
-                                        &intro_size, &encoded_size,
-                                        &next_desc, desc->desc_str, 1)) {
-    /* Post the descriptor. */
-    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
-    if (!rend_get_service_id(parsed->pk, serviceid)) {
-      smartlist_t *descs = smartlist_new();
-      smartlist_add(descs, desc);
-
-      /* We are about to trigger HS descriptor upload so send the OK now
-       * because after that 650 event(s) are possible so better to have the
-       * 250 OK before them to avoid out of order replies. */
-      send_control_done(conn);
-
-      /* Trigger the descriptor upload */
-      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
-      smartlist_free(descs);
-    }
-
-    rend_service_descriptor_free(parsed);
-  } else {
-    connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-  }
-
-  tor_free(intro_content);
-  rend_encoded_v2_service_descriptor_free(desc);
- done:
-  tor_free(argline);
-  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  return 0;
-}
-
-/* Helper function for ADD_ONION that adds an ephemeral service depending on
- * the given hs_version.
- *
- * The secret key in pk depends on the hs_version. The ownership of the key
- * used in pk is given to the HS subsystem so the caller must stop accessing
- * it after.
- *
- * The port_cfgs is a list of service port. Ownership transferred to service.
- * The max_streams refers to the MaxStreams= key.
- * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
- * The auth_type is the authentication type of the clients in auth_clients.
- * The ownership of that list is transferred to the service.
- *
- * On success (RSAE_OKAY), the address_out points to a newly allocated string
- * containing the onion address without the .onion part. On error, address_out
- * is untouched. */
-static hs_service_add_ephemeral_status_t
-add_onion_helper_add_service(int hs_version,
-                             add_onion_secret_key_t *pk,
-                             smartlist_t *port_cfgs, int max_streams,
-                             int max_streams_close_circuit, int auth_type,
-                             smartlist_t *auth_clients, char **address_out)
-{
-  hs_service_add_ephemeral_status_t ret;
-
-  tor_assert(pk);
-  tor_assert(port_cfgs);
-  tor_assert(address_out);
-
-  switch (hs_version) {
-  case HS_VERSION_TWO:
-    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
-                                     max_streams_close_circuit, auth_type,
-                                     auth_clients, address_out);
-    break;
-  case HS_VERSION_THREE:
-    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
-                                   max_streams_close_circuit, address_out);
-    break;
-  default:
-    tor_assert_unreached();
-  }
-
-  return ret;
-}
-
-/** Called when we get a ADD_ONION command; parse the body, and set up
- * the new ephemeral Onion Service. */
-static int
-handle_control_add_onion(control_connection_t *conn,
-                         uint32_t len,
-                         const char *body)
-{
-  smartlist_t *args;
-  int arg_len;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("ADD_ONION", conn, body, 2, -1);
-  if (!args)
-    return 0;
-  arg_len = smartlist_len(args);
-
-  /* Parse all of the arguments that do not involve handling cryptographic
-   * material first, since there's no reason to touch that at all if any of
-   * the other arguments are malformed.
-   */
-  smartlist_t *port_cfgs = smartlist_new();
-  smartlist_t *auth_clients = NULL;
-  smartlist_t *auth_created_clients = NULL;
-  int discard_pk = 0;
-  int detach = 0;
-  int max_streams = 0;
-  int max_streams_close_circuit = 0;
-  rend_auth_type_t auth_type = REND_NO_AUTH;
-  /* Default to adding an anonymous hidden service if no flag is given */
-  int non_anonymous = 0;
-  for (int i = 1; i < arg_len; i++) {
-    static const char *port_prefix = "Port=";
-    static const char *flags_prefix = "Flags=";
-    static const char *max_s_prefix = "MaxStreams=";
-    static const char *auth_prefix = "ClientAuth=";
-
-    const char *arg = smartlist_get(args, (int)i);
-    if (!strcasecmpstart(arg, port_prefix)) {
-      /* "Port=VIRTPORT[,TARGET]". */
-      const char *port_str = arg + strlen(port_prefix);
-
-      rend_service_port_config_t *cfg =
-          rend_service_parse_port_config(port_str, ",", NULL);
-      if (!cfg) {
-        connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-        goto out;
-      }
-      smartlist_add(port_cfgs, cfg);
-    } else if (!strcasecmpstart(arg, max_s_prefix)) {
-      /* "MaxStreams=[0..65535]". */
-      const char *max_s_str = arg + strlen(max_s_prefix);
-      int ok = 0;
-      max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL);
-      if (!ok) {
-        connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n");
-        goto out;
-      }
-    } else if (!strcasecmpstart(arg, flags_prefix)) {
-      /* "Flags=Flag[,Flag]", where Flag can be:
-       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
-       *                   the response.
-       *   * 'Detach' - Do not tie this onion service to any particular control
-       *                connection.
-       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
-       *                                exceeded.
-       *   * 'BasicAuth' - Client authorization using the 'basic' method.
-       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
-       *                      flag is present, tor must be in non-anonymous
-       *                      hidden service mode. If this flag is absent,
-       *                      tor must be in anonymous hidden service mode.
-       */
-      static const char *discard_flag = "DiscardPK";
-      static const char *detach_flag = "Detach";
-      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
-      static const char *basicauth_flag = "BasicAuth";
-      static const char *non_anonymous_flag = "NonAnonymous";
-
-      smartlist_t *flags = smartlist_new();
-      int bad = 0;
-
-      smartlist_split_string(flags, arg + strlen(flags_prefix), ",",
-                             SPLIT_IGNORE_BLANK, 0);
-      if (smartlist_len(flags) < 1) {
-        connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n");
-        bad = 1;
-      }
-      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
-      {
-        if (!strcasecmp(flag, discard_flag)) {
-          discard_pk = 1;
-        } else if (!strcasecmp(flag, detach_flag)) {
-          detach = 1;
-        } else if (!strcasecmp(flag, max_s_close_flag)) {
-          max_streams_close_circuit = 1;
-        } else if (!strcasecmp(flag, basicauth_flag)) {
-          auth_type = REND_BASIC_AUTH;
-        } else if (!strcasecmp(flag, non_anonymous_flag)) {
-          non_anonymous = 1;
-        } else {
-          connection_printf_to_buf(conn,
-                                   "512 Invalid 'Flags' argument: %s\r\n",
-                                   escaped(flag));
-          bad = 1;
-          break;
-        }
-      } SMARTLIST_FOREACH_END(flag);
-      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
-      smartlist_free(flags);
-      if (bad)
-        goto out;
-    } else if (!strcasecmpstart(arg, auth_prefix)) {
-      char *err_msg = NULL;
-      int created = 0;
-      rend_authorized_client_t *client =
-        add_onion_helper_clientauth(arg + strlen(auth_prefix),
-                                    &created, &err_msg);
-      if (!client) {
-        if (err_msg) {
-          connection_write_str_to_buf(err_msg, conn);
-          tor_free(err_msg);
-        }
-        goto out;
-      }
-
-      if (auth_clients != NULL) {
-        int bad = 0;
-        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
-          if (strcmp(ac->client_name, client->client_name) == 0) {
-            bad = 1;
-            break;
-          }
-        } SMARTLIST_FOREACH_END(ac);
-        if (bad) {
-          connection_printf_to_buf(conn,
-                                   "512 Duplicate name in ClientAuth\r\n");
-          rend_authorized_client_free(client);
-          goto out;
-        }
-      } else {
-        auth_clients = smartlist_new();
-        auth_created_clients = smartlist_new();
-      }
-      smartlist_add(auth_clients, client);
-      if (created) {
-        smartlist_add(auth_created_clients, client);
-      }
-    } else {
-      connection_printf_to_buf(conn, "513 Invalid argument\r\n");
-      goto out;
-    }
-  }
-  if (smartlist_len(port_cfgs) == 0) {
-    connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n");
-    goto out;
-  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
-    connection_printf_to_buf(conn, "512 No auth type specified\r\n");
-    goto out;
-  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
-    connection_printf_to_buf(conn, "512 No auth clients specified\r\n");
-    goto out;
-  } else if ((auth_type == REND_BASIC_AUTH &&
-              smartlist_len(auth_clients) > 512) ||
-             (auth_type == REND_STEALTH_AUTH &&
-              smartlist_len(auth_clients) > 16)) {
-    connection_printf_to_buf(conn, "512 Too many auth clients\r\n");
-    goto out;
-  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
-                                                              get_options())) {
-    /* If we failed, and the non-anonymous flag is set, Tor must be in
-     * anonymous hidden service mode.
-     * The error message changes based on the current Tor config:
-     * 512 Tor is in anonymous hidden service mode
-     * 512 Tor is in non-anonymous hidden service mode
-     * (I've deliberately written them out in full here to aid searchability.)
-     */
-    connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service "
-                             "mode\r\n",
-                             non_anonymous ? "" : "non-");
-    goto out;
-  }
-
-  /* Parse the "keytype:keyblob" argument. */
-  int hs_version = 0;
-  add_onion_secret_key_t pk = { NULL };
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-
-  if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
-                              &key_new_alg, &key_new_blob, &pk, &hs_version,
-                              &err_msg) < 0) {
-    if (err_msg) {
-      connection_write_str_to_buf(err_msg, conn);
-      tor_free(err_msg);
-    }
-    goto out;
-  }
-  tor_assert(!err_msg);
-
-  /* Hidden service version 3 don't have client authentication support so if
-   * ClientAuth was given, send back an error. */
-  if (hs_version == HS_VERSION_THREE && auth_clients) {
-    connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n");
-    goto out;
-  }
-
-  /* Create the HS, using private key pk, client authentication auth_type,
-   * the list of auth_clients, and port config port_cfg.
-   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
-   * regardless of success/failure.
-   */
-  char *service_id = NULL;
-  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
-                                         max_streams,
-                                         max_streams_close_circuit, auth_type,
-                                         auth_clients, &service_id);
-  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
-  auth_clients = NULL; /* so is auth_clients */
-  switch (ret) {
-  case RSAE_OKAY:
-  {
-    if (detach) {
-      if (!detached_onion_services)
-        detached_onion_services = smartlist_new();
-      smartlist_add(detached_onion_services, service_id);
-    } else {
-      if (!conn->ephemeral_onion_services)
-        conn->ephemeral_onion_services = smartlist_new();
-      smartlist_add(conn->ephemeral_onion_services, service_id);
-    }
-
-    tor_assert(service_id);
-    connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id);
-    if (key_new_alg) {
-      tor_assert(key_new_blob);
-      connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n",
-                               key_new_alg, key_new_blob);
-    }
-    if (auth_created_clients) {
-      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
-        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
-                                                auth_type);
-        tor_assert(encoded);
-        connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n",
-                                 ac->client_name, encoded);
-        memwipe(encoded, 0, strlen(encoded));
-        tor_free(encoded);
-      });
-    }
-
-    connection_printf_to_buf(conn, "250 OK\r\n");
-    break;
-  }
-  case RSAE_BADPRIVKEY:
-    connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n");
-    break;
-  case RSAE_ADDREXISTS:
-    connection_printf_to_buf(conn, "550 Onion address collision\r\n");
-    break;
-  case RSAE_BADVIRTPORT:
-    connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-    break;
-  case RSAE_BADAUTH:
-    connection_printf_to_buf(conn, "512 Invalid client authorization\r\n");
-    break;
-  case RSAE_INTERNAL: FALLTHROUGH;
-  default:
-    connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n");
-  }
-  if (key_new_blob) {
-    memwipe(key_new_blob, 0, strlen(key_new_blob));
-    tor_free(key_new_blob);
-  }
-
- out:
-  if (port_cfgs) {
-    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
-                      rend_service_port_config_free(p));
-    smartlist_free(port_cfgs);
-  }
-
-  if (auth_clients) {
-    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
-                      rend_authorized_client_free(ac));
-    smartlist_free(auth_clients);
-  }
-  if (auth_created_clients) {
-    // Do not free entries; they are the same as auth_clients
-    smartlist_free(auth_created_clients);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
-/** Helper function to handle parsing the KeyType:KeyBlob argument to the
- * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
- * and the private key not discarded, the algorithm and serialized private key,
- * or NULL and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned key_new_blob and err_msg.
- *
- * Note: The error messages returned are deliberately vague to avoid echoing
- * key material.
- */
-STATIC int
-add_onion_helper_keyarg(const char *arg, int discard_pk,
-                        const char **key_new_alg_out, char **key_new_blob_out,
-                        add_onion_secret_key_t *decoded_key, int *hs_version,
-                        char **err_msg_out)
-{
-  smartlist_t *key_args = smartlist_new();
-  crypto_pk_t *pk = NULL;
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-  int ret = -1;
-
-  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(key_args) != 2) {
-    err_msg = tor_strdup("512 Invalid key type/blob\r\n");
-    goto err;
-  }
-
-  /* The format is "KeyType:KeyBlob". */
-  static const char *key_type_new = "NEW";
-  static const char *key_type_best = "BEST";
-  static const char *key_type_rsa1024 = "RSA1024";
-  static const char *key_type_ed25519_v3 = "ED25519-V3";
-
-  const char *key_type = smartlist_get(key_args, 0);
-  const char *key_blob = smartlist_get(key_args, 1);
-
-  if (!strcasecmp(key_type_rsa1024, key_type)) {
-    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
-    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
-    if (!pk) {
-      err_msg = tor_strdup("512 Failed to decode RSA key\r\n");
-      goto err;
-    }
-    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
-      crypto_pk_free(pk);
-      err_msg = tor_strdup("512 Invalid RSA key size\r\n");
-      goto err;
-    }
-    decoded_key->v2 = pk;
-    *hs_version = HS_VERSION_TWO;
-  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
-    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
-    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
-                      strlen(key_blob)) != sizeof(sk->seckey)) {
-      tor_free(sk);
-      err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n");
-      goto err;
-    }
-    decoded_key->v3 = sk;
-    *hs_version = HS_VERSION_THREE;
-  } else if (!strcasecmp(key_type_new, key_type)) {
-    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
-    if (!strcasecmp(key_type_rsa1024, key_blob) ||
-        !strcasecmp(key_type_best, key_blob)) {
-      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
-      pk = crypto_pk_new();
-      if (crypto_pk_generate_key(pk)) {
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_rsa1024);
-        goto err;
-      }
-      if (!discard_pk) {
-        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
-          crypto_pk_free(pk);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_rsa1024);
-          goto err;
-        }
-        key_new_alg = key_type_rsa1024;
-      }
-      decoded_key->v2 = pk;
-      *hs_version = HS_VERSION_TWO;
-    } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) {
-      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-      if (ed25519_secret_key_generate(sk, 1) < 0) {
-        tor_free(sk);
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_ed25519_v3);
-        goto err;
-      }
-      if (!discard_pk) {
-        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
-        key_new_blob = tor_malloc_zero(len);
-        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
-                          sizeof(sk->seckey), 0) != (len - 1)) {
-          tor_free(sk);
-          tor_free(key_new_blob);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_ed25519_v3);
-          goto err;
-        }
-        key_new_alg = key_type_ed25519_v3;
-      }
-      decoded_key->v3 = sk;
-      *hs_version = HS_VERSION_THREE;
-    } else {
-      err_msg = tor_strdup("513 Invalid key type\r\n");
-      goto err;
-    }
-  } else {
-    err_msg = tor_strdup("513 Invalid key type\r\n");
-    goto err;
-  }
-
-  /* Succeeded in loading or generating a private key. */
-  ret = 0;
-
- err:
-  SMARTLIST_FOREACH(key_args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(key_args);
-
-  if (err_msg_out) {
-    *err_msg_out = err_msg;
-  } else {
-    tor_free(err_msg);
-  }
-  *key_new_alg_out = key_new_alg;
-  *key_new_blob_out = key_new_blob;
-
-  return ret;
-}
-
-/** Helper function to handle parsing a ClientAuth argument to the
- * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
- * and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned auth_client and err_msg.
- *
- * If 'created' is specified, it will be set to 1 when a new cookie has
- * been generated.
- */
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg)
-{
-  int ok = 0;
-
-  tor_assert(arg);
-  tor_assert(created);
-  tor_assert(err_msg);
-  *err_msg = NULL;
-
-  smartlist_t *auth_args = smartlist_new();
-  rend_authorized_client_t *client =
-    tor_malloc_zero(sizeof(rend_authorized_client_t));
-  smartlist_split_string(auth_args, arg, ":", 0, 0);
-  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
-    *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n");
-    goto err;
-  }
-  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
-  if (smartlist_len(auth_args) == 2) {
-    char *decode_err_msg = NULL;
-    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
-                                client->descriptor_cookie,
-                                NULL, &decode_err_msg) < 0) {
-      tor_assert(decode_err_msg);
-      tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg);
-      tor_free(decode_err_msg);
-      goto err;
-    }
-    *created = 0;
-  } else {
-    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
-    *created = 1;
-  }
-
-  if (!rend_valid_client_name(client->client_name)) {
-    *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n");
-    goto err;
-  }
-
-  ok = 1;
- err:
-  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
-  smartlist_free(auth_args);
-  if (!ok) {
-    rend_authorized_client_free(client);
-    client = NULL;
-  }
-  return client;
-}
-
-/** Called when we get a DEL_ONION command; parse the body, and remove
- * the existing ephemeral Onion Service. */
-static int
-handle_control_del_onion(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  int hs_version = 0;
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("DEL_ONION", conn, body, 1, 1);
-  if (!args)
-    return 0;
-
-  const char *service_id = smartlist_get(args, 0);
-  if (rend_valid_v2_service_id(service_id)) {
-    hs_version = HS_VERSION_TWO;
-  } else if (hs_address_is_valid(service_id)) {
-    hs_version = HS_VERSION_THREE;
-  } else {
-    connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n");
-    goto out;
-  }
-
-  /* Determine if the onion service belongs to this particular control
-   * connection, or if it is in the global list of detached services.  If it
-   * is in neither, either the service ID is invalid in some way, or it
-   * explicitly belongs to a different control connection, and an error
-   * should be returned.
-   */
-  smartlist_t *services[2] = {
-    conn->ephemeral_onion_services,
-    detached_onion_services
-  };
-  smartlist_t *onion_services = NULL;
-  int idx = -1;
-  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
-    idx = smartlist_string_pos(services[i], service_id);
-    if (idx != -1) {
-      onion_services = services[i];
-      break;
-    }
-  }
-  if (onion_services == NULL) {
-    connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n");
-  } else {
-    int ret = -1;
-    switch (hs_version) {
-    case HS_VERSION_TWO:
-      ret = rend_service_del_ephemeral(service_id);
-      break;
-    case HS_VERSION_THREE:
-      ret = hs_service_del_ephemeral(service_id);
-      break;
-    default:
-      /* The ret value will be -1 thus hitting the warning below. This should
-       * never happen because of the check at the start of the function. */
-      break;
-    }
-    if (ret < 0) {
-      /* This should *NEVER* fail, since the service is on either the
-       * per-control connection list, or the global one.
-       */
-      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
-               escaped(service_id));
-      tor_fragile_assert();
-    }
-
-    /* Remove/scrub the service_id from the appropriate list. */
-    char *cp = smartlist_get(onion_services, idx);
-    smartlist_del(onion_services, idx);
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-
-    send_control_done(conn);
-  }
-
- out:
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
 /** Called when <b>conn</b> has no more bytes left on its outbuf. */
 int
 connection_control_finished_flushing(control_connection_t *conn)
@@ -5349,6 +279,44 @@ peek_connection_has_http_command(connection_t *conn)
   return peek_buf_has_http_command(conn->inbuf);
 }
 
+/**
+ * Helper: take a nul-terminated command of given length, and find where the
+ * command starts and the arguments begin.  Separate them, allocate a new
+ * string in <b>current_cmd_out</b> for the command, and return a pointer
+ * to the arguments.
+ **/
+STATIC char *
+control_split_incoming_command(char *incoming_cmd,
+                               size_t *data_len,
+                               char **current_cmd_out)
+{
+  const bool is_multiline = *data_len && incoming_cmd[0] == '+';
+  size_t cmd_len = 0;
+  while (cmd_len < *data_len
+         && !TOR_ISSPACE(incoming_cmd[cmd_len]))
+    ++cmd_len;
+
+  *current_cmd_out = tor_memdup_nulterm(incoming_cmd, cmd_len);
+  char *args = incoming_cmd+cmd_len;
+  tor_assert(*data_len>=cmd_len);
+  *data_len -= cmd_len;
+  if (is_multiline) {
+    // Only match horizontal space: any line after the first is data,
+    // not arguments.
+    while ((*args == '\t' || *args == ' ') && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  } else {
+    while (TOR_ISSPACE(*args) && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  }
+
+  return args;
+}
+
 static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy"
   "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n"
@@ -5375,6 +343,60 @@ static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "</body>\n"
   "</html>\n";
 
+/** Return an error on a control connection that tried to use the v0 protocol.
+ */
+static void
+control_send_v0_reject(control_connection_t *conn)
+{
+  size_t body_len;
+  char buf[128];
+  set_uint16(buf+2, htons(0x0000)); /* type == error */
+  set_uint16(buf+4, htons(0x0001)); /* code == internal error */
+  strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
+          "and later; upgrade your controller.",
+          sizeof(buf)-6);
+  body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
+  set_uint16(buf+0, htons(body_len));
+  connection_buf_add(buf, 4+body_len, TO_CONN(conn));
+
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Return an error on a control connection that tried to use HTTP.
+ */
+static void
+control_send_http_reject(control_connection_t *conn)
+{
+  connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
+  log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Check if a control connection has tried to use a known invalid protocol.
+ * If it has, then:
+ *  - send a reject response,
+ *  - log a notice-level message, and
+ *  - return false. */
+static bool
+control_protocol_is_valid(control_connection_t *conn)
+{
+  /* Detect v0 commands and send a "no more v0" message. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_control0_command(TO_CONN(conn))) {
+    control_send_v0_reject(conn);
+    return 0;
+  }
+
+  /* If the user has the HTTP proxy port and the control port confused. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_http_command(TO_CONN(conn))) {
+    control_send_http_reject(conn);
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Called when data has arrived on a v1 control connection: Try to fetch
  * commands from conn->inbuf, and execute them.
  */
@@ -5383,7 +405,6 @@ connection_control_process_inbuf(control_connection_t *conn)
 {
   size_t data_len;
   uint32_t cmd_data_len;
-  int cmd_len;
   char *args;
 
   tor_assert(conn);
@@ -5396,30 +417,7 @@ connection_control_process_inbuf(control_connection_t *conn)
     conn->incoming_cmd_cur_len = 0;
   }
 
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_control0_command(TO_CONN(conn))) {
-    /* Detect v0 commands and send a "no more v0" message. */
-    size_t body_len;
-    char buf[128];
-    set_uint16(buf+2, htons(0x0000)); /* type == error */
-    set_uint16(buf+4, htons(0x0001)); /* code == internal error */
-    strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
-            "and later; upgrade your controller.",
-            sizeof(buf)-6);
-    body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
-    set_uint16(buf+0, htons(body_len));
-    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
-
-    connection_mark_and_flush(TO_CONN(conn));
-    return 0;
-  }
-
-  /* If the user has the HTTP proxy port and the control port confused. */
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_http_command(TO_CONN(conn))) {
-    connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
-    log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
-    connection_mark_and_flush(TO_CONN(conn));
+  if (!control_protocol_is_valid(conn)) {
     return 0;
   }
 
@@ -5438,7 +436,7 @@ connection_control_process_inbuf(control_connection_t *conn)
         return 0;
       else if (r == -1) {
         if (data_len + conn->incoming_cmd_cur_len > MAX_COMMAND_LINE_LENGTH) {
-          connection_write_str_to_buf("500 Line too long.\r\n", conn);
+          control_write_endreply(conn, 500, "Line too long.");
           connection_stop_reading(TO_CONN(conn));
           connection_mark_and_flush(TO_CONN(conn));
         }
@@ -5475,22 +473,15 @@ connection_control_process_inbuf(control_connection_t *conn)
     /* Otherwise, read another line. */
   }
   data_len = conn->incoming_cmd_cur_len;
+
   /* Okay, we now have a command sitting on conn->incoming_cmd. See if we
    * recognize it.
    */
-  cmd_len = 0;
-  while ((size_t)cmd_len < data_len
-         && !TOR_ISSPACE(conn->incoming_cmd[cmd_len]))
-    ++cmd_len;
-
-  conn->incoming_cmd[cmd_len]='\0';
-  args = conn->incoming_cmd+cmd_len+1;
-  tor_assert(data_len>(size_t)cmd_len);
-  data_len -= (cmd_len+1); /* skip the command and NUL we added after it */
-  while (TOR_ISSPACE(*args)) {
-    ++args;
-    --data_len;
-  }
+  tor_free(conn->current_cmd);
+  args = control_split_incoming_command(conn->incoming_cmd, &data_len,
+                                        &conn->current_cmd);
+  if (BUG(!conn->current_cmd))
+    return -1;
 
   /* If the connection is already closing, ignore further commands */
   if (TO_CONN(conn)->marked_for_close) {
@@ -5498,1448 +489,50 @@ connection_control_process_inbuf(control_connection_t *conn)
   }
 
   /* Otherwise, Quit is always valid. */
-  if (!strcasecmp(conn->incoming_cmd, "QUIT")) {
-    connection_write_str_to_buf("250 closing connection\r\n", conn);
+  if (!strcasecmp(conn->current_cmd, "QUIT")) {
+    control_write_endreply(conn, 250, "closing connection");
     connection_mark_and_flush(TO_CONN(conn));
     return 0;
   }
 
   if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      !is_valid_initial_command(conn, conn->incoming_cmd)) {
-    connection_write_str_to_buf("514 Authentication required.\r\n", conn);
+      !is_valid_initial_command(conn, conn->current_cmd)) {
+    control_write_endreply(conn, 514, "Authentication required.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
   if (data_len >= UINT32_MAX) {
-    connection_write_str_to_buf("500 A 4GB command? Nice try.\r\n", conn);
+    control_write_endreply(conn, 500, "A 4GB command? Nice try.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
-  /* XXXX Why is this not implemented as a table like the GETINFO
-   * items are?  Even handling the plus signs at the beginnings of
-   * commands wouldn't be very hard with proper macros. */
   cmd_data_len = (uint32_t)data_len;
-  if (!strcasecmp(conn->incoming_cmd, "SETCONF")) {
-    if (handle_control_setconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) {
-    if (handle_control_resetconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) {
-    if (handle_control_getconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) {
-    if (handle_control_loadconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) {
-    if (handle_control_setevents(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) {
-    if (handle_control_authenticate(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) {
-    if (handle_control_saveconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) {
-    if (handle_control_signal(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) {
-    if (handle_control_takeownership(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) {
-    if (handle_control_mapaddress(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) {
-    if (handle_control_getinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) {
-    if (handle_control_extendcircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) {
-    if (handle_control_setcircuitpurpose(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) {
-    connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn);
-  } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) {
-    if (handle_control_attachstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) {
-    if (handle_control_postdescriptor(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) {
-    if (handle_control_redirectstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) {
-    if (handle_control_closestream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) {
-    if (handle_control_closecircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) {
-    if (handle_control_usefeature(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) {
-    if (handle_control_resolve(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) {
-    if (handle_control_protocolinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) {
-    if (handle_control_authchallenge(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) {
-    if (handle_control_dropguards(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) {
-    if (handle_control_hsfetch(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) {
-    if (handle_control_hspost(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) {
-    int ret = handle_control_add_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the private key. */
-    if (ret)
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) {
-    int ret = handle_control_del_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */
-    if (ret)
-      return -1;
-  } else {
-    connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n",
-                             conn->incoming_cmd);
-  }
+  if (handle_control_command(conn, cmd_data_len, args) < 0)
+    return -1;
 
   conn->incoming_cmd_cur_len = 0;
   goto again;
 }
 
-/** Something major has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-int
-control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
-                             int reason_code)
-{
-  const char *status;
-  char reasons[64] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
-    return 0;
-  tor_assert(circ);
-
-  switch (tp)
-    {
-    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case CIRC_EVENT_BUILT: status = "BUILT"; break;
-    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
-    case CIRC_EVENT_FAILED: status = "FAILED"; break;
-    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
-    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
-    char unk_reason_buf[16];
-    if (!reason_str) {
-      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
-      reason_str = unk_reason_buf;
-    }
-    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
-    } else {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=%s", reason_str);
-    }
-  }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS,
-                                "650 CIRC %lu %s%s%s%s\r\n",
-                                (unsigned long)circ->global_identifier,
-                                status, sp,
-                                circdesc,
-                                reasons);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/** Something minor has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-static int
-control_event_circuit_status_minor(origin_circuit_t *circ,
-                                   circuit_status_minor_event_t e,
-                                   int purpose, const struct timeval *tv)
-{
-  const char *event_desc;
-  char event_tail[160] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
-    return 0;
-  tor_assert(circ);
-
-  switch (e)
-    {
-    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
-      event_desc = "PURPOSE_CHANGED";
-
-      {
-        /* event_tail can currently be up to 68 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "");
-      }
-
-      break;
-    case CIRC_MINOR_EVENT_CANNIBALIZED:
-      event_desc = "CANNIBALIZED";
-
-      {
-        /* event_tail can currently be up to 130 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        const struct timeval *old_timestamp_began = tv;
-        char tbuf[ISO_TIME_USEC_LEN+1];
-        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
-
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "",
-                     tbuf);
-      }
-
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
-                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
-                       (unsigned long)circ->global_identifier,
-                       event_desc, sp,
-                       circdesc,
-                       event_tail);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
- * interested controllers.
- */
-int
-control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                      int old_purpose)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-                                            old_purpose,
-                                            NULL);
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
- * created-time from <b>old_tv_created</b>: tell any interested controllers.
- */
-int
-control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                   int old_purpose,
-                                   const struct timeval *old_tv_created)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_CANNIBALIZED,
-                                            old_purpose,
-                                            old_tv_created);
-}
-
-/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
- * <b>buf</b>, determine the address:port combination requested on
- * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
- * failure. */
-static int
-write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
-{
-  char buf2[256];
-  if (conn->chosen_exit_name)
-    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
-      return -1;
-  if (!conn->socks_request)
-    return -1;
-  if (tor_snprintf(buf, len, "%s%s%s:%d",
-               conn->socks_request->address,
-               conn->chosen_exit_name ? buf2 : "",
-               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
-                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
-               conn->socks_request->port)<0)
-    return -1;
-  return 0;
-}
-
-/** Something has happened to the stream associated with AP connection
- * <b>conn</b>: tell any interested control connections. */
-int
-control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
-                            int reason_code)
-{
-  char reason_buf[64];
-  char addrport_buf[64];
-  const char *status;
-  circuit_t *circ;
-  origin_circuit_t *origin_circ = NULL;
-  char buf[256];
-  const char *purpose = "";
-  tor_assert(conn->socks_request);
-
-  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
-    return 0;
-
-  if (tp == STREAM_EVENT_CLOSED &&
-      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
-    return 0;
-
-  write_stream_target_to_buf(conn, buf, sizeof(buf));
-
-  reason_buf[0] = '\0';
-  switch (tp)
-    {
-    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
-    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
-    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
-    case STREAM_EVENT_FAILED: status = "FAILED"; break;
-    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
-    case STREAM_EVENT_NEW: status = "NEW"; break;
-    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
-    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
-    case STREAM_EVENT_REMAP: status = "REMAP"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (reason_code && (tp == STREAM_EVENT_FAILED ||
-                      tp == STREAM_EVENT_CLOSED ||
-                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
-    const char *reason_str = stream_end_reason_to_control_string(reason_code);
-    char *r = NULL;
-    if (!reason_str) {
-      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
-      reason_str = r;
-    }
-    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=END REMOTE_REASON=%s", reason_str);
-    else
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=%s", reason_str);
-    tor_free(r);
-  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
-    switch (reason_code) {
-    case REMAP_STREAM_SOURCE_CACHE:
-      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
-      break;
-    case REMAP_STREAM_SOURCE_EXIT:
-      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
-      break;
-    default:
-      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
-                   reason_code);
-      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
-      break;
-    }
-  }
-
-  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
-    /*
-     * When the control conn is an AF_UNIX socket and we have no address,
-     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
-     * dnsserv.c.
-     */
-    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
-      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
-                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
-    } else {
-      /*
-       * else leave it blank so control on AF_UNIX doesn't need to make
-       * something up.
-       */
-      addrport_buf[0] = '\0';
-    }
-  } else {
-    addrport_buf[0] = '\0';
-  }
-
-  if (tp == STREAM_EVENT_NEW_RESOLVE) {
-    purpose = " PURPOSE=DNS_REQUEST";
-  } else if (tp == STREAM_EVENT_NEW) {
-    if (conn->use_begindir) {
-      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
-      int linked_dir_purpose = -1;
-      if (linked && linked->type == CONN_TYPE_DIR)
-        linked_dir_purpose = linked->purpose;
-      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
-        purpose = " PURPOSE=DIR_UPLOAD";
-      else
-        purpose = " PURPOSE=DIR_FETCH";
-    } else
-      purpose = " PURPOSE=USER";
-  }
-
-  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-  if (circ && CIRCUIT_IS_ORIGIN(circ))
-    origin_circ = TO_ORIGIN_CIRCUIT(circ);
-  send_control_event(EVENT_STREAM_STATUS,
-                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
-                     (ENTRY_TO_CONN(conn)->global_identifier),
-                     status,
-                        origin_circ?
-                           (unsigned long)origin_circ->global_identifier : 0ul,
-                        buf, reason_buf, addrport_buf, purpose);
-
-  /* XXX need to specify its intended exit, etc? */
-
-  return 0;
-}
-
-/** Figure out the best name for the target router of an OR connection
- * <b>conn</b>, and write it into the <b>len</b>-character buffer
- * <b>name</b>. */
-static void
-orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
-{
-  const node_t *node = node_get_by_id(conn->identity_digest);
-  if (node) {
-    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
-    node_get_verbose_nickname(node, name);
-  } else if (! tor_digest_is_zero(conn->identity_digest)) {
-    name[0] = '$';
-    base16_encode(name+1, len-1, conn->identity_digest,
-                  DIGEST_LEN);
-  } else {
-    tor_snprintf(name, len, "%s:%d",
-                 conn->base_.address, conn->base_.port);
-  }
-}
-
-/** Called when the status of an OR connection <b>conn</b> changes: tell any
- * interested control connections. <b>tp</b> is the new status for the
- * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
- * may be the reason why.
- */
-int
-control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
-                             int reason)
-{
-  int ncircs = 0;
-  const char *status;
-  char name[128];
-  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
-
-  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
-    return 0;
-
-  switch (tp)
-    {
-    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
-    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
-    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
-    case OR_CONN_EVENT_NEW: status = "NEW"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (conn->chan) {
-    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
-  } else {
-    ncircs = 0;
-  }
-  ncircs += connection_or_get_num_circuits(conn);
-  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
-    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
-  }
-
-  orconn_target_get_name(name, sizeof(name), conn);
-  send_control_event(EVENT_OR_CONN_STATUS,
-                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
-                              name, status,
-                              reason ? " REASON=" : "",
-                              orconn_end_reason_to_control_string(reason),
-                              ncircs_buf,
-                              (conn->base_.global_identifier));
-
-  return 0;
-}
-
-/**
- * Print out STREAM_BW event for a single conn
- */
-int
-control_event_stream_bandwidth(edge_connection_t *edge_conn)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    if (!edge_conn->n_read && !edge_conn->n_written)
-      return 0;
-
-    tor_gettimeofday(&now);
-    format_iso_time_nospace_usec(tbuf, &now);
-    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                       (edge_conn->base_.global_identifier),
-                       (unsigned long)edge_conn->n_read,
-                       (unsigned long)edge_conn->n_written,
-                       tbuf);
-
-    edge_conn->n_written = edge_conn->n_read = 0;
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth streams have used. */
-int
-control_event_stream_bandwidth_used(void)
-{
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    smartlist_t *conns = get_connection_array();
-    edge_connection_t *edge_conn;
-    struct timeval now;
-    char tbuf[ISO_TIME_USEC_LEN+1];
-
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
-    {
-        if (conn->type != CONN_TYPE_AP)
-          continue;
-        edge_conn = TO_EDGE_CONN(conn);
-        if (!edge_conn->n_read && !edge_conn->n_written)
-          continue;
-
-        tor_gettimeofday(&now);
-        format_iso_time_nospace_usec(tbuf, &now);
-        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                           (edge_conn->base_.global_identifier),
-                           (unsigned long)edge_conn->n_read,
-                           (unsigned long)edge_conn->n_written,
-                           tbuf);
-
-        edge_conn->n_written = edge_conn->n_read = 0;
-    }
-    SMARTLIST_FOREACH_END(conn);
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control connections
- * how much bandwidth origin circuits have used. */
-int
-control_event_circ_bandwidth_used(void)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-
-    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
-  }
-  SMARTLIST_FOREACH_END(circ);
-
-  return 0;
-}
-
-/**
- * Emit a CIRC_BW event line for a specific circuit.
- *
- * This function sets the values it emits to 0, and does not emit
- * an event if there is no new data to report since the last call.
- *
- * Therefore, it may be called at any frequency.
- */
-int
-control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-
-  tor_assert(ocirc);
-
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  /* n_read_circ_bw and n_written_circ_bw are always updated
-   * when there is any new cell on a circuit, and set to 0 after
-   * the event, below.
-   *
-   * Therefore, checking them is sufficient to determine if there
-   * is new data to report. */
-  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
-    return 0;
-
-  tor_gettimeofday(&now);
-  format_iso_time_nospace_usec(tbuf, &now);
-  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
-                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
-                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
-                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
-                     ocirc->global_identifier,
-                     (unsigned long)ocirc->n_read_circ_bw,
-                     (unsigned long)ocirc->n_written_circ_bw,
-                     tbuf,
-                     (unsigned long)ocirc->n_delivered_read_circ_bw,
-                     (unsigned long)ocirc->n_overhead_read_circ_bw,
-                     (unsigned long)ocirc->n_delivered_written_circ_bw,
-                     (unsigned long)ocirc->n_overhead_written_circ_bw);
-  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-
-  return 0;
-}
-
-/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
-  * bandwidth counters. */
-int
-control_event_conn_bandwidth(connection_t *conn)
-{
-  const char *conn_type_str;
-  if (!get_options()->TestingEnableConnBwEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
-    return 0;
-  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
-    return 0;
-  switch (conn->type) {
-    case CONN_TYPE_OR:
-      conn_type_str = "OR";
-      break;
-    case CONN_TYPE_DIR:
-      conn_type_str = "DIR";
-      break;
-    case CONN_TYPE_EXIT:
-      conn_type_str = "EXIT";
-      break;
-    default:
-      return 0;
-  }
-  send_control_event(EVENT_CONN_BW,
-                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
-                     "READ=%lu WRITTEN=%lu\r\n",
-                     (conn->global_identifier),
-                     conn_type_str,
-                     (unsigned long)conn->n_read_conn_bw,
-                     (unsigned long)conn->n_written_conn_bw);
-  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth connections have used. */
-int
-control_event_conn_bandwidth_used(void)
-{
-  if (get_options()->TestingEnableConnBwEvent &&
-      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
-    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
-                      control_event_conn_bandwidth(conn));
-  }
-  return 0;
-}
-
-/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
- * cells, removed cells, and waiting times by cell command and direction.
- * Store results in <b>cell_stats</b>.  Free cell statistics of the
- * circuit afterwards. */
-void
-sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
-{
-  memset(cell_stats, 0, sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
-                          const testing_cell_stats_entry_t *, ent) {
-    tor_assert(ent->command <= CELL_COMMAND_MAX_);
-    if (!ent->removed && !ent->exitward) {
-      cell_stats->added_cells_appward[ent->command] += 1;
-    } else if (!ent->removed && ent->exitward) {
-      cell_stats->added_cells_exitward[ent->command] += 1;
-    } else if (!ent->exitward) {
-      cell_stats->removed_cells_appward[ent->command] += 1;
-      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
-    } else {
-      cell_stats->removed_cells_exitward[ent->command] += 1;
-      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
-    }
-  } SMARTLIST_FOREACH_END(ent);
-  circuit_clear_testing_cell_stats(circ);
-}
-
-/** Helper: append a cell statistics string to <code>event_parts</code>,
- * prefixed with <code>key</code>=.  Statistics consist of comma-separated
- * key:value pairs with lower-case command strings as keys and cell
- * numbers or total waiting times as values.  A key:value pair is included
- * if the entry in <code>include_if_non_zero</code> is not zero, but with
- * the (possibly zero) entry from <code>number_to_include</code>.  Both
- * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
- * entry in <code>include_if_non_zero</code> is positive, no string will
- * be added to <code>event_parts</code>. */
-void
-append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
-                             const uint64_t *include_if_non_zero,
-                             const uint64_t *number_to_include)
-{
-  smartlist_t *key_value_strings = smartlist_new();
-  int i;
-  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
-    if (include_if_non_zero[i] > 0) {
-      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
-                             cell_command_to_string(i),
-                             (number_to_include[i]));
-    }
-  }
-  if (smartlist_len(key_value_strings) > 0) {
-    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
-    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
-    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
-    tor_free(joined);
-  }
-  smartlist_free(key_value_strings);
-}
-
-/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
- * CELL_STATS event and write result string to <b>event_string</b>. */
-void
-format_cell_stats(char **event_string, circuit_t *circ,
-                  cell_stats_t *cell_stats)
-{
-  smartlist_t *event_parts = smartlist_new();
-  if (CIRCUIT_IS_ORIGIN(circ)) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "ID=%lu",
-                 (unsigned long)ocirc->global_identifier);
-  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
-    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
-                 (unsigned long)or_circ->p_circ_id);
-    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
-                 (or_circ->p_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "InboundAdded",
-                                 cell_stats->added_cells_appward,
-                                 cell_stats->added_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundRemoved",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->removed_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundTime",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->total_time_appward);
-  }
-  if (circ->n_chan) {
-    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
-                     (unsigned long)circ->n_circ_id);
-    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
-                 (circ->n_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "OutboundAdded",
-                                 cell_stats->added_cells_exitward,
-                                 cell_stats->added_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundRemoved",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->removed_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundTime",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->total_time_exitward);
-  }
-  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
-  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
-  smartlist_free(event_parts);
-}
-
-/** A second or more has elapsed: tell any interested control connection
- * how many cells have been processed for a given circuit. */
-int
-control_event_circuit_cell_stats(void)
-{
-  cell_stats_t *cell_stats;
-  char *event_string;
-  if (!get_options()->TestingEnableCellStatsEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
-    return 0;
-  cell_stats = tor_malloc(sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!circ->testing_cell_stats)
-      continue;
-    sum_up_cell_stats_by_command(circ, cell_stats);
-    format_cell_stats(&event_string, circ, cell_stats);
-    send_control_event(EVENT_CELL_STATS,
-                       "650 CELL_STATS %s\r\n", event_string);
-    tor_free(event_string);
-  }
-  SMARTLIST_FOREACH_END(circ);
-  tor_free(cell_stats);
-  return 0;
-}
-
-/* about 5 minutes worth. */
-#define N_BW_EVENTS_TO_CACHE 300
-/* Index into cached_bw_events to next write. */
-static int next_measurement_idx = 0;
-/* number of entries set in n_measurements */
-static int n_measurements = 0;
-static struct cached_bw_event_s {
-  uint32_t n_read;
-  uint32_t n_written;
-} cached_bw_events[N_BW_EVENTS_TO_CACHE];
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth we used. */
-int
-control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
-{
-  cached_bw_events[next_measurement_idx].n_read = n_read;
-  cached_bw_events[next_measurement_idx].n_written = n_written;
-  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
-    next_measurement_idx = 0;
-  if (n_measurements < N_BW_EVENTS_TO_CACHE)
-    ++n_measurements;
-
-  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
-    send_control_event(EVENT_BANDWIDTH_USED,
-                       "650 BW %lu %lu\r\n",
-                       (unsigned long)n_read,
-                       (unsigned long)n_written);
-  }
-
-  return 0;
-}
-
-STATIC char *
-get_bw_samples(void)
-{
-  int i;
-  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
-    % N_BW_EVENTS_TO_CACHE;
-  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-
-  smartlist_t *elements = smartlist_new();
-
-  for (i = 0; i < n_measurements; ++i) {
-    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-    const struct cached_bw_event_s *bwe = &cached_bw_events[idx];
-
-    smartlist_add_asprintf(elements, "%u,%u",
-                           (unsigned)bwe->n_read,
-                           (unsigned)bwe->n_written);
-
-    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
-  }
-
-  char *result = smartlist_join_strings(elements, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-
-  return result;
-}
-
-/** Called when we are sending a log message to the controllers: suspend
- * sending further log messages to the controllers until we're done.  Used by
- * CONN_LOG_PROTECT. */
-void
-disable_control_logging(void)
-{
-  ++disable_log_messages;
-}
-
-/** We're done sending a log message to the controllers: re-enable controller
- * logging.  Used by CONN_LOG_PROTECT. */
-void
-enable_control_logging(void)
-{
-  if (--disable_log_messages < 0)
-    tor_assert(0);
-}
-
-/** We got a log message: tell any interested control connections. */
-void
-control_event_logmsg(int severity, uint32_t domain, const char *msg)
-{
-  int event;
-
-  /* Don't even think of trying to add stuff to a buffer from a cpuworker
-   * thread. (See #25987 for plan to fix.) */
-  if (! in_main_thread())
-    return;
-
-  if (disable_log_messages)
-    return;
-
-  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
-      severity <= LOG_NOTICE) {
-    char *esc = esc_for_log(msg);
-    ++disable_log_messages;
-    control_event_general_status(severity, "BUG REASON=%s", esc);
-    --disable_log_messages;
-    tor_free(esc);
-  }
-
-  event = log_severity_to_event(severity);
-  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
-    char *b = NULL;
-    const char *s;
-    if (strchr(msg, '\n')) {
-      char *cp;
-      b = tor_strdup(msg);
-      for (cp = b; *cp; ++cp)
-        if (*cp == '\r' || *cp == '\n')
-          *cp = ' ';
-    }
-    switch (severity) {
-      case LOG_DEBUG: s = "DEBUG"; break;
-      case LOG_INFO: s = "INFO"; break;
-      case LOG_NOTICE: s = "NOTICE"; break;
-      case LOG_WARN: s = "WARN"; break;
-      case LOG_ERR: s = "ERR"; break;
-      default: s = "UnknownLogSeverity"; break;
-    }
-    ++disable_log_messages;
-    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
-    if (severity == LOG_ERR) {
-      /* Force a flush, since we may be about to die horribly */
-      queued_events_flush_all(1);
-    }
-    --disable_log_messages;
-    tor_free(b);
-  }
-}
-
-/**
- * Logging callback: called when there is a queued pending log callback.
- */
-void
-control_event_logmsg_pending(void)
-{
-  if (! in_main_thread()) {
-    /* We can't handle this case yet, since we're using a
-     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
-     * use a different mechanism instead: see #25987.
-     **/
-    return;
-  }
-  tor_assert(flush_queued_events_event);
-  mainloop_event_activate(flush_queued_events_event);
-}
-
-/** Called whenever we receive new router descriptors: tell any
- * interested control connections.  <b>routers</b> is a list of
- * routerinfo_t's.
- */
-int
-control_event_descriptors_changed(smartlist_t *routers)
-{
-  char *msg;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
-    return 0;
-
-  {
-    smartlist_t *names = smartlist_new();
-    char *ids;
-    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
-        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
-        router_get_verbose_nickname(b, ri);
-        smartlist_add(names, b);
-      });
-    ids = smartlist_join_strings(names, " ", 0, NULL);
-    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
-    send_control_event_string(EVENT_NEW_DESC,  msg);
-    tor_free(ids);
-    tor_free(msg);
-    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
-    smartlist_free(names);
-  }
-  return 0;
-}
-
-/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
- * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
- * <b>error</b> is non-NULL, it is an error code describing the failure
- * mode of the mapping.
- */
-int
-control_event_address_mapped(const char *from, const char *to, time_t expires,
-                             const char *error, const int cached)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
-    return 0;
-
-  if (expires < 3 || expires == TIME_MAX)
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s NEVER %s%s"
-                                "CACHED=\"%s\"\r\n",
-                                  from, to, error?error:"", error?" ":"",
-                                cached?"YES":"NO");
-  else {
-    char buf[ISO_TIME_LEN+1];
-    char buf2[ISO_TIME_LEN+1];
-    format_local_iso_time(buf,expires);
-    format_iso_time(buf2,expires);
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s \"%s\""
-                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
-                                from, to, buf,
-                                error?error:"", error?" ":"",
-                                buf2, cached?"YES":"NO");
-  }
-
-  return 0;
-}
-
 /** Cached liveness for network liveness events and GETINFO
  */
 
 static int network_is_live = 0;
 
-static int
+int
 get_cached_network_liveness(void)
 {
   return network_is_live;
 }
 
-static void
+void
 set_cached_network_liveness(int liveness)
 {
   network_is_live = liveness;
 }
 
-/** The network liveness has changed; this is called from circuitstats.c
- * whenever we receive a cell, or when timeout expires and we assume the
- * network is down. */
-int
-control_event_network_liveness_update(int liveness)
-{
-  if (liveness > 0) {
-    if (get_cached_network_liveness() <= 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(1);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS UP\r\n");
-    }
-    /* else was already live, no-op */
-  } else {
-    if (get_cached_network_liveness() > 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(0);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS DOWN\r\n");
-    }
-    /* else was already dead, no-op */
-  }
-
-  return 0;
-}
-
-/** Helper function for NS-style events. Constructs and sends an event
- * of type <b>event</b> with string <b>event_string</b> out of the set of
- * networkstatuses <b>statuses</b>. Currently it is used for NS events
- * and NEWCONSENSUS events. */
-static int
-control_event_networkstatus_changed_helper(smartlist_t *statuses,
-                                           uint16_t event,
-                                           const char *event_string)
-{
-  smartlist_t *strs;
-  char *s, *esc = NULL;
-  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
-    return 0;
-
-  strs = smartlist_new();
-  smartlist_add_strdup(strs, "650+");
-  smartlist_add_strdup(strs, event_string);
-  smartlist_add_strdup(strs, "\r\n");
-  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
-    {
-      s = networkstatus_getinfo_helper_single(rs);
-      if (!s) continue;
-      smartlist_add(strs, s);
-    });
-
-  s = smartlist_join_strings(strs, "", 0, NULL);
-  write_escaped_data(s, strlen(s), &esc);
-  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
-  smartlist_free(strs);
-  tor_free(s);
-  send_control_event_string(event,  esc);
-  send_control_event_string(event,
-                            "650 OK\r\n");
-
-  tor_free(esc);
-  return 0;
-}
-
-/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
- * an NS event to any controller that cares. */
-int
-control_event_networkstatus_changed(smartlist_t *statuses)
-{
-  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
-}
-
-/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
- * event consisting of an NS-style line for each relay in the consensus. */
-int
-control_event_newconsensus(const networkstatus_t *consensus)
-{
-  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
-    return 0;
-  return control_event_networkstatus_changed_helper(
-           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
-}
-
-/** Called when we compute a new circuitbuildtimeout */
-int
-control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                               const char *args)
-{
-  const char *type_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
-    return 0;
-
-  switch (type) {
-    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
-      type_string = "COMPUTED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESET:
-      type_string = "RESET";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
-      type_string = "SUSPENDED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_DISCARD:
-      type_string = "DISCARD";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESUME:
-      type_string = "RESUME";
-      break;
-    default:
-      type_string = "UNKNOWN";
-      break;
-  }
-
-  send_control_event(EVENT_BUILDTIMEOUT_SET,
-                     "650 BUILDTIMEOUT_SET %s %s\r\n",
-                     type_string, args);
-
-  return 0;
-}
-
-/** Called when a signal has been processed from signal_callback */
-int
-control_event_signal(uintptr_t signal_num)
-{
-  const char *signal_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
-    return 0;
-
-  switch (signal_num) {
-    case SIGHUP:
-      signal_string = "RELOAD";
-      break;
-    case SIGUSR1:
-      signal_string = "DUMP";
-      break;
-    case SIGUSR2:
-      signal_string = "DEBUG";
-      break;
-    case SIGNEWNYM:
-      signal_string = "NEWNYM";
-      break;
-    case SIGCLEARDNSCACHE:
-      signal_string = "CLEARDNSCACHE";
-      break;
-    case SIGHEARTBEAT:
-      signal_string = "HEARTBEAT";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
-               (unsigned long)signal_num);
-      return -1;
-  }
-
-  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
-                     signal_string);
-  return 0;
-}
-
-/** Called when a single local_routerstatus_t has changed: Sends an NS event
- * to any controller that cares. */
-int
-control_event_networkstatus_changed_single(const routerstatus_t *rs)
-{
-  smartlist_t *statuses;
-  int r;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NS))
-    return 0;
-
-  statuses = smartlist_new();
-  smartlist_add(statuses, (void*)rs);
-  r = control_event_networkstatus_changed(statuses);
-  smartlist_free(statuses);
-  return r;
-}
-
-/** Our own router descriptor has changed; tell any controllers that care.
- */
-int
-control_event_my_descriptor_changed(void)
-{
-  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
-  return 0;
-}
-
-/** Helper: sends a status event where <b>type</b> is one of
- * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
- * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
- * string corresponding to <b>args</b>. */
-static int
-control_event_status(int type, int severity, const char *format, va_list args)
-{
-  char *user_buf = NULL;
-  char format_buf[160];
-  const char *status, *sev;
-
-  switch (type) {
-    case EVENT_STATUS_GENERAL:
-      status = "STATUS_GENERAL";
-      break;
-    case EVENT_STATUS_CLIENT:
-      status = "STATUS_CLIENT";
-      break;
-    case EVENT_STATUS_SERVER:
-      status = "STATUS_SERVER";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status type %d", type);
-      return -1;
-  }
-  switch (severity) {
-    case LOG_NOTICE:
-      sev = "NOTICE";
-      break;
-    case LOG_WARN:
-      sev = "WARN";
-      break;
-    case LOG_ERR:
-      sev = "ERR";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
-      return -1;
-  }
-  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
-                   status, sev)<0) {
-    log_warn(LD_BUG, "Format string too long.");
-    return -1;
-  }
-  tor_vasprintf(&user_buf, format, args);
-
-  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
-  tor_free(user_buf);
-  return 0;
-}
-
-#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
-  int r;                                                        \
-  do {                                                          \
-    va_list ap;                                                 \
-    if (!EVENT_IS_INTERESTING(event))                           \
-      return 0;                                                 \
-                                                                \
-    va_start(ap, format);                                       \
-    r = control_event_status((event), (sev), format, ap);       \
-    va_end(ap);                                                 \
-  } while (0)
-
-/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_general_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_general_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_client_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_client_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_server_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_server_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Called when the status of an entry guard with the given <b>nickname</b>
- * and identity <b>digest</b> has changed to <b>status</b>: tells any
- * controllers that care. */
-int
-control_event_guard(const char *nickname, const char *digest,
-                    const char *status)
-{
-  char hbuf[HEX_DIGEST_LEN+1];
-  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
-  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
-    return 0;
-
-  {
-    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
-    const node_t *node = node_get_by_id(digest);
-    if (node) {
-      node_get_verbose_nickname(node, buf);
-    } else {
-      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
-    }
-    send_control_event(EVENT_GUARD,
-                       "650 GUARD ENTRY %s %s\r\n", buf, status);
-  }
-  return 0;
-}
-
-/** Called when a configuration option changes. This is generally triggered
- * by SETCONF requests and RELOAD/SIGHUP signals. The <b>elements</b> is
- * a smartlist_t containing (key, value, ...) pairs in sequence.
- * <b>value</b> can be NULL. */
-int
-control_event_conf_changed(const smartlist_t *elements)
-{
-  int i;
-  char *result;
-  smartlist_t *lines;
-  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) ||
-      smartlist_len(elements) == 0) {
-    return 0;
-  }
-  lines = smartlist_new();
-  for (i = 0; i < smartlist_len(elements); i += 2) {
-    char *k = smartlist_get(elements, i);
-    char *v = smartlist_get(elements, i+1);
-    if (v == NULL) {
-      smartlist_add_asprintf(lines, "650-%s", k);
-    } else {
-      smartlist_add_asprintf(lines, "650-%s=%s", k, v);
-    }
-  }
-  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
-  send_control_event(EVENT_CONF_CHANGED,
-    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
-  tor_free(result);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  return 0;
-}
-
-/** Helper: Return a newly allocated string containing a path to the
- * file where we store our authentication cookie. */
-char *
-get_controller_cookie_file_name(void)
-{
-  const or_options_t *options = get_options();
-  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
-    return tor_strdup(options->CookieAuthFile);
-  } else {
-    return get_datadir_fname("control_auth_cookie");
-  }
-}
-
-/* Initialize the cookie-based authentication system of the
- * ControlPort. If <b>enabled</b> is 0, then disable the cookie
- * authentication system.  */
-int
-init_control_cookie_authentication(int enabled)
-{
-  char *fname = NULL;
-  int retval;
-
-  if (!enabled) {
-    authentication_cookie_is_set = 0;
-    return 0;
-  }
-
-  fname = get_controller_cookie_file_name();
-  retval = init_cookie_authentication(fname, "", /* no header */
-                                      AUTHENTICATION_COOKIE_LEN,
-                                   get_options()->CookieAuthFileGroupReadable,
-                                      &authentication_cookie,
-                                      &authentication_cookie_is_set);
-  tor_free(fname);
-  return retval;
-}
-
 /** A copy of the process specifier of Tor's owning controller, or
  * NULL if this Tor instance is not currently owned by a process. */
 static char *owning_controller_process_spec = NULL;
@@ -7008,895 +601,12 @@ monitor_owning_controller_process(const char *process_spec)
   }
 }
 
-/** Convert the name of a bootstrapping phase <b>s</b> into strings
- * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
-static int
-bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
-                           const char **summary)
-{
-  switch (s) {
-    case BOOTSTRAP_STATUS_UNDEF:
-      *tag = "undef";
-      *summary = "Undefined";
-      break;
-    case BOOTSTRAP_STATUS_STARTING:
-      *tag = "starting";
-      *summary = "Starting";
-      break;
-    case BOOTSTRAP_STATUS_CONN_DIR:
-      *tag = "conn_dir";
-      *summary = "Connecting to directory server";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE:
-      *tag = "status_handshake";
-      *summary = "Finishing handshake";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_DIR:
-      *tag = "handshake_dir";
-      *summary = "Finishing handshake with directory server";
-      break;
-    case BOOTSTRAP_STATUS_ONEHOP_CREATE:
-      *tag = "onehop_create";
-      *summary = "Establishing an encrypted directory connection";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_STATUS:
-      *tag = "requesting_status";
-      *summary = "Asking for networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_STATUS:
-      *tag = "loading_status";
-      *summary = "Loading networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_KEYS:
-      *tag = "loading_keys";
-      *summary = "Loading authority key certs";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS:
-      *tag = "requesting_descriptors";
-      /* XXXX this appears to incorrectly report internal on most loads */
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Asking for relay descriptors for internal paths" :
-        "Asking for relay descriptors";
-      break;
-    /* If we're sure there are no exits in the consensus,
-     * inform the controller by adding "internal"
-     * to the status summaries.
-     * (We only check this while loading descriptors,
-     * so we may not know in the earlier stages.)
-     * But if there are exits, we can't be sure whether
-     * we're creating internal or exit paths/circuits.
-     * XXXX Or should be use different tags or statuses
-     * for internal and exit/all? */
-    case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS:
-      *tag = "loading_descriptors";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Loading relay descriptors for internal paths" :
-        "Loading relay descriptors";
-      break;
-    case BOOTSTRAP_STATUS_CONN_OR:
-      *tag = "conn_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Connecting to the Tor network internally" :
-        "Connecting to the Tor network";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_OR:
-      *tag = "handshake_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Finishing handshake with first hop of internal circuit" :
-        "Finishing handshake with first hop";
-      break;
-    case BOOTSTRAP_STATUS_CIRCUIT_CREATE:
-      *tag = "circuit_create";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Establishing an internal Tor circuit" :
-        "Establishing a Tor circuit";
-      break;
-    case BOOTSTRAP_STATUS_DONE:
-      *tag = "done";
-      *summary = "Done";
-      break;
-    default:
-//      log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s);
-      *tag = *summary = "unknown";
-      return -1;
-  }
-  return 0;
-}
-
-/** What percentage through the bootstrap process are we? We remember
- * this so we can avoid sending redundant bootstrap status events, and
- * so we can guess context for the bootstrap messages which are
- * ambiguous. It starts at 'undef', but gets set to 'starting' while
- * Tor initializes. */
-static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_percent, but only takes on the enumerated values in
- * bootstrap_status_t.
- */
-static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** As bootstrap_percent, but holds the bootstrapping level at which we last
- * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
- * to avoid flooding the log with a new message every time we get a few more
- * microdescriptors */
-static int notice_bootstrap_percent = 0;
-
-/** How many problems have we had getting to the next bootstrapping phase?
- * These include failure to establish a connection to a Tor relay,
- * failures to finish the TLS handshake, failures to validate the
- * consensus document, etc. */
-static int bootstrap_problems = 0;
-
-/** We only tell the controller once we've hit a threshold of problems
- * for the current phase. */
-#define BOOTSTRAP_PROBLEM_THRESHOLD 10
-
-/** When our bootstrapping progress level changes, but our bootstrapping
- * status has not advanced, we only log at NOTICE when we have made at least
- * this much progress.
- */
-#define BOOTSTRAP_PCT_INCREMENT 5
-
-/** Do the actual logging and notifications for
- * control_event_bootstrap().  Doesn't change any state beyond that.
- */
-static void
-control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
-                             int progress)
-{
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *tag, *summary;
-
-  bootstrap_status_to_string(status, &tag, &summary);
-  /* Locally reset status if there's incremental progress */
-  if (progress)
-    status = progress;
-
-  tor_log(loglevel, LD_CONTROL,
-          "Bootstrapped %d%%: %s", status, summary);
-  tor_snprintf(buf, sizeof(buf),
-               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
-               status, tag, summary);
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "NOTICE %s", buf);
-  control_event_client_status(LOG_NOTICE, "%s", buf);
-}
-
-/** Called when Tor has made progress at bootstrapping its directory
- * information and initial circuits.
- *
- * <b>status</b> is the new status, that is, what task we will be doing
- * next. <b>progress</b> is zero if we just started this task, else it
- * represents progress on the task.
- */
-void
-control_event_bootstrap(bootstrap_status_t status, int progress)
-{
-  int loglevel = LOG_NOTICE;
-
-  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  /* special case for handshaking status, since our TLS handshaking code
-   * can't distinguish what the connection is going to be for. */
-  if (status == BOOTSTRAP_STATUS_HANDSHAKE) {
-    if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_DIR;
-    } else {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_OR;
-    }
-  }
-
-  if (status <= bootstrap_percent) {
-    /* If there's no new progress, return early. */
-    if (!progress || progress <= bootstrap_percent)
-      return;
-    /* Log at INFO if not enough progress happened. */
-    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
-      loglevel = LOG_INFO;
-  }
-
-  control_event_bootstrap_core(loglevel, status, progress);
-
-  if (status > bootstrap_percent) {
-    bootstrap_phase = status; /* new milestone reached */
-    bootstrap_percent = status;
-  }
-  if (progress > bootstrap_percent) {
-    /* incremental progress within a milestone */
-    bootstrap_percent = progress;
-    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
-  }
-  if (loglevel == LOG_NOTICE &&
-      bootstrap_percent > notice_bootstrap_percent) {
-    /* Remember that we gave a notice at this level. */
-    notice_bootstrap_percent = bootstrap_percent;
-  }
-}
-
-/** Flag whether we've opened an OR_CONN yet  */
-static int bootstrap_first_orconn = 0;
-
-/** Like bootstrap_phase, but for (possibly deferred) directory progress */
-static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
-static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-
-/** Defer directory info bootstrap events until we have successfully
- * completed our first connection to a router.  */
-void
-control_event_boot_dir(bootstrap_status_t status, int progress)
-{
-  if (status > bootstrap_dir_progress) {
-    bootstrap_dir_progress = status;
-    bootstrap_dir_phase = status;
-  }
-  if (progress && progress >= bootstrap_dir_progress) {
-    bootstrap_dir_progress = progress;
-  }
-
-  /* Don't report unless we have successfully opened at least one OR_CONN */
-  if (!bootstrap_first_orconn)
-    return;
-
-  control_event_bootstrap(status, progress);
-}
-
-/** Set a flag to allow reporting of directory bootstrap progress.
- * (Code that reports completion of an OR_CONN calls this.)  Also,
- * report directory progress so far. */
-void
-control_event_boot_first_orconn(void)
-{
-  bootstrap_first_orconn = 1;
-  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a human-readable hint
- * as to why, and <b>reason</b> provides a controller-facing short
- * tag.  <b>conn</b> is the connection that caused this problem and
- * can be NULL if a connection cannot be easily identified.
- */
-void
-control_event_bootstrap_problem(const char *warn, const char *reason,
-                                const connection_t *conn, int dowarn)
-{
-  int status = bootstrap_percent;
-  const char *tag = "", *summary = "";
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *recommendation = "ignore";
-  int severity;
-  char *or_id = NULL, *hostaddr = NULL;
-  or_connection_t *or_conn = NULL;
-
-  /* bootstrap_percent must not be in "undefined" state here. */
-  tor_assert(status >= 0);
-
-  if (bootstrap_percent == 100)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  bootstrap_problems++;
-
-  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
-    dowarn = 1;
-
-  /* Don't warn about our bootstrapping status if we are hibernating or
-   * shutting down. */
-  if (we_are_hibernating())
-    dowarn = 0;
-
-  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
-
-  severity = dowarn ? LOG_WARN : LOG_INFO;
-
-  if (dowarn)
-    recommendation = "warn";
-
-  if (conn && conn->type == CONN_TYPE_OR) {
-    /* XXX TO_OR_CONN can't deal with const */
-    or_conn = TO_OR_CONN((connection_t *)conn);
-    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
-  } else {
-    or_id = tor_strdup("?");
-  }
-
-  if (conn)
-    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
-  else
-    hostaddr = tor_strdup("?");
-
-  log_fn(severity,
-         LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; "
-         "count %d; recommendation %s; host %s at %s)",
-         status, summary, warn, reason,
-         bootstrap_problems, recommendation,
-         or_id, hostaddr);
-
-  connection_or_report_broken_states(severity, LD_HANDSHAKE);
-
-  tor_snprintf(buf, sizeof(buf),
-      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
-      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
-      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
-      recommendation,
-      or_id, hostaddr);
-
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "WARN %s", buf);
-  control_event_client_status(LOG_WARN, "%s", buf);
-
-  tor_free(hostaddr);
-  tor_free(or_id);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a hint as to why, and
- * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
- * is the connection that caused this problem.
- */
-MOCK_IMPL(void,
-control_event_bootstrap_prob_or, (const char *warn, int reason,
-                                  or_connection_t *or_conn))
-{
-  int dowarn = 0;
-
-  if (or_conn->have_noted_bootstrap_problem)
-    return;
-
-  or_conn->have_noted_bootstrap_problem = 1;
-
-  if (reason == END_OR_CONN_REASON_NO_ROUTE)
-    dowarn = 1;
-
-  /* If we are using bridges and all our OR connections are now
-     closed, it means that we totally failed to connect to our
-     bridges. Throw a warning. */
-  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
-    dowarn = 1;
-
-  control_event_bootstrap_problem(warn,
-                                  orconn_end_reason_to_control_string(reason),
-                                  TO_CONN(or_conn), dowarn);
-}
-
-/** We just generated a new summary of which countries we've seen clients
- * from recently. Send a copy to the controller in case it wants to
- * display it for the user. */
-void
-control_event_clients_seen(const char *controller_str)
-{
-  send_control_event(EVENT_CLIENTS_SEEN,
-    "650 CLIENTS_SEEN %s\r\n", controller_str);
-}
-
-/** A new pluggable transport called <b>transport_name</b> was
- *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
- *  "server" or "client" depending on the mode of the pluggable
- *  transport.
- *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
- */
-void
-control_event_transport_launched(const char *mode, const char *transport_name,
-                                 tor_addr_t *addr, uint16_t port)
-{
-  send_control_event(EVENT_TRANSPORT_LAUNCHED,
-                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
-                     mode, transport_name, fmt_addr(addr), port);
-}
-
-/** Convert rendezvous auth type to string for HS_DESC control events
- */
-const char *
-rend_auth_type_to_string(rend_auth_type_t auth_type)
-{
-  const char *str;
-
-  switch (auth_type) {
-    case REND_NO_AUTH:
-      str = "NO_AUTH";
-      break;
-    case REND_BASIC_AUTH:
-      str = "BASIC_AUTH";
-      break;
-    case REND_STEALTH_AUTH:
-      str = "STEALTH_AUTH";
-      break;
-    default:
-      str = "UNKNOWN";
-  }
-
-  return str;
-}
-
-/** Return a longname the node whose identity is <b>id_digest</b>. If
- * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
- * returned instead.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-MOCK_IMPL(const char *,
-node_describe_longname_by_id,(const char *id_digest))
-{
-  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
-  node_get_verbose_nickname_by_id(id_digest, longname);
-  return longname;
-}
-
-/** Return either the onion address if the given pointer is a non empty
- * string else the unknown string. */
-static const char *
-rend_hsaddress_str_or_unknown(const char *onion_address)
-{
-  static const char *str_unknown = "UNKNOWN";
-  const char *str_ret = str_unknown;
-
-  /* No valid pointer, unknown it is. */
-  if (!onion_address) {
-    goto end;
-  }
-  /* Empty onion address thus we don't know, unknown it is. */
-  if (onion_address[0] == '\0') {
-    goto end;
-  }
-  /* All checks are good so return the given onion address. */
-  str_ret = onion_address;
-
- end:
-  return str_ret;
-}
-
-/** send HS_DESC requested event.
- *
- * <b>rend_query</b> is used to fetch requested onion address and auth type.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id_base32</b> is the ID of requested hs descriptor.
- * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
- */
-void
-control_event_hs_descriptor_requested(const char *onion_address,
-                                      rend_auth_type_t auth_type,
-                                      const char *id_digest,
-                                      const char *desc_id,
-                                      const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** For an HS descriptor query <b>rend_data</b>, using the
- * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
- * which descriptor ID in the query is the right one.
- *
- * Return a pointer of the binary descriptor ID found in the query's object
- * or NULL if not found. */
-static const char *
-get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
-{
-  int replica;
-  const char *desc_id = NULL;
-  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
-
-  /* Possible if the fetch was done using a descriptor ID. This means that
-   * the HSFETCH command was used. */
-  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
-    desc_id = rend_data_v2->desc_id_fetch;
-    goto end;
-  }
-
-  /* Without a directory fingerprint at this stage, we can't do much. */
-  if (hsdir_fp == NULL) {
-     goto end;
-  }
-
-  /* OK, we have an onion address so now let's find which descriptor ID
-   * is the one associated with the HSDir fingerprint. */
-  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
-       replica++) {
-    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
-
-    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
-      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
-        /* Found it! This descriptor ID is the right one. */
-        desc_id = digest;
-        goto end;
-      }
-    } SMARTLIST_FOREACH_END(fingerprint);
-  }
-
- end:
-  return desc_id;
-}
-
-/** send HS_DESC CREATED event when a local service generates a descriptor.
- *
- * <b>onion_address</b> is service address.
- * <b>desc_id</b> is the descriptor ID.
- * <b>replica</b> is the the descriptor replica number. If it is negative, it
- * is ignored.
- */
-void
-control_event_hs_descriptor_created(const char *onion_address,
-                                    const char *desc_id,
-                                    int replica)
-{
-  char *replica_field = NULL;
-
-  if (BUG(!onion_address || !desc_id)) {
-    return;
-  }
-
-  if (replica >= 0) {
-    tor_asprintf(&replica_field, " REPLICA=%d", replica);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
-                     onion_address, desc_id,
-                     replica_field ? replica_field : "");
-  tor_free(replica_field);
-}
-
-/** send HS_DESC upload event.
- *
- * <b>onion_address</b> is service address.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id</b> is the ID of requested hs descriptor.
- */
-void
-control_event_hs_descriptor_upload(const char *onion_address,
-                                   const char *id_digest,
-                                   const char *desc_id,
-                                   const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!onion_address || !id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
-                     onion_address,
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hsv2_descriptor_received
- * control_event_hsv2_descriptor_failed
- * control_event_hsv3_descriptor_failed
- *
- * So do not call this function directly.
- */
-static void
-event_hs_descriptor_receive_end(const char *action,
-                                const char *onion_address,
-                                const char *desc_id,
-                                rend_auth_type_t auth_type,
-                                const char *hsdir_id_digest,
-                                const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !onion_address)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s %s %s%s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     desc_id ? desc_id : "",
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hs_descriptor_uploaded
- * control_event_hs_descriptor_upload_failed
- *
- * So do not call this function directly.
- */
-void
-control_event_hs_descriptor_upload_end(const char *action,
-                                       const char *onion_address,
-                                       const char *id_digest,
-                                       const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !id_digest)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     node_describe_longname_by_id(id_digest),
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC RECEIVED event
- *
- * called when we successfully received a hidden service descriptor.
- */
-void
-control_event_hsv2_descriptor_received(const char *onion_address,
-                                       const rend_data_t *rend_data,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/* Send HS_DESC RECEIVED event
- *
- * Called when we successfully received a hidden service descriptor. */
-void
-control_event_hsv3_descriptor_received(const char *onion_address,
-                                       const char *desc_id,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/** send HS_DESC UPLOADED event
- *
- * called when we successfully uploaded a hidden service descriptor.
- */
-void
-control_event_hs_descriptor_uploaded(const char *id_digest,
-                                     const char *onion_address)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-
-  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
-                                         id_digest, NULL);
-}
-
-/** Send HS_DESC event to inform controller that query <b>rend_data</b>
- * failed to retrieve hidden service descriptor from directory identified by
- * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
- * add it to REASON= field.
- */
-void
-control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
-                                  desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC event to inform controller that the query to
- * <b>onion_address</b> failed to retrieve hidden service descriptor
- * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
- * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
- * field. */
-void
-control_event_hsv3_descriptor_failed(const char *onion_address,
-                                     const char *desc_id,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !reason)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC_CONTENT event after completion of a successful fetch
- * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
- * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
- * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
- * not trigger the control event. */
-void
-control_event_hs_descriptor_content(const char *onion_address,
-                                    const char *desc_id,
-                                    const char *hsdir_id_digest,
-                                    const char *content)
-{
-  static const char *event_name = "HS_DESC_CONTENT";
-  char *esc_content = NULL;
-
-  if (!onion_address || !desc_id) {
-    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
-             onion_address, desc_id);
-    return;
-  }
-
-  if (content == NULL) {
-    /* Point it to empty content so it can still be escaped. */
-    content = "";
-  }
-  write_escaped_data(content, strlen(content), &esc_content);
-
-  send_control_event(EVENT_HS_DESC_CONTENT,
-                     "650+%s %s %s %s\r\n%s650 OK\r\n",
-                     event_name,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     desc_id,
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     esc_content);
-  tor_free(esc_content);
-}
-
-/** Send HS_DESC event to inform controller upload of hidden service
- * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
- * is not NULL, add it to REASON= field.
- */
-void
-control_event_hs_descriptor_upload_failed(const char *id_digest,
-                                          const char *onion_address,
-                                          const char *reason)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-  control_event_hs_descriptor_upload_end("FAILED", onion_address,
-                                         id_digest, reason);
-}
-
 /** Free any leftover allocated memory of the control.c subsystem. */
 void
 control_free_all(void)
 {
-  smartlist_t *queued_events = NULL;
-
-  stats_prev_n_read = stats_prev_n_written = 0;
-
-  if (authentication_cookie) /* Free the auth cookie */
-    tor_free(authentication_cookie);
-  if (detached_onion_services) { /* Free the detached onion services */
-    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
-    smartlist_free(detached_onion_services);
-  }
-
-  if (queued_control_events_lock) {
-    tor_mutex_acquire(queued_control_events_lock);
-    flush_queued_event_pending = 0;
-    queued_events = queued_control_events;
-    queued_control_events = NULL;
-    tor_mutex_release(queued_control_events_lock);
-  }
-  if (queued_events) {
-    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
-                      queued_event_free(ev));
-    smartlist_free(queued_events);
-  }
-  if (flush_queued_events_event) {
-    mainloop_event_free(flush_queued_events_event);
-    flush_queued_events_event = NULL;
-  }
-  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-  notice_bootstrap_percent = 0;
-  bootstrap_problems = 0;
-  bootstrap_first_orconn = 0;
-  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-  authentication_cookie_is_set = 0;
-  global_event_mask = 0;
-  disable_log_messages = 0;
-  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
-}
-
-#ifdef TOR_UNIT_TESTS
-/* For testing: change the value of global_event_mask */
-void
-control_testing_set_global_event_mask(uint64_t mask)
-{
-  global_event_mask = mask;
+  control_auth_free_all();
+  control_events_free_all();
+  control_cmd_free_all();
+  control_event_bootstrap_reset();
 }
-#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control.h b/src/feature/control/control.h
index a09c1cd11b..7e72b2736b 100644
--- a/src/feature/control/control.h
+++ b/src/feature/control/control.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,76 +12,6 @@
 #ifndef TOR_CONTROL_H
 #define TOR_CONTROL_H
 
-/** Used to indicate the type of a circuit event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum circuit_status_event_t {
-  CIRC_EVENT_LAUNCHED = 0,
-  CIRC_EVENT_BUILT    = 1,
-  CIRC_EVENT_EXTENDED = 2,
-  CIRC_EVENT_FAILED   = 3,
-  CIRC_EVENT_CLOSED   = 4,
-} circuit_status_event_t;
-
-/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
- * The various types are defined in control-spec.txt . */
-typedef enum circuit_status_minor_event_t {
-  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-  CIRC_MINOR_EVENT_CANNIBALIZED,
-} circuit_status_minor_event_t;
-
-/** Used to indicate the type of a stream event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum stream_status_event_t {
-  STREAM_EVENT_SENT_CONNECT = 0,
-  STREAM_EVENT_SENT_RESOLVE = 1,
-  STREAM_EVENT_SUCCEEDED    = 2,
-  STREAM_EVENT_FAILED       = 3,
-  STREAM_EVENT_CLOSED       = 4,
-  STREAM_EVENT_NEW          = 5,
-  STREAM_EVENT_NEW_RESOLVE  = 6,
-  STREAM_EVENT_FAILED_RETRIABLE = 7,
-  STREAM_EVENT_REMAP        = 8
-} stream_status_event_t;
-
-/** Used to indicate the type of an OR connection event passed to the
- * controller.  The various types are defined in control-spec.txt */
-typedef enum or_conn_status_event_t {
-  OR_CONN_EVENT_LAUNCHED     = 0,
-  OR_CONN_EVENT_CONNECTED    = 1,
-  OR_CONN_EVENT_FAILED       = 2,
-  OR_CONN_EVENT_CLOSED       = 3,
-  OR_CONN_EVENT_NEW          = 4,
-} or_conn_status_event_t;
-
-/** Used to indicate the type of a buildtime event */
-typedef enum buildtimeout_set_event_t {
-  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
-  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
-  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
-  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
-  BUILDTIMEOUT_SET_EVENT_RESUME = 4
-} buildtimeout_set_event_t;
-
-/** Enum describing various stages of bootstrapping, for use with controller
- * bootstrap status events. The values range from 0 to 100. */
-typedef enum {
-  BOOTSTRAP_STATUS_UNDEF=-1,
-  BOOTSTRAP_STATUS_STARTING=0,
-  BOOTSTRAP_STATUS_CONN_DIR=5,
-  BOOTSTRAP_STATUS_HANDSHAKE=-2,
-  BOOTSTRAP_STATUS_HANDSHAKE_DIR=10,
-  BOOTSTRAP_STATUS_ONEHOP_CREATE=15,
-  BOOTSTRAP_STATUS_REQUESTING_STATUS=20,
-  BOOTSTRAP_STATUS_LOADING_STATUS=25,
-  BOOTSTRAP_STATUS_LOADING_KEYS=40,
-  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
-  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
-  BOOTSTRAP_STATUS_CONN_OR=80,
-  BOOTSTRAP_STATUS_HANDSHAKE_OR=85,
-  BOOTSTRAP_STATUS_CIRCUIT_CREATE=90,
-  BOOTSTRAP_STATUS_DONE=100
-} bootstrap_status_t;
-
 control_connection_t *TO_CONTROL_CONN(connection_t *);
 
 #define CONTROL_CONN_STATE_MIN_ 1
@@ -92,18 +22,6 @@ control_connection_t *TO_CONTROL_CONN(connection_t *);
 #define CONTROL_CONN_STATE_NEEDAUTH 2
 #define CONTROL_CONN_STATE_MAX_ 2
 
-/** Reason for remapping an AP connection's address: we have a cached
- * answer. */
-#define REMAP_STREAM_SOURCE_CACHE 1
-/** Reason for remapping an AP connection's address: the exit node told us an
- * answer. */
-#define REMAP_STREAM_SOURCE_EXIT 2
-
-void control_initialize_event_queue(void);
-
-void control_update_global_event_mask(void);
-void control_adjust_event_log_severity(void);
-
 void control_ports_write_to_file(void);
 
 /** Log information about the connection <b>conn</b>, protecting it as with
@@ -124,294 +42,28 @@ void connection_control_closed(control_connection_t *conn);
 
 int connection_control_process_inbuf(control_connection_t *conn);
 
-#define EVENT_NS 0x000F
-int control_event_is_interesting(int event);
-
-void control_per_second_events(void);
-int control_any_per_second_event_enabled(void);
-
-int control_event_circuit_status(origin_circuit_t *circ,
-                                 circuit_status_event_t e, int reason);
-int control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                          int old_purpose);
-int control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                       int old_purpose,
-                                       const struct timeval *old_tv_created);
-int control_event_stream_status(entry_connection_t *conn,
-                                stream_status_event_t e,
-                                int reason);
-int control_event_or_conn_status(or_connection_t *conn,
-                                 or_conn_status_event_t e, int reason);
-int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
-int control_event_stream_bandwidth(edge_connection_t *edge_conn);
-int control_event_stream_bandwidth_used(void);
-int control_event_circ_bandwidth_used(void);
-int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
-int control_event_conn_bandwidth(connection_t *conn);
-int control_event_conn_bandwidth_used(void);
-int control_event_circuit_cell_stats(void);
-void control_event_logmsg(int severity, uint32_t domain, const char *msg);
-void control_event_logmsg_pending(void);
-int control_event_descriptors_changed(smartlist_t *routers);
-int control_event_address_mapped(const char *from, const char *to,
-                                 time_t expires, const char *error,
-                                 const int cached);
-int control_event_my_descriptor_changed(void);
-int control_event_network_liveness_update(int liveness);
-int control_event_networkstatus_changed(smartlist_t *statuses);
-
-int control_event_newconsensus(const networkstatus_t *consensus);
-int control_event_networkstatus_changed_single(const routerstatus_t *rs);
-int control_event_general_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_client_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_server_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-
-int control_event_general_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_client_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_server_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-
-int control_event_guard(const char *nickname, const char *digest,
-                        const char *status);
-int control_event_conf_changed(const smartlist_t *elements);
-int control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                                   const char *args);
-int control_event_signal(uintptr_t signal);
-
-int init_control_cookie_authentication(int enabled);
-char *get_controller_cookie_file_name(void);
-struct config_line_t;
-smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
 void disable_control_logging(void);
 void enable_control_logging(void);
 
 void monitor_owning_controller_process(const char *process_spec);
 
-void control_event_bootstrap(bootstrap_status_t status, int progress);
-MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
-                                                 int reason,
-                                                 or_connection_t *or_conn));
-void control_event_boot_dir(bootstrap_status_t status, int progress);
-void control_event_boot_first_orconn(void);
-void control_event_bootstrap_problem(const char *warn, const char *reason,
-                                     const connection_t *conn, int dowarn);
-
-void control_event_clients_seen(const char *controller_str);
-void control_event_transport_launched(const char *mode,
-                                      const char *transport_name,
-                                      tor_addr_t *addr, uint16_t port);
 const char *rend_auth_type_to_string(rend_auth_type_t auth_type);
-MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
-void control_event_hs_descriptor_requested(const char *onion_address,
-                                           rend_auth_type_t auth_type,
-                                           const char *id_digest,
-                                           const char *desc_id,
-                                           const char *hsdir_index);
-void control_event_hs_descriptor_created(const char *onion_address,
-                                         const char *desc_id,
-                                         int replica);
-void control_event_hs_descriptor_upload(const char *onion_address,
-                                        const char *desc_id,
-                                        const char *hs_dir,
-                                        const char *hsdir_index);
-void control_event_hs_descriptor_upload_end(const char *action,
-                                            const char *onion_address,
-                                            const char *hs_dir,
-                                            const char *reason);
-void control_event_hs_descriptor_uploaded(const char *hs_dir,
-                                          const char *onion_address);
-/* Hidden service v2 HS_DESC specific. */
-void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                          const char *id_digest,
-                                          const char *reason);
-void control_event_hsv2_descriptor_received(const char *onion_address,
-                                            const rend_data_t *rend_data,
-                                            const char *id_digest);
-/* Hidden service v3 HS_DESC specific. */
-void control_event_hsv3_descriptor_failed(const char *onion_address,
-                                          const char *desc_id,
-                                          const char *hsdir_id_digest,
-                                          const char *reason);
-void control_event_hsv3_descriptor_received(const char *onion_address,
-                                            const char *desc_id,
-                                            const char *hsdir_id_digest);
-void control_event_hs_descriptor_upload_failed(const char *hs_dir,
-                                               const char *onion_address,
-                                               const char *reason);
-void control_event_hs_descriptor_content(const char *onion_address,
-                                         const char *desc_id,
-                                         const char *hsdir_fp,
-                                         const char *content);
 void control_free_all(void);
 
-#ifdef CONTROL_PRIVATE
-#include "lib/crypt_ops/crypto_ed25519.h"
-
-/* Recognized asynchronous event types.  It's okay to expand this list
- * because it is used both as a list of v0 event types, and as indices
- * into the bitfield to determine which controllers want which events.
- */
-/* This bitfield has no event zero    0x0000 */
-#define EVENT_MIN_                    0x0001
-#define EVENT_CIRCUIT_STATUS          0x0001
-#define EVENT_STREAM_STATUS           0x0002
-#define EVENT_OR_CONN_STATUS          0x0003
-#define EVENT_BANDWIDTH_USED          0x0004
-#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
-#define EVENT_NEW_DESC                0x0006
-#define EVENT_DEBUG_MSG               0x0007
-#define EVENT_INFO_MSG                0x0008
-#define EVENT_NOTICE_MSG              0x0009
-#define EVENT_WARN_MSG                0x000A
-#define EVENT_ERR_MSG                 0x000B
-#define EVENT_ADDRMAP                 0x000C
-/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
-   can reclaim 0x000D. */
-#define EVENT_DESCCHANGED             0x000E
-/* Exposed above */
-// #define EVENT_NS                   0x000F
-#define EVENT_STATUS_CLIENT           0x0010
-#define EVENT_STATUS_SERVER           0x0011
-#define EVENT_STATUS_GENERAL          0x0012
-#define EVENT_GUARD                   0x0013
-#define EVENT_STREAM_BANDWIDTH_USED   0x0014
-#define EVENT_CLIENTS_SEEN            0x0015
-#define EVENT_NEWCONSENSUS            0x0016
-#define EVENT_BUILDTIMEOUT_SET        0x0017
-#define EVENT_GOT_SIGNAL              0x0018
-#define EVENT_CONF_CHANGED            0x0019
-#define EVENT_CONN_BW                 0x001A
-#define EVENT_CELL_STATS              0x001B
-/* UNUSED :                           0x001C */
-#define EVENT_CIRC_BANDWIDTH_USED     0x001D
-#define EVENT_TRANSPORT_LAUNCHED      0x0020
-#define EVENT_HS_DESC                 0x0021
-#define EVENT_HS_DESC_CONTENT         0x0022
-#define EVENT_NETWORK_LIVENESS        0x0023
-#define EVENT_MAX_                    0x0023
+#ifdef CONTROL_MODULE_PRIVATE
+struct signal_name_t {
+  int sig;
+  const char *signal_name;
+};
+extern const struct signal_name_t signal_table[];
+int get_cached_network_liveness(void);
+void set_cached_network_liveness(int liveness);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
 
-/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
-#define EVENT_CAPACITY_               0x0040
-
-/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
- * different structure, as it can only handle a maximum left shift of 1<<63. */
-
-#if EVENT_MAX_ >= EVENT_CAPACITY_
-#error control_connection_t.event_mask has an event greater than its capacity
+#ifdef CONTROL_PRIVATE
+STATIC char *control_split_incoming_command(char *incoming_cmd,
+                                            size_t *data_len,
+                                            char **current_cmd_out);
 #endif
 
-#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
-
-#define EVENT_MASK_NONE_             ((uint64_t)0x0)
-
-#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
-#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
-                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
-                                          - EVENT_MIN_))
-
-#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
-                                      & EVENT_MASK_BELOW_MAX_)
-
-/* Used only by control.c and test.c */
-STATIC size_t write_escaped_data(const char *data, size_t len, char **out);
-STATIC size_t read_escaped_data(const char *data, size_t len, char **out);
-
-#ifdef TOR_UNIT_TESTS
-MOCK_DECL(STATIC void,
-          send_control_event_string,(uint16_t event, const char *msg));
-
-MOCK_DECL(STATIC void,
-          queue_control_event_string,(uint16_t event, char *msg));
-
-void control_testing_set_global_event_mask(uint64_t mask);
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Helper structure: temporarily stores cell statistics for a circuit. */
-typedef struct cell_stats_t {
-  /** Number of cells added in app-ward direction by command. */
-  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells added in exit-ward direction by command. */
-  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in app-ward direction by command. */
-  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in exit-ward direction by command. */
-  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in app-ward direction by command. */
-  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in exit-ward direction by command. */
-  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
-} cell_stats_t;
-void sum_up_cell_stats_by_command(circuit_t *circ,
-                                  cell_stats_t *cell_stats);
-void append_cell_stats_by_command(smartlist_t *event_parts,
-                                  const char *key,
-                                  const uint64_t *include_if_non_zero,
-                                  const uint64_t *number_to_include);
-void format_cell_stats(char **event_string, circuit_t *circ,
-                       cell_stats_t *cell_stats);
-STATIC char *get_bw_samples(void);
-
-/* ADD_ONION secret key to create an ephemeral service. The command supports
- * multiple versions so this union stores the key and passes it to the HS
- * subsystem depending on the requested version. */
-typedef union add_onion_secret_key_t {
-  /* Hidden service v2 secret key. */
-  crypto_pk_t *v2;
-  /* Hidden service v3 secret key. */
-  ed25519_secret_key_t *v3;
-} add_onion_secret_key_t;
-
-STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
-                                   const char **key_new_alg_out,
-                                   char **key_new_blob_out,
-                                   add_onion_secret_key_t *decoded_key,
-                                   int *hs_version, char **err_msg_out);
-
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
-
-STATIC int getinfo_helper_onions(
-    control_connection_t *control_conn,
-    const char *question,
-    char **answer,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_networkstatus(
-    const char *flavor,
-    download_status_t **dl_to_emit,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_cert(
-    const char *fp_sk_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_desc(
-    const char *desc_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_bridge(
-    const char *bridge_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC int getinfo_helper_downloads(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_dir(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_current_time(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-
-#endif /* defined(CONTROL_PRIVATE) */
-
 #endif /* !defined(TOR_CONTROL_H) */
diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c
new file mode 100644
index 0000000000..b60623ab5c
--- /dev/null
+++ b/src/feature/control/control_auth.c
@@ -0,0 +1,441 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.c
+ * \brief Authentication for Tor's control-socket interface.
+ **/
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_proto.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+
+#include "lib/crypt_ops/crypto_s2k.h"
+
+/** If we're using cookie-type authentication, how long should our cookies be?
+ */
+#define AUTHENTICATION_COOKIE_LEN 32
+
+/** If true, we've set authentication_cookie to a secret code and
+ * stored it to disk. */
+static int authentication_cookie_is_set = 0;
+/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
+ * and which we're using to authenticate controllers.  (If the controller can
+ * read it off disk, it has permission to connect.) */
+static uint8_t *authentication_cookie = NULL;
+
+#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
+  "Tor safe cookie authentication server-to-controller hash"
+#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
+  "Tor safe cookie authentication controller-to-server hash"
+#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
+
+/** Helper: Return a newly allocated string containing a path to the
+ * file where we store our authentication cookie. */
+char *
+get_controller_cookie_file_name(void)
+{
+  const or_options_t *options = get_options();
+  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
+    return tor_strdup(options->CookieAuthFile);
+  } else {
+    return get_datadir_fname("control_auth_cookie");
+  }
+}
+
+/* Initialize the cookie-based authentication system of the
+ * ControlPort. If <b>enabled</b> is 0, then disable the cookie
+ * authentication system.  */
+int
+init_control_cookie_authentication(int enabled)
+{
+  char *fname = NULL;
+  int retval;
+
+  if (!enabled) {
+    authentication_cookie_is_set = 0;
+    return 0;
+  }
+
+  fname = get_controller_cookie_file_name();
+  retval = init_cookie_authentication(fname, "", /* no header */
+                                      AUTHENTICATION_COOKIE_LEN,
+                                   get_options()->CookieAuthFileGroupReadable,
+                                      &authentication_cookie,
+                                      &authentication_cookie_is_set);
+  tor_free(fname);
+  return retval;
+}
+
+/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
+ * Return a smartlist of acceptable passwords (unterminated strings of
+ * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
+ * failure.
+ */
+smartlist_t *
+decode_hashed_passwords(config_line_t *passwords)
+{
+  char decoded[64];
+  config_line_t *cl;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(passwords);
+
+  for (cl = passwords; cl; cl = cl->next) {
+    const char *hashed = cl->value;
+
+    if (!strcmpstart(hashed, "16:")) {
+      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
+                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
+          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
+        goto err;
+      }
+    } else {
+        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
+            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
+          goto err;
+        }
+    }
+    smartlist_add(sl,
+                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
+  }
+
+  return sl;
+
+ err:
+  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
+  smartlist_free(sl);
+  return NULL;
+}
+
+const control_cmd_syntax_t authchallenge_syntax = {
+   .min_args = 1,
+   .max_args = 1,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHCHALLENGE command. */
+int
+handle_control_authchallenge(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  char *client_nonce;
+  size_t client_nonce_len;
+  char server_hash[DIGEST256_LEN];
+  char server_hash_encoded[HEX_DIGEST256_LEN+1];
+  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
+  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
+
+  if (strcasecmp(smartlist_get(args->args, 0), "SAFECOOKIE")) {
+    control_write_endreply(conn, 513,
+                           "AUTHCHALLENGE only supports SAFECOOKIE "
+                           "authentication");
+    goto fail;
+  }
+  if (!authentication_cookie_is_set) {
+    control_write_endreply(conn, 515, "Cookie authentication is disabled");
+    goto fail;
+  }
+  if (args->kwargs == NULL || args->kwargs->next != NULL) {
+    control_write_endreply(conn, 512,
+                           "Wrong number of arguments for AUTHCHALLENGE");
+    goto fail;
+  }
+  if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHCHALLENGE does not accept keyword "
+                           "arguments.");
+    goto fail;
+  }
+
+  bool contains_quote = strchr(args->raw_body, '\"');
+  if (contains_quote) {
+    /* The nonce was quoted */
+    client_nonce = tor_strdup(args->kwargs->value);
+    client_nonce_len = strlen(client_nonce);
+  } else {
+    /* The nonce was should be in hex. */
+    const char *hex_nonce = args->kwargs->value;
+    client_nonce_len = strlen(hex_nonce) / 2;
+    client_nonce = tor_malloc(client_nonce_len);
+    if (base16_decode(client_nonce, client_nonce_len, hex_nonce,
+                      strlen(hex_nonce)) != (int)client_nonce_len) {
+      control_write_endreply(conn, 513, "Invalid base16 client nonce");
+      tor_free(client_nonce);
+      goto fail;
+    }
+  }
+
+  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+  /* Now compute and send the server-to-controller response, and the
+   * server's nonce. */
+  tor_assert(authentication_cookie != NULL);
+
+  {
+    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
+                      client_nonce_len +
+                      SAFECOOKIE_SERVER_NONCE_LEN);
+    char *tmp = tor_malloc_zero(tmp_len);
+    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
+    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
+           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+    crypto_hmac_sha256(server_hash,
+                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
+                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    crypto_hmac_sha256(client_hash,
+                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
+                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    conn->safecookie_client_hash = client_hash;
+
+    tor_free(tmp);
+  }
+
+  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
+                server_hash, sizeof(server_hash));
+  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
+                server_nonce, sizeof(server_nonce));
+
+  control_printf_endreply(conn, 250,
+                          "AUTHCHALLENGE SERVERHASH=%s SERVERNONCE=%s",
+                          server_hash_encoded,
+                          server_nonce_encoded);
+
+  tor_free(client_nonce);
+  return 0;
+ fail:
+  connection_mark_for_close(TO_CONN(conn));
+  return -1;
+}
+
+const control_cmd_syntax_t authenticate_syntax = {
+   .max_args = 0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHENTICATE message.  Check whether the
+ * authentication is valid, and if so, update the connection's state to
+ * OPEN.  Reply with DONE or ERROR.
+ */
+int
+handle_control_authenticate(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  bool used_quoted_string = false;
+  const or_options_t *options = get_options();
+  const char *errstr = "Unknown error";
+  char *password;
+  size_t password_len;
+  int bad_cookie=0, bad_password=0;
+  smartlist_t *sl = NULL;
+
+  if (args->kwargs == NULL) {
+    password = tor_strdup("");
+    password_len = 0;
+  } else if (args->kwargs->next) {
+    control_write_endreply(conn, 512, "Too many arguments to AUTHENTICATE.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHENTICATE does not accept keyword arguments.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strchr(args->raw_body, '\"')) {
+    used_quoted_string = true;
+    password = tor_strdup(args->kwargs->value);
+    password_len = strlen(password);
+  } else {
+    const char *hex_passwd = args->kwargs->value;
+    password_len = strlen(hex_passwd) / 2;
+    password = tor_malloc(password_len+1);
+    if (base16_decode(password, password_len+1, hex_passwd, strlen(hex_passwd))
+                      != (int) password_len) {
+      control_write_endreply(conn, 551,
+            "Invalid hexadecimal encoding.  Maybe you tried a plain text "
+            "password?  If so, the standard requires that you put it in "
+            "double quotes.");
+      connection_mark_for_close(TO_CONN(conn));
+      tor_free(password);
+      return 0;
+    }
+  }
+
+  if (conn->safecookie_client_hash != NULL) {
+    /* The controller has chosen safe cookie authentication; the only
+     * acceptable authentication value is the controller-to-server
+     * response. */
+
+    tor_assert(authentication_cookie_is_set);
+
+    if (password_len != DIGEST256_LEN) {
+      log_warn(LD_CONTROL,
+               "Got safe cookie authentication response with wrong length "
+               "(%d)", (int)password_len);
+      errstr = "Wrong length for safe cookie response.";
+      goto err;
+    }
+
+    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
+      log_warn(LD_CONTROL,
+               "Got incorrect safe cookie authentication response");
+      errstr = "Safe cookie response did not match expected value.";
+      goto err;
+    }
+
+    tor_free(conn->safecookie_client_hash);
+    goto ok;
+  }
+
+  if (!options->CookieAuthentication && !options->HashedControlPassword &&
+      !options->HashedControlSessionPassword) {
+    /* if Tor doesn't demand any stronger authentication, then
+     * the controller can get in with anything. */
+    goto ok;
+  }
+
+  if (options->CookieAuthentication) {
+    int also_password = options->HashedControlPassword != NULL ||
+      options->HashedControlSessionPassword != NULL;
+    if (password_len != AUTHENTICATION_COOKIE_LEN) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
+                 "(%d)", (int)password_len);
+        errstr = "Wrong length on authentication cookie.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else if (tor_memneq(authentication_cookie, password, password_len)) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
+        errstr = "Authentication cookie did not match expected value.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else {
+      goto ok;
+    }
+  }
+
+  if (options->HashedControlPassword ||
+      options->HashedControlSessionPassword) {
+    int bad = 0;
+    smartlist_t *sl_tmp;
+    char received[DIGEST_LEN];
+    int also_cookie = options->CookieAuthentication;
+    sl = smartlist_new();
+    if (options->HashedControlPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (options->HashedControlSessionPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (bad) {
+      if (!also_cookie) {
+        log_warn(LD_BUG,
+                 "Couldn't decode HashedControlPassword: invalid base16");
+        errstr="Couldn't decode HashedControlPassword value in configuration.";
+        goto err;
+      }
+      bad_password = 1;
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+    } else {
+      SMARTLIST_FOREACH(sl, char *, expected,
+      {
+        secret_to_key_rfc2440(received,DIGEST_LEN,
+                              password,password_len,expected);
+        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
+                      received, DIGEST_LEN))
+          goto ok;
+      });
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+
+      if (used_quoted_string)
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration";
+      else
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration. Maybe you tried a plain text password? "
+          "If so, the standard requires that you put it in double quotes.";
+      bad_password = 1;
+      if (!also_cookie)
+        goto err;
+    }
+  }
+
+  /** We only get here if both kinds of authentication failed. */
+  tor_assert(bad_password && bad_cookie);
+  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
+  errstr = "Password did not match HashedControlPassword *or* authentication "
+    "cookie.";
+
+ err:
+  tor_free(password);
+  control_printf_endreply(conn, 515, "Authentication failed: %s", errstr);
+  connection_mark_for_close(TO_CONN(conn));
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+ ok:
+  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
+           ")", conn->base_.s);
+  send_control_done(conn);
+  conn->base_.state = CONTROL_CONN_STATE_OPEN;
+  tor_free(password);
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+}
+
+void
+control_auth_free_all(void)
+{
+  if (authentication_cookie) /* Free the auth cookie */
+    tor_free(authentication_cookie);
+  authentication_cookie_is_set = 0;
+}
diff --git a/src/feature/control/control_auth.h b/src/feature/control/control_auth.h
new file mode 100644
index 0000000000..d4c1dd78a7
--- /dev/null
+++ b/src/feature/control/control_auth.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.h
+ * \brief Header file for control_auth.c.
+ **/
+
+#ifndef TOR_CONTROL_AUTH_H
+#define TOR_CONTROL_AUTH_H
+
+struct control_cmd_args_t;
+struct control_cmd_syntax_t;
+
+int init_control_cookie_authentication(int enabled);
+char *get_controller_cookie_file_name(void);
+struct config_line_t;
+smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
+
+int handle_control_authchallenge(control_connection_t *conn,
+                                 const struct control_cmd_args_t *args);
+int handle_control_authenticate(control_connection_t *conn,
+                                const struct control_cmd_args_t *args);
+void control_auth_free_all(void);
+
+extern const struct control_cmd_syntax_t authchallenge_syntax;
+extern const struct control_cmd_syntax_t authenticate_syntax;
+
+#endif /* !defined(TOR_CONTROL_AUTH_H) */
diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c
new file mode 100644
index 0000000000..fee7612ba2
--- /dev/null
+++ b/src/feature/control/control_bootstrap.c
@@ -0,0 +1,389 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_bootstrap.c
+ * \brief Provide bootstrap progress events for the control port.
+ */
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "core/or/connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/reasons.h"
+#include "feature/control/control_events.h"
+#include "feature/hibernate/hibernate.h"
+#include "lib/malloc/malloc.h"
+
+/** A sufficiently large size to record the last bootstrap phase string. */
+#define BOOTSTRAP_MSG_LEN 1024
+
+/** What was the last bootstrap phase message we sent? We keep track
+ * of this so we can respond to getinfo status/bootstrap-phase queries. */
+static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
+
+/** Table to convert bootstrap statuses to strings.  */
+static const struct {
+  bootstrap_status_t status;
+  const char *tag;
+  const char *summary;
+} boot_to_str_tab[] = {
+  { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" },
+  { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" },
+
+  /* Initial connection to any relay */
+
+  { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt",
+    "Connected to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy",
+    "Connected to proxy" },
+  { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" },
+  { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE, "handshake",
+    "Handshaking with a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done",
+    "Handshake with a relay done" },
+
+  /* Loading directory info */
+
+  { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create",
+    "Establishing an encrypted directory connection" },
+  { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status",
+    "Asking for networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status",
+    "Loading networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys",
+    "Loading authority key certs" },
+  { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors",
+    "Asking for relay descriptors" },
+  { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors",
+    "Loading relay descriptors" },
+  { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo",
+    "Loaded enough directory info to build circuits" },
+
+  /* Connecting to a relay for AP circuits */
+
+  { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt",
+    "Connecting to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt",
+    "Connected to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy",
+    "Connecting to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy",
+    "Connected to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN, "ap_conn",
+    "Connecting to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done",
+    "Connected to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake",
+    "Finishing handshake with a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done",
+    "Handshake finished with a relay to build circuits" },
+
+  /* Creating AP circuits */
+
+  { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create",
+    "Establishing a Tor circuit" },
+  { BOOTSTRAP_STATUS_DONE, "done", "Done" },
+};
+#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0]))
+
+/** Convert the name of a bootstrapping phase <b>s</b> into strings
+ * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
+static int
+bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
+                           const char **summary)
+{
+  for (size_t i = 0; i < N_BOOT_TO_STR; i++) {
+    if (s == boot_to_str_tab[i].status) {
+      *tag = boot_to_str_tab[i].tag;
+      *summary = boot_to_str_tab[i].summary;
+      return 0;
+    }
+  }
+
+  *tag = *summary = "unknown";
+  return -1;
+}
+
+/** What percentage through the bootstrap process are we? We remember
+ * this so we can avoid sending redundant bootstrap status events, and
+ * so we can guess context for the bootstrap messages which are
+ * ambiguous. It starts at 'undef', but gets set to 'starting' while
+ * Tor initializes. */
+static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_percent, but only takes on the enumerated values in
+ * bootstrap_status_t.
+ */
+static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** As bootstrap_percent, but holds the bootstrapping level at which we last
+ * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
+ * to avoid flooding the log with a new message every time we get a few more
+ * microdescriptors */
+static int notice_bootstrap_percent = 0;
+
+/** How many problems have we had getting to the next bootstrapping phase?
+ * These include failure to establish a connection to a Tor relay,
+ * failures to finish the TLS handshake, failures to validate the
+ * consensus document, etc. */
+static int bootstrap_problems = 0;
+
+/** We only tell the controller once we've hit a threshold of problems
+ * for the current phase. */
+#define BOOTSTRAP_PROBLEM_THRESHOLD 10
+
+/** When our bootstrapping progress level changes, but our bootstrapping
+ * status has not advanced, we only log at NOTICE when we have made at least
+ * this much progress.
+ */
+#define BOOTSTRAP_PCT_INCREMENT 5
+
+/** Do the actual logging and notifications for
+ * control_event_bootstrap().  Doesn't change any state beyond that.
+ */
+static void
+control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
+                             int progress)
+{
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *tag, *summary;
+
+  bootstrap_status_to_string(status, &tag, &summary);
+  /* Locally reset status if there's incremental progress */
+  if (progress)
+    status = progress;
+
+  tor_log(loglevel, LD_CONTROL,
+          "Bootstrapped %d%% (%s): %s", status, tag, summary);
+  tor_snprintf(buf, sizeof(buf),
+               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
+               status, tag, summary);
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "NOTICE %s", buf);
+  control_event_client_status(LOG_NOTICE, "%s", buf);
+}
+
+int
+control_get_bootstrap_percent(void)
+{
+  return bootstrap_percent;
+}
+
+/** Called when Tor has made progress at bootstrapping its directory
+ * information and initial circuits.
+ *
+ * <b>status</b> is the new status, that is, what task we will be doing
+ * next. <b>progress</b> is zero if we just started this task, else it
+ * represents progress on the task.
+ */
+void
+control_event_bootstrap(bootstrap_status_t status, int progress)
+{
+  int loglevel = LOG_NOTICE;
+
+  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  if (status <= bootstrap_percent) {
+    /* If there's no new progress, return early. */
+    if (!progress || progress <= bootstrap_percent)
+      return;
+    /* Log at INFO if not enough progress happened. */
+    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
+      loglevel = LOG_INFO;
+  }
+
+  control_event_bootstrap_core(loglevel, status, progress);
+
+  if (status > bootstrap_percent) {
+    bootstrap_phase = status; /* new milestone reached */
+    bootstrap_percent = status;
+  }
+  if (progress > bootstrap_percent) {
+    /* incremental progress within a milestone */
+    bootstrap_percent = progress;
+    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
+  }
+  if (loglevel == LOG_NOTICE &&
+      bootstrap_percent > notice_bootstrap_percent) {
+    /* Remember that we gave a notice at this level. */
+    notice_bootstrap_percent = bootstrap_percent;
+  }
+}
+
+/** Flag whether we've opened an OR_CONN yet  */
+static int bootstrap_first_orconn = 0;
+
+/** Like bootstrap_phase, but for (possibly deferred) directory progress */
+static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
+static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+
+/** Defer directory info bootstrap events until we have successfully
+ * completed our first connection to a router.  */
+void
+control_event_boot_dir(bootstrap_status_t status, int progress)
+{
+  if (status > bootstrap_dir_progress) {
+    bootstrap_dir_progress = status;
+    bootstrap_dir_phase = status;
+  }
+  if (progress && progress >= bootstrap_dir_progress) {
+    bootstrap_dir_progress = progress;
+  }
+
+  /* Don't report unless we have successfully opened at least one OR_CONN */
+  if (!bootstrap_first_orconn)
+    return;
+
+  control_event_bootstrap(status, progress);
+}
+
+/** Set a flag to allow reporting of directory bootstrap progress.
+ * (Code that reports completion of an OR_CONN calls this.)  Also,
+ * report directory progress so far. */
+void
+control_event_boot_first_orconn(void)
+{
+  bootstrap_first_orconn = 1;
+  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a human-readable hint
+ * as to why, and <b>reason</b> provides a controller-facing short
+ * tag.  <b>conn</b> is the connection that caused this problem and
+ * can be NULL if a connection cannot be easily identified.
+ */
+void
+control_event_bootstrap_problem(const char *warn, const char *reason,
+                                const connection_t *conn, int dowarn)
+{
+  int status = bootstrap_percent;
+  const char *tag = "", *summary = "";
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *recommendation = "ignore";
+  int severity;
+  char *or_id = NULL, *hostaddr = NULL;
+  or_connection_t *or_conn = NULL;
+
+  /* bootstrap_percent must not be in "undefined" state here. */
+  tor_assert(status >= 0);
+
+  if (bootstrap_percent == 100)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  bootstrap_problems++;
+
+  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
+    dowarn = 1;
+
+  /* Don't warn about our bootstrapping status if we are hibernating or
+   * shutting down. */
+  if (we_are_hibernating())
+    dowarn = 0;
+
+  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
+
+  severity = dowarn ? LOG_WARN : LOG_INFO;
+
+  if (dowarn)
+    recommendation = "warn";
+
+  if (conn && conn->type == CONN_TYPE_OR) {
+    /* XXX TO_OR_CONN can't deal with const */
+    or_conn = TO_OR_CONN((connection_t *)conn);
+    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
+  } else {
+    or_id = tor_strdup("?");
+  }
+
+  if (conn)
+    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
+  else
+    hostaddr = tor_strdup("?");
+
+  log_fn(severity,
+         LD_CONTROL, "Problem bootstrapping. Stuck at %d%% (%s): %s. (%s; %s; "
+         "count %d; recommendation %s; host %s at %s)",
+         status, tag, summary, warn, reason,
+         bootstrap_problems, recommendation,
+         or_id, hostaddr);
+
+  connection_or_report_broken_states(severity, LD_HANDSHAKE);
+
+  tor_snprintf(buf, sizeof(buf),
+      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
+      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
+      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
+      recommendation,
+      or_id, hostaddr);
+
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "WARN %s", buf);
+  control_event_client_status(LOG_WARN, "%s", buf);
+
+  tor_free(hostaddr);
+  tor_free(or_id);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a hint as to why, and
+ * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
+ * is the connection that caused this problem.
+ */
+MOCK_IMPL(void,
+control_event_bootstrap_prob_or, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
+{
+  int dowarn = 0;
+
+  if (or_conn->have_noted_bootstrap_problem)
+    return;
+
+  or_conn->have_noted_bootstrap_problem = 1;
+
+  if (reason == END_OR_CONN_REASON_NO_ROUTE)
+    dowarn = 1;
+
+  /* If we are using bridges and all our OR connections are now
+     closed, it means that we totally failed to connect to our
+     bridges. Throw a warning. */
+  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
+    dowarn = 1;
+
+  control_event_bootstrap_problem(warn,
+                                  orconn_end_reason_to_control_string(reason),
+                                  TO_CONN(or_conn), dowarn);
+}
+
+/** Return a copy of the last sent bootstrap message. */
+char *
+control_event_boot_last_msg(void)
+{
+  return tor_strdup(last_sent_bootstrap_message);
+}
+
+/** Reset bootstrap tracking state. */
+void
+control_event_bootstrap_reset(void)
+{
+  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+  notice_bootstrap_percent = 0;
+  bootstrap_problems = 0;
+  bootstrap_first_orconn = 0;
+  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
+}
diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c
new file mode 100644
index 0000000000..eb14f101e7
--- /dev/null
+++ b/src/feature/control/control_cmd.c
@@ -0,0 +1,2422 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.c
+ * \brief Implement various commands for Tor's control-socket interface.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/confmgt/confmgt.h"
+#include "app/main/main.h"
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/dnsserv.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_hs.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/hs/hs_control.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "feature/rend/rendservice.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/rend/rend_authorized_client_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+
+static int control_setconf_helper(control_connection_t *conn,
+                                  const control_cmd_args_t *args,
+                                  int use_defaults);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/**
+ * Release all storage held in <b>args</b>
+ **/
+void
+control_cmd_args_free_(control_cmd_args_t *args)
+{
+  if (! args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, tor_free(c));
+    smartlist_free(args->args);
+  }
+  config_free_lines(args->kwargs);
+  tor_free(args->cmddata);
+
+  tor_free(args);
+}
+
+/** Erase all memory held in <b>args</b>. */
+void
+control_cmd_args_wipe(control_cmd_args_t *args)
+{
+  if (!args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, memwipe(c, 0, strlen(c)));
+  }
+  for (config_line_t *line = args->kwargs; line; line = line->next) {
+    memwipe(line->key, 0, strlen(line->key));
+    memwipe(line->value, 0, strlen(line->value));
+  }
+  if (args->cmddata)
+    memwipe(args->cmddata, 0, args->cmddata_len);
+}
+
+/**
+ * Return true iff any element of the NULL-terminated <b>array</b> matches
+ * <b>kwd</b>. Case-insensitive.
+ **/
+static bool
+string_array_contains_keyword(const char **array, const char *kwd)
+{
+  for (unsigned i = 0; array[i]; ++i) {
+    if (! strcasecmp(array[i], kwd))
+      return true;
+  }
+  return false;
+}
+
+/** Helper for argument parsing: check whether the keyword arguments just
+ * parsed in <b>result</b> were well-formed according to <b>syntax</b>.
+ *
+ * On success, return 0.  On failure, return -1 and set *<b>error_out</b>
+ * to a newly allocated error string.
+ **/
+static int
+kvline_check_keyword_args(const control_cmd_args_t *result,
+                          const control_cmd_syntax_t *syntax,
+                          char **error_out)
+{
+  if (result->kwargs == NULL) {
+    tor_asprintf(error_out, "Cannot parse keyword argument(s)");
+    return -1;
+  }
+
+  if (! syntax->allowed_keywords) {
+    /* All keywords are permitted. */
+    return 0;
+  }
+
+  /* Check for unpermitted arguments */
+  const config_line_t *line;
+  for (line = result->kwargs; line; line = line->next) {
+    if (! string_array_contains_keyword(syntax->allowed_keywords,
+                                        line->key)) {
+      tor_asprintf(error_out, "Unrecognized keyword argument %s",
+                   escaped(line->key));
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Helper: parse the arguments to a command according to <b>syntax</b>.  On
+ * success, set *<b>error_out</b> to NULL and return a newly allocated
+ * control_cmd_args_t.  On failure, set *<b>error_out</b> to newly allocated
+ * error string, and return NULL.
+ **/
+STATIC control_cmd_args_t *
+control_cmd_parse_args(const char *command,
+                       const control_cmd_syntax_t *syntax,
+                       size_t body_len,
+                       const char *body,
+                       char **error_out)
+{
+  *error_out = NULL;
+  control_cmd_args_t *result = tor_malloc_zero(sizeof(control_cmd_args_t));
+  const char *cmdline;
+  char *cmdline_alloc = NULL;
+  tor_assert(syntax->max_args < INT_MAX || syntax->max_args == UINT_MAX);
+
+  result->command = command;
+
+  if (syntax->store_raw_body) {
+    tor_assert(body[body_len] == 0);
+    result->raw_body = body;
+  }
+
+  const char *eol = memchr(body, '\n', body_len);
+  if (syntax->want_cmddata) {
+    if (! eol || (eol+1) == body+body_len) {
+      *error_out = tor_strdup("Empty body");
+      goto err;
+    }
+    cmdline_alloc = tor_memdup_nulterm(body, eol-body);
+    cmdline = cmdline_alloc;
+    ++eol;
+    result->cmddata_len = read_escaped_data(eol, (body+body_len)-eol,
+                                           &result->cmddata);
+  } else {
+    if (eol && (eol+1) != body+body_len) {
+      *error_out = tor_strdup("Unexpected body");
+      goto err;
+    }
+    cmdline = body;
+  }
+
+  result->args = smartlist_new();
+  smartlist_split_string(result->args, cmdline, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK,
+                         (int)(syntax->max_args+1));
+  size_t n_args = smartlist_len(result->args);
+  if (n_args < syntax->min_args) {
+    tor_asprintf(error_out, "Need at least %u argument(s)",
+                 syntax->min_args);
+    goto err;
+  } else if (n_args > syntax->max_args && ! syntax->accept_keywords) {
+    tor_asprintf(error_out, "Cannot accept more than %u argument(s)",
+                 syntax->max_args);
+    goto err;
+  }
+
+  if (n_args > syntax->max_args) {
+    /* We have extra arguments after the positional arguments, and we didn't
+       treat them as an error, so they must count as keyword arguments: Either
+       K=V pairs, or flags, or both. */
+    tor_assert(n_args == syntax->max_args + 1);
+    tor_assert(syntax->accept_keywords);
+    char *remainder = smartlist_pop_last(result->args);
+    result->kwargs = kvline_parse(remainder, syntax->kvline_flags);
+    tor_free(remainder);
+    if (kvline_check_keyword_args(result, syntax, error_out) < 0) {
+      goto err;
+    }
+  }
+
+  tor_assert_nonfatal(*error_out == NULL);
+  goto done;
+ err:
+  tor_assert_nonfatal(*error_out != NULL);
+  control_cmd_args_free(result);
+ done:
+  tor_free(cmdline_alloc);
+  return result;
+}
+
+/**
+ * Return true iff <b>lines</b> contains <b>flags</b> as a no-value
+ * (keyword-only) entry.
+ **/
+static bool
+config_lines_contain_flag(const config_line_t *lines, const char *flag)
+{
+  const config_line_t *line = config_line_find_case(lines, flag);
+  return line && !strcmp(line->value, "");
+}
+
+static const control_cmd_syntax_t setconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a SETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body.*/
+static int
+handle_control_setconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 0);
+}
+
+static const control_cmd_syntax_t resetconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a RESETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body. */
+static int
+handle_control_resetconf(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 1);
+}
+
+static const control_cmd_syntax_t getconf_syntax = {
+  .max_args=UINT_MAX
+};
+
+/** Called when we receive a GETCONF message.  Parse the request, and
+ * reply with a CONFVALUE or an ERROR message */
+static int
+handle_control_getconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  const or_options_t *options = get_options();
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    if (!option_is_recognized(q)) {
+      control_reply_add_printf(unrecognized, 552,
+                               "Unrecognized configuration key \"%s\"", q);
+    } else {
+      config_line_t *answer = option_get_assignment(options,q);
+      if (!answer) {
+        const char *name = option_get_canonical_name(q);
+        control_reply_add_one_kv(answers, 250, KV_OMIT_VALS, name, "");
+      }
+
+      while (answer) {
+        config_line_t *next;
+        control_reply_add_one_kv(answers, 250, KV_RAW, answer->key,
+                                 answer->value);
+        next = answer->next;
+        tor_free(answer->key);
+        tor_free(answer->value);
+        tor_free(answer);
+        answer = next;
+      }
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if (smartlist_len(unrecognized)) {
+    control_write_reply_lines(conn, unrecognized);
+  } else if (smartlist_len(answers)) {
+    control_write_reply_lines(conn, answers);
+  } else {
+    send_control_done(conn);
+  }
+
+  control_reply_free(answers);
+  control_reply_free(unrecognized);
+  return 0;
+}
+
+static const control_cmd_syntax_t loadconf_syntax = {
+  .want_cmddata = true
+};
+
+/** Called when we get a +LOADCONF message. */
+static int
+handle_control_loadconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  setopt_err_t retval;
+  char *errstring = NULL;
+
+  retval = options_init_from_string(NULL, args->cmddata,
+                                    CMD_RUN_TOR, NULL, &errstring);
+
+  if (retval != SETOPT_OK)
+    log_warn(LD_CONTROL,
+             "Controller gave us config file that didn't validate: %s",
+             errstring);
+
+#define SEND_ERRMSG(code, msg)                          \
+  control_printf_endreply(conn, code, msg "%s%s",       \
+                          errstring ? ": " : "",        \
+                          errstring ? errstring : "")
+  switch (retval) {
+  case SETOPT_ERR_PARSE:
+    SEND_ERRMSG(552, "Invalid config file");
+    break;
+  case SETOPT_ERR_TRANSITION:
+    SEND_ERRMSG(553, "Transition not allowed");
+    break;
+  case SETOPT_ERR_SETTING:
+    SEND_ERRMSG(553, "Unable to set option");
+    break;
+  case SETOPT_ERR_MISC:
+  default:
+    SEND_ERRMSG(550, "Unable to load config");
+    break;
+  case SETOPT_OK:
+    send_control_done(conn);
+    break;
+  }
+#undef SEND_ERRMSG
+  tor_free(errstring);
+  return 0;
+}
+
+static const control_cmd_syntax_t setevents_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a SETEVENTS message: update conn->event_mask,
+ * and reply with DONE or ERROR. */
+static int
+handle_control_setevents(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  int event_code;
+  event_mask_t event_mask = 0;
+  const smartlist_t *events = args->args;
+
+  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
+    {
+      if (!strcasecmp(ev, "EXTENDED") ||
+          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
+        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
+                 "supported.", ev);
+        continue;
+      } else {
+        int i;
+        event_code = -1;
+
+        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+          if (!strcasecmp(ev, control_event_table[i].event_name)) {
+            event_code = control_event_table[i].event_code;
+            break;
+          }
+        }
+
+        if (event_code == -1) {
+          control_printf_endreply(conn, 552, "Unrecognized event \"%s\"", ev);
+          return 0;
+        }
+      }
+      event_mask |= (((event_mask_t)1) << event_code);
+    }
+  SMARTLIST_FOREACH_END(ev);
+
+  conn->event_mask = event_mask;
+
+  control_update_global_event_mask();
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t saveconf_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a SAVECONF command. Try to flush the current options to
+ * disk, and report success or failure. */
+static int
+handle_control_saveconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  bool force = config_lines_contain_flag(args->kwargs, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
+    control_write_endreply(conn, 551,
+                           "Unable to write configuration to disk.");
+  } else {
+    send_control_done(conn);
+  }
+  return 0;
+}
+
+static const control_cmd_syntax_t signal_syntax = {
+  .min_args = 1,
+  .max_args = 1,
+};
+
+/** Called when we get a SIGNAL command. React to the provided signal, and
+ * report success or failure. (If the signal results in a shutdown, success
+ * may not be reported.) */
+static int
+handle_control_signal(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  int sig = -1;
+  int i;
+
+  tor_assert(smartlist_len(args->args) == 1);
+  const char *s = smartlist_get(args->args, 0);
+
+  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if (!strcasecmp(s, signal_table[i].signal_name)) {
+      sig = signal_table[i].sig;
+      break;
+    }
+  }
+
+  if (sig < 0)
+    control_printf_endreply(conn, 552, "Unrecognized signal code \"%s\"", s);
+  if (sig < 0)
+    return 0;
+
+  send_control_done(conn);
+  /* Flush the "done" first if the signal might make us shut down. */
+  if (sig == SIGTERM || sig == SIGINT)
+    connection_flush(TO_CONN(conn));
+
+  activate_signal(sig);
+
+  return 0;
+}
+
+static const control_cmd_syntax_t takeownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
+ * as an owning connection, so that we will exit if the connection
+ * closes. */
+static int
+handle_control_takeownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 1;
+
+  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t dropownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a DROPOWNERSHIP command.  Mark this connection
+ * as a non-owning connection, so that we will not exit if the connection
+ * closes. */
+static int
+handle_control_dropownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 0;
+
+  log_info(LD_CONTROL, "Control connection %d has dropped ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+/** Given a text circuit <b>id</b>, return the corresponding circuit. */
+static origin_circuit_t *
+get_circ(const char *id)
+{
+  uint32_t n_id;
+  int ok;
+  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  return circuit_get_by_global_id(n_id);
+}
+
+/** Given a text stream <b>id</b>, return the corresponding AP connection. */
+static entry_connection_t *
+get_stream(const char *id)
+{
+  uint64_t n_id;
+  int ok;
+  connection_t *conn;
+  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  conn = connection_get_by_global_id(n_id);
+  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
+    return NULL;
+  return TO_ENTRY_CONN(conn);
+}
+
+/** Helper for setconf and resetconf. Acts like setconf, except
+ * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
+ * contents of body.
+ */
+static int
+control_setconf_helper(control_connection_t *conn,
+                       const control_cmd_args_t *args,
+                       int use_defaults)
+{
+  setopt_err_t opt_err;
+  char *errstring = NULL;
+  const unsigned flags =
+    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
+
+  // We need a copy here, since confmgt.c wants to canonicalize cases.
+  config_line_t *lines = config_lines_dup(args->kwargs);
+
+  opt_err = options_trial_assign(lines, flags, &errstring);
+  {
+#define SEND_ERRMSG(code, msg)                                  \
+    control_printf_endreply(conn, code, msg ": %s", errstring);
+
+    switch (opt_err) {
+      case SETOPT_ERR_MISC:
+        SEND_ERRMSG(552, "Unrecognized option");
+        break;
+      case SETOPT_ERR_PARSE:
+        SEND_ERRMSG(513, "Unacceptable option value");
+        break;
+      case SETOPT_ERR_TRANSITION:
+        SEND_ERRMSG(553, "Transition not allowed");
+        break;
+      case SETOPT_ERR_SETTING:
+      default:
+        SEND_ERRMSG(553, "Unable to set option");
+        break;
+      case SETOPT_OK:
+        config_free_lines(lines);
+        send_control_done(conn);
+        return 0;
+    }
+#undef SEND_ERRMSG
+    log_warn(LD_CONTROL,
+             "Controller gave us config lines that didn't validate: %s",
+             errstring);
+    config_free_lines(lines);
+    tor_free(errstring);
+    return 0;
+  }
+}
+
+/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
+ * containing funny characters. */
+static int
+address_is_invalid_mapaddress_target(const char *addr)
+{
+  if (!strcmpstart(addr, "*."))
+    return address_is_invalid_destination(addr+2, 1);
+  else
+    return address_is_invalid_destination(addr, 1);
+}
+
+static const control_cmd_syntax_t mapaddress_syntax = {
+  // no positional arguments are expected
+  .max_args=0,
+  // an arbitrary number of K=V entries are supported.
+  .accept_keywords=true,
+};
+
+/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
+ * and report success or failure. */
+static int
+handle_control_mapaddress(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  smartlist_t *reply;
+  char *r;
+  size_t sz;
+
+  reply = smartlist_new();
+  const config_line_t *line;
+  for (line = args->kwargs; line; line = line->next) {
+    const char *from = line->key;
+    const char *to = line->value;
+    {
+      if (address_is_invalid_mapaddress_target(to)) {
+        smartlist_add_asprintf(reply,
+                     "512-syntax error: invalid address '%s'", to);
+        log_warn(LD_CONTROL,
+                 "Skipping invalid argument '%s' in MapAddress msg", to);
+      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
+                 !strcmp(from, "::")) {
+        const char type =
+          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
+          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
+        const char *address = addressmap_register_virtual_address(
+                                                     type, tor_strdup(to));
+        if (!address) {
+          smartlist_add_asprintf(reply,
+                   "451-resource exhausted: skipping '%s=%s'", from,to);
+          log_warn(LD_CONTROL,
+                   "Unable to allocate address for '%s' in MapAddress msg",
+                   safe_str_client(to));
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
+        }
+      } else {
+        const char *msg;
+        if (addressmap_register_auto(from, to, 1,
+                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
+          smartlist_add_asprintf(reply,
+                                 "512-syntax error: invalid address mapping "
+                                 " '%s=%s': %s", from, to, msg);
+          log_warn(LD_CONTROL,
+                   "Skipping invalid argument '%s=%s' in MapAddress msg: %s",
+                   from, to, msg);
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", from, to);
+        }
+      }
+    }
+  }
+
+  if (smartlist_len(reply)) {
+    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
+    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
+    connection_buf_add(r, sz, TO_CONN(conn));
+    tor_free(r);
+  } else {
+    control_write_endreply(conn, 512, "syntax error: "
+                           "not enough arguments to mapaddress.");
+  }
+
+  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
+  smartlist_free(reply);
+  return 0;
+}
+
+/** Given a string, convert it to a circuit purpose. */
+static uint8_t
+circuit_purpose_from_string(const char *string)
+{
+  if (!strcasecmpstart(string, "purpose="))
+    string += strlen("purpose=");
+
+  if (!strcasecmp(string, "general"))
+    return CIRCUIT_PURPOSE_C_GENERAL;
+  else if (!strcasecmp(string, "controller"))
+    return CIRCUIT_PURPOSE_CONTROLLER;
+  else
+    return CIRCUIT_PURPOSE_UNKNOWN;
+}
+
+static const control_cmd_syntax_t extendcircuit_syntax = {
+  .min_args=1,
+  .max_args=1, // see note in function
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS
+};
+
+/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
+ * circuit, and report success or failure. */
+static int
+handle_control_extendcircuit(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  smartlist_t *router_nicknames=smartlist_new(), *nodes=NULL;
+  origin_circuit_t *circ = NULL;
+  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  const config_line_t *kwargs = args->kwargs;
+  const char *circ_id = smartlist_get(args->args, 0);
+  const char *path_str = NULL;
+  char *path_str_alloc = NULL;
+
+  /* The syntax for this command is unfortunate. The second argument is
+     optional, and is a comma-separated list long-format fingerprints, which
+     can (historically!) contain an equals sign.
+
+     Here we check the second argument to see if it's a path, and if so we
+     remove it from the kwargs list and put it in path_str.
+  */
+  if (kwargs) {
+    const config_line_t *arg1 = kwargs;
+    if (!strcmp(arg1->value, "")) {
+      path_str = arg1->key;
+      kwargs = kwargs->next;
+    } else if (arg1->key[0] == '$') {
+      tor_asprintf(&path_str_alloc, "%s=%s", arg1->key, arg1->value);
+      path_str = path_str_alloc;
+      kwargs = kwargs->next;
+    }
+  }
+
+  const config_line_t *purpose_line = config_line_find_case(kwargs, "PURPOSE");
+  bool zero_circ = !strcmp("0", circ_id);
+
+  if (purpose_line) {
+    intended_purpose = circuit_purpose_from_string(purpose_line->value);
+    if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purpose_line->value);
+      goto done;
+    }
+  }
+
+  if (zero_circ) {
+    if (!path_str) {
+      // "EXTENDCIRCUIT 0" with no path.
+      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
+      if (!circ) {
+        control_write_endreply(conn, 551, "Couldn't start circuit");
+      } else {
+        control_printf_endreply(conn, 250, "EXTENDED %lu",
+                                (unsigned long)circ->global_identifier);
+      }
+      goto done;
+    }
+  }
+
+  if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  if (!path_str) {
+    control_write_endreply(conn, 512, "syntax error: path required.");
+    goto done;
+  }
+
+  smartlist_split_string(router_nicknames, path_str, ",", 0, 0);
+
+  nodes = smartlist_new();
+  bool first_node = zero_circ;
+  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
+    const node_t *node = node_get_by_nickname(n, 0);
+    if (!node) {
+      control_printf_endreply(conn, 552, "No such router \"%s\"", n);
+      goto done;
+    }
+    if (!node_has_preferred_descriptor(node, first_node)) {
+      control_printf_endreply(conn, 552, "No descriptor for \"%s\"", n);
+      goto done;
+    }
+    smartlist_add(nodes, (void*)node);
+    first_node = false;
+  } SMARTLIST_FOREACH_END(n);
+
+  if (!smartlist_len(nodes)) {
+    control_write_endreply(conn, 512, "No router names provided");
+    goto done;
+  }
+
+  if (zero_circ) {
+    /* start a new circuit */
+    circ = origin_circuit_init(intended_purpose, 0);
+  }
+
+  /* now circ refers to something that is ready to be extended */
+  first_node = zero_circ;
+  SMARTLIST_FOREACH(nodes, const node_t *, node,
+  {
+    extend_info_t *info = extend_info_from_node(node, first_node);
+    if (!info) {
+      tor_assert_nonfatal(first_node);
+      log_warn(LD_CONTROL,
+               "controller tried to connect to a node that lacks a suitable "
+               "descriptor, or which doesn't have any "
+               "addresses that are allowed by the firewall configuration; "
+               "circuit marked for closing.");
+      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+    circuit_append_new_exit(circ, info);
+    if (circ->build_state->desired_path_len > 1) {
+      circ->build_state->onehop_tunnel = 0;
+    }
+    extend_info_free(info);
+    first_node = 0;
+  });
+
+  /* now that we've populated the cpath, start extending */
+  if (zero_circ) {
+    int err_reason = 0;
+    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
+      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+  } else {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
+      int err_reason = 0;
+      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
+      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
+        log_info(LD_CONTROL,
+                 "send_next_onion_skin failed; circuit marked for closing.");
+        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+        control_write_endreply(conn, 551, "Couldn't send onion skin");
+        goto done;
+      }
+    }
+  }
+
+  control_printf_endreply(conn, 250, "EXTENDED %lu",
+                          (unsigned long)circ->global_identifier);
+  if (zero_circ) /* send a 'launched' event, for completeness */
+    circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
+ done:
+  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
+  smartlist_free(router_nicknames);
+  smartlist_free(nodes);
+  tor_free(path_str_alloc);
+  return 0;
+}
+
+static const control_cmd_syntax_t setcircuitpurpose_syntax = {
+  .max_args=1,
+  .accept_keywords=true,
+};
+
+/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
+ * circuit and it's a valid purpose, change it. */
+static int
+handle_control_setcircuitpurpose(control_connection_t *conn,
+                                 const control_cmd_args_t *args)
+{
+  origin_circuit_t *circ = NULL;
+  uint8_t new_purpose;
+  const char *circ_id = smartlist_get(args->args,0);
+
+  if (!(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  {
+    const config_line_t *purp = config_line_find_case(args->kwargs, "PURPOSE");
+    if (!purp) {
+      control_write_endreply(conn, 552, "No purpose given");
+      goto done;
+    }
+    new_purpose = circuit_purpose_from_string(purp->value);
+    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purp->value);
+      goto done;
+    }
+  }
+
+  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
+  send_control_done(conn);
+
+ done:
+  return 0;
+}
+
+static const char *attachstream_keywords[] = {
+  "HOP", NULL
+};
+static const control_cmd_syntax_t attachstream_syntax = {
+  .min_args=2, .max_args=2,
+  .accept_keywords=true,
+  .allowed_keywords=attachstream_keywords
+};
+
+/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
+ * stream, and report success or failure. */
+static int
+handle_control_attachstream(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  entry_connection_t *ap_conn = NULL;
+  origin_circuit_t *circ = NULL;
+  crypt_path_t *cpath=NULL;
+  int hop=0, hop_line_ok=1;
+  const char *stream_id = smartlist_get(args->args, 0);
+  const char *circ_id = smartlist_get(args->args, 1);
+  int zero_circ = !strcmp(circ_id, "0");
+  const config_line_t *hoparg = config_line_find_case(args->kwargs, "HOP");
+
+  if (!(ap_conn = get_stream(stream_id))) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"", stream_id);
+    return 0;
+  } else if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  } else if (circ) {
+    if (hoparg) {
+      hop = (int) tor_parse_ulong(hoparg->value, 10, 0, INT_MAX,
+                                  &hop_line_ok, NULL);
+      if (!hop_line_ok) { /* broken hop line */
+        control_printf_endreply(conn, 552, "Bad value hop=%s",
+                                hoparg->value);
+        return 0;
+      }
+    }
+  }
+
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
+    control_write_endreply(conn, 555,
+                           "Connection is not managed by controller.");
+    return 0;
+  }
+
+  /* Do we need to detach it first? */
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
+    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
+    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
+    /* Un-mark it as ending, since we're going to reuse it. */
+    edge_conn->edge_has_sent_end = 0;
+    edge_conn->end_reason = 0;
+    if (tmpcirc)
+      circuit_detach_stream(tmpcirc, edge_conn);
+    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
+    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
+  }
+
+  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to non-open origin circuit");
+    return 0;
+  }
+  /* Is this a single hop circuit? */
+  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to this one-hop circuit.");
+    return 0;
+  }
+
+  if (circ && hop>0) {
+    /* find this hop in the circuit, and set cpath */
+    cpath = circuit_get_cpath_hop(circ, hop);
+    if (!cpath) {
+      control_printf_endreply(conn, 551, "Circuit doesn't have %d hops.", hop);
+      return 0;
+    }
+  }
+  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
+    control_write_endreply(conn, 551, "Unable to attach stream");
+    return 0;
+  }
+  send_control_done(conn);
+  return 0;
+}
+
+static const char *postdescriptor_keywords[] = {
+  "cache", "purpose", NULL,
+};
+
+static const control_cmd_syntax_t postdescriptor_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .allowed_keywords = postdescriptor_keywords,
+  .want_cmddata = true,
+};
+
+/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
+ * descriptor, and report success or failure. */
+static int
+handle_control_postdescriptor(control_connection_t *conn,
+                              const control_cmd_args_t *args)
+{
+  const char *msg=NULL;
+  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
+  int cache = 0; /* eventually, we may switch this to 1 */
+  const config_line_t *line;
+
+  line = config_line_find_case(args->kwargs, "purpose");
+  if (line) {
+    purpose = router_purpose_from_string(line->value);
+    if (purpose == ROUTER_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+  line = config_line_find_case(args->kwargs, "cache");
+  if (line) {
+    if (!strcasecmp(line->value, "no"))
+      cache = 0;
+    else if (!strcasecmp(line->value, "yes"))
+      cache = 1;
+    else {
+      control_printf_endreply(conn, 552, "Unknown cache request \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+
+  switch (router_load_single_router(args->cmddata, purpose, cache, &msg)) {
+  case -1:
+    if (!msg) msg = "Could not parse descriptor";
+    control_write_endreply(conn, 554, msg);
+    break;
+  case 0:
+    if (!msg) msg = "Descriptor not added";
+    control_write_endreply(conn, 251, msg);
+    break;
+  case 1:
+    send_control_done(conn);
+    break;
+  }
+
+ done:
+  return 0;
+}
+
+static const control_cmd_syntax_t redirectstream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, // XXX should be 3.
+};
+
+/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
+ * address of the named AP stream, and report success or failure. */
+static int
+handle_control_redirectstream(control_connection_t *conn,
+                              const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn = NULL;
+  char *new_addr = NULL;
+  uint16_t new_port = 0;
+  const smartlist_t *args = cmd_args->args;
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
+           || !ap_conn->socks_request) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  } else {
+    int ok = 1;
+    if (smartlist_len(args) > 2) { /* they included a port too */
+      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
+                                            10, 1, 65535, &ok, NULL);
+    }
+    if (!ok) {
+      control_printf_endreply(conn, 512, "Cannot parse port \"%s\"",
+                              (char*)smartlist_get(args, 2));
+    } else {
+      new_addr = tor_strdup(smartlist_get(args, 1));
+    }
+  }
+
+  if (!new_addr)
+    return 0;
+
+  strlcpy(ap_conn->socks_request->address, new_addr,
+          sizeof(ap_conn->socks_request->address));
+  if (new_port)
+    ap_conn->socks_request->port = new_port;
+  tor_free(new_addr);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closestream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, /* XXXX This is the original behavior, but
+                         * maybe we should change the spec. */
+};
+
+/** Called when we get a CLOSESTREAM command; try to close the named stream
+ * and report success or failure. */
+static int
+handle_control_closestream(control_connection_t *conn,
+                           const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn=NULL;
+  uint8_t reason=0;
+  int ok;
+  const smartlist_t *args = cmd_args->args;
+
+  tor_assert(smartlist_len(args) >= 2);
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0))))
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  else {
+    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
+                                       &ok, NULL);
+    if (!ok) {
+      control_printf_endreply(conn, 552, "Unrecognized reason \"%s\"",
+                              (char*)smartlist_get(args, 1));
+      ap_conn = NULL;
+    }
+  }
+  if (!ap_conn)
+    return 0;
+
+  connection_mark_unattached_ap(ap_conn, reason);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closecircuit_syntax = {
+  .min_args=1, .max_args=1,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+  // XXXX we might want to exclude unrecognized flags, but for now we
+  // XXXX just ignore them for backward compatibility.
+};
+
+/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
+ * and report success or failure. */
+static int
+handle_control_closecircuit(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  const char *circ_id = smartlist_get(args->args, 0);
+  origin_circuit_t *circ = NULL;
+
+  if (!(circ=get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  }
+
+  bool safe =  config_lines_contain_flag(args->kwargs, "IfUnused");
+
+  if (!safe || !circ->p_streams) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
+  }
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t resolve_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a RESOLVE command: start trying to resolve
+ * the listed addresses. */
+static int
+handle_control_resolve(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  smartlist_t *failed;
+  int is_reverse = 0;
+
+  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
+    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
+             "isn't listening for ADDRMAP events.  It probably won't see "
+             "the answer.");
+  }
+
+  {
+    const config_line_t *modearg = config_line_find_case(args->kwargs, "mode");
+    if (modearg && !strcasecmp(modearg->value, "reverse"))
+      is_reverse = 1;
+  }
+  failed = smartlist_new();
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strlen(line->value)) {
+      const char *addr = line->key;
+      if (dnsserv_launch_request(addr, is_reverse, conn)<0)
+        smartlist_add(failed, (char*)addr);
+    } else {
+      // XXXX arguably we should reject unrecognized keyword arguments,
+      // XXXX but the old implementation didn't do that.
+    }
+  }
+
+  send_control_done(conn);
+  SMARTLIST_FOREACH(failed, const char *, arg, {
+      control_event_address_mapped(arg, arg, time(NULL),
+                                   "internal", 0);
+  });
+
+  smartlist_free(failed);
+  return 0;
+}
+
+static const control_cmd_syntax_t protocolinfo_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Return a comma-separated list of authentication methods for
+    handle_control_protocolinfo().  Caller must free this string. */
+static char *
+get_authmethods(const or_options_t *options)
+{
+  int cookies = options->CookieAuthentication;
+  char *methods;
+  int passwd = (options->HashedControlPassword != NULL ||
+                options->HashedControlSessionPassword != NULL);
+  smartlist_t *mlist = smartlist_new();
+
+  if (cookies) {
+    smartlist_add(mlist, (char*)"COOKIE");
+    smartlist_add(mlist, (char*)"SAFECOOKIE");
+  }
+  if (passwd)
+    smartlist_add(mlist, (char*)"HASHEDPASSWORD");
+  if (!cookies && !passwd)
+    smartlist_add(mlist, (char*)"NULL");
+  methods = smartlist_join_strings(mlist, ",", 0, NULL);
+  smartlist_free(mlist);
+
+  return methods;
+}
+
+/** Return escaped cookie filename.  Caller must free this string.
+    Return NULL if cookie authentication is disabled. */
+static char *
+get_esc_cfile(const or_options_t *options)
+{
+  char *cfile = NULL, *abs_cfile = NULL, *esc_cfile = NULL;
+
+  if (!options->CookieAuthentication)
+    return NULL;
+
+  cfile = get_controller_cookie_file_name();
+  abs_cfile = make_path_absolute(cfile);
+  esc_cfile = esc_for_log(abs_cfile);
+  tor_free(cfile);
+  tor_free(abs_cfile);
+  return esc_cfile;
+}
+
+/** Compose the auth methods line of a PROTOCOLINFO reply. */
+static void
+add_authmethods(smartlist_t *reply)
+{
+  const or_options_t *options = get_options();
+  char *methods = get_authmethods(options);
+  char *esc_cfile = get_esc_cfile(options);
+
+  control_reply_add_str(reply, 250, "AUTH");
+  control_reply_append_kv(reply, "METHODS", methods);
+  if (esc_cfile)
+    control_reply_append_kv(reply, "COOKIEFILE", esc_cfile);
+
+  tor_free(methods);
+  tor_free(esc_cfile);
+}
+
+/** Called when we get a PROTOCOLINFO command: send back a reply. */
+static int
+handle_control_protocolinfo(control_connection_t *conn,
+                            const control_cmd_args_t *cmd_args)
+{
+  const char *bad_arg = NULL;
+  const smartlist_t *args = cmd_args->args;
+  smartlist_t *reply = NULL;
+
+  conn->have_sent_protocolinfo = 1;
+
+  SMARTLIST_FOREACH(args, const char *, arg, {
+      int ok;
+      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
+      if (!ok) {
+        bad_arg = arg;
+        break;
+      }
+    });
+  if (bad_arg) {
+    control_printf_endreply(conn, 513, "No such version %s",
+                            escaped(bad_arg));
+    /* Don't tolerate bad arguments when not authenticated. */
+    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
+      connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  }
+  reply = smartlist_new();
+  control_reply_add_str(reply, 250, "PROTOCOLINFO 1");
+  add_authmethods(reply);
+  control_reply_add_str(reply, 250, "VERSION");
+  control_reply_append_kv(reply, "Tor", escaped(VERSION));
+  control_reply_add_done(reply);
+
+  control_write_reply_lines(conn, reply);
+  control_reply_free(reply);
+  return 0;
+}
+
+static const control_cmd_syntax_t usefeature_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a USEFEATURE command: parse the feature list, and
+ * set up the control_connection's options properly. */
+static int
+handle_control_usefeature(control_connection_t *conn,
+                          const control_cmd_args_t *cmd_args)
+{
+  const smartlist_t *args = cmd_args->args;
+  int bad = 0;
+  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
+      if (!strcasecmp(arg, "VERBOSE_NAMES"))
+        ;
+      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
+        ;
+      else {
+        control_printf_endreply(conn, 552, "Unrecognized feature \"%s\"",
+                                arg);
+        bad = 1;
+        break;
+      }
+  } SMARTLIST_FOREACH_END(arg);
+
+  if (!bad) {
+    send_control_done(conn);
+  }
+
+  return 0;
+}
+
+static const control_cmd_syntax_t dropguards_syntax = {
+  .max_args = 0,
+};
+
+/** Implementation for the DROPGUARDS command. */
+static int
+handle_control_dropguards(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  (void) args; /* We don't take arguments. */
+
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
+  remove_all_entry_guards();
+  send_control_done(conn);
+
+  return 0;
+}
+
+static const char *hsfetch_keywords[] = {
+  "SERVER", NULL,
+};
+static const control_cmd_syntax_t hsfetch_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = hsfetch_keywords,
+};
+
+/** Implementation for the HSFETCH command. */
+static int
+handle_control_hsfetch(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+
+{
+  char digest[DIGEST_LEN], *desc_id = NULL;
+  smartlist_t *hsdirs = NULL;
+  static const char *v2_str = "v2-";
+  const size_t v2_str_len = strlen(v2_str);
+  rend_data_t *rend_query = NULL;
+  ed25519_public_key_t v3_pk;
+  uint32_t version;
+  const char *hsaddress = NULL;
+
+  /* Extract the first argument (either HSAddress or DescID). */
+  const char *arg1 = smartlist_get(args->args, 0);
+  /* Test if it's an HS address without the .onion part. */
+  if (rend_valid_v2_service_id(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_TWO;
+  } else if (strcmpstart(arg1, v2_str) == 0 &&
+             rend_valid_descriptor_id(arg1 + v2_str_len) &&
+             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
+                           REND_DESC_ID_V2_LEN_BASE32) ==
+                sizeof(digest)) {
+    /* We have a well formed version 2 descriptor ID. Keep the decoded value
+     * of the id. */
+    desc_id = digest;
+    version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_THREE;
+    hs_parse_address(hsaddress, &v3_pk, NULL, NULL);
+  } else {
+    control_printf_endreply(conn, 513, "Invalid argument \"%s\"", arg1);
+    goto done;
+  }
+
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strcasecmp(line->key, "SERVER")) {
+      const char *server = line->value;
+
+      const node_t *node = node_get_by_hex_id(server, 0);
+      if (!node) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found", server);
+        goto done;
+      }
+      if (!hsdirs) {
+        /* Stores routerstatus_t cmddata for each specified server. */
+        hsdirs = smartlist_new();
+      }
+      /* Valid server, add it to our local list. */
+      smartlist_add(hsdirs, node->rs);
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  if (version == HS_VERSION_TWO) {
+    rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
+                                         REND_NO_AUTH);
+    if (rend_query == NULL) {
+      control_write_endreply(conn, 551, "Error creating the HS query");
+      goto done;
+    }
+  }
+
+  /* Using a descriptor ID, we force the user to provide at least one
+   * hsdir server using the SERVER= option. */
+  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
+    control_write_endreply(conn, 512, "SERVER option is required");
+    goto done;
+  }
+
+  /* We are about to trigger HSDir fetch so send the OK now because after
+   * that 650 event(s) are possible so better to have the 250 OK before them
+   * to avoid out of order replies. */
+  send_control_done(conn);
+
+  /* Trigger the fetch using the built rend query and possibly a list of HS
+   * directory to use. This function ignores the client cache thus this will
+   * always send a fetch command. */
+  if (version == HS_VERSION_TWO) {
+    rend_client_fetch_v2_desc(rend_query, hsdirs);
+  } else if (version == HS_VERSION_THREE) {
+    hs_control_hsfetch_command(&v3_pk, hsdirs);
+  }
+
+ done:
+  /* Contains data pointer that we don't own thus no cleanup. */
+  smartlist_free(hsdirs);
+  rend_data_free(rend_query);
+  return 0;
+}
+
+static const char *hspost_keywords[] = {
+  "SERVER", "HSADDRESS", NULL
+};
+static const control_cmd_syntax_t hspost_syntax = {
+  .min_args = 0, .max_args = 0,
+  .accept_keywords = true,
+  .want_cmddata = true,
+  .allowed_keywords = hspost_keywords
+};
+
+/** Implementation for the HSPOST command. */
+static int
+handle_control_hspost(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  smartlist_t *hs_dirs = NULL;
+  const char *encoded_desc = args->cmddata;
+  size_t encoded_desc_len = args->cmddata_len;
+  const char *onion_address = NULL;
+  const config_line_t *line;
+
+  for (line = args->kwargs; line; line = line->next) {
+    if (!strcasecmpstart(line->key, "SERVER")) {
+      const char *server = line->value;
+      const node_t *node = node_get_by_hex_id(server, 0);
+
+      if (!node || !node->rs) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found",
+                                server);
+        goto done;
+      }
+      /* Valid server, add it to our local list. */
+      if (!hs_dirs)
+        hs_dirs = smartlist_new();
+      smartlist_add(hs_dirs, node->rs);
+    } else if (!strcasecmpstart(line->key, "HSADDRESS")) {
+      const char *address = line->value;
+      if (!hs_address_is_valid(address)) {
+        control_write_endreply(conn, 512, "Malformed onion address");
+        goto done;
+      }
+      onion_address = address;
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  /* Handle the v3 case. */
+  if (onion_address) {
+    if (hs_control_hspost_command(encoded_desc, onion_address, hs_dirs) < 0) {
+      control_write_endreply(conn, 554, "Invalid descriptor");
+    } else {
+      send_control_done(conn);
+    }
+    goto done;
+  }
+
+  /* From this point on, it is only v2. */
+
+  /*  parse it. */
+  rend_encoded_v2_service_descriptor_t *desc =
+      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
+  desc->desc_str = tor_memdup_nulterm(encoded_desc, encoded_desc_len);
+
+  rend_service_descriptor_t *parsed = NULL;
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
+                                        &intro_size, &encoded_size,
+                                        &next_desc, desc->desc_str, 1)) {
+    /* Post the descriptor. */
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    if (!rend_get_service_id(parsed->pk, serviceid)) {
+      smartlist_t *descs = smartlist_new();
+      smartlist_add(descs, desc);
+
+      /* We are about to trigger HS descriptor upload so send the OK now
+       * because after that 650 event(s) are possible so better to have the
+       * 250 OK before them to avoid out of order replies. */
+      send_control_done(conn);
+
+      /* Trigger the descriptor upload */
+      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
+      smartlist_free(descs);
+    }
+
+    rend_service_descriptor_free(parsed);
+  } else {
+    control_write_endreply(conn, 554, "Invalid descriptor");
+  }
+
+  tor_free(intro_content);
+  rend_encoded_v2_service_descriptor_free(desc);
+ done:
+  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
+  return 0;
+}
+
+/* Helper function for ADD_ONION that adds an ephemeral service depending on
+ * the given hs_version.
+ *
+ * The secret key in pk depends on the hs_version. The ownership of the key
+ * used in pk is given to the HS subsystem so the caller must stop accessing
+ * it after.
+ *
+ * The port_cfgs is a list of service port. Ownership transferred to service.
+ * The max_streams refers to the MaxStreams= key.
+ * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
+ * The auth_type is the authentication type of the clients in auth_clients.
+ * The ownership of that list is transferred to the service.
+ *
+ * On success (RSAE_OKAY), the address_out points to a newly allocated string
+ * containing the onion address without the .onion part. On error, address_out
+ * is untouched. */
+static hs_service_add_ephemeral_status_t
+add_onion_helper_add_service(int hs_version,
+                             add_onion_secret_key_t *pk,
+                             smartlist_t *port_cfgs, int max_streams,
+                             int max_streams_close_circuit, int auth_type,
+                             smartlist_t *auth_clients, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+
+  tor_assert(pk);
+  tor_assert(port_cfgs);
+  tor_assert(address_out);
+
+  switch (hs_version) {
+  case HS_VERSION_TWO:
+    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
+                                     max_streams_close_circuit, auth_type,
+                                     auth_clients, address_out);
+    break;
+  case HS_VERSION_THREE:
+    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
+                                   max_streams_close_circuit, address_out);
+    break;
+  default:
+    tor_assert_unreached();
+  }
+
+  return ret;
+}
+
+/** The list of onion services that have been added via ADD_ONION that do not
+ * belong to any particular control connection.
+ */
+static smartlist_t *detached_onion_services = NULL;
+
+/**
+ * Return a list of detached onion services, or NULL if none exist.
+ **/
+smartlist_t *
+get_detached_onion_services(void)
+{
+  return detached_onion_services;
+}
+
+static const char *add_onion_keywords[] = {
+   "Port", "Flags", "MaxStreams", "ClientAuth", NULL
+};
+static const control_cmd_syntax_t add_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = add_onion_keywords
+};
+
+/** Called when we get a ADD_ONION command; parse the body, and set up
+ * the new ephemeral Onion Service. */
+static int
+handle_control_add_onion(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  /* Parse all of the arguments that do not involve handling cryptographic
+   * material first, since there's no reason to touch that at all if any of
+   * the other arguments are malformed.
+   */
+  smartlist_t *port_cfgs = smartlist_new();
+  smartlist_t *auth_clients = NULL;
+  smartlist_t *auth_created_clients = NULL;
+  int discard_pk = 0;
+  int detach = 0;
+  int max_streams = 0;
+  int max_streams_close_circuit = 0;
+  rend_auth_type_t auth_type = REND_NO_AUTH;
+  int non_anonymous = 0;
+  const config_line_t *arg;
+
+  for (arg = args->kwargs; arg; arg = arg->next) {
+    if (!strcasecmp(arg->key, "Port")) {
+      /* "Port=VIRTPORT[,TARGET]". */
+      rend_service_port_config_t *cfg =
+          rend_service_parse_port_config(arg->value, ",", NULL);
+      if (!cfg) {
+        control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+        goto out;
+      }
+      smartlist_add(port_cfgs, cfg);
+    } else if (!strcasecmp(arg->key, "MaxStreams")) {
+      /* "MaxStreams=[0..65535]". */
+      int ok = 0;
+      max_streams = (int)tor_parse_long(arg->value, 10, 0, 65535, &ok, NULL);
+      if (!ok) {
+        control_write_endreply(conn, 512, "Invalid MaxStreams");
+        goto out;
+      }
+    } else if (!strcasecmp(arg->key, "Flags")) {
+      /* "Flags=Flag[,Flag]", where Flag can be:
+       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
+       *                   the response.
+       *   * 'Detach' - Do not tie this onion service to any particular control
+       *                connection.
+       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
+       *                                exceeded.
+       *   * 'BasicAuth' - Client authorization using the 'basic' method.
+       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
+       *                      flag is present, tor must be in non-anonymous
+       *                      hidden service mode. If this flag is absent,
+       *                      tor must be in anonymous hidden service mode.
+       */
+      static const char *discard_flag = "DiscardPK";
+      static const char *detach_flag = "Detach";
+      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
+      static const char *basicauth_flag = "BasicAuth";
+      static const char *non_anonymous_flag = "NonAnonymous";
+
+      smartlist_t *flags = smartlist_new();
+      int bad = 0;
+
+      smartlist_split_string(flags, arg->value, ",", SPLIT_IGNORE_BLANK, 0);
+      if (smartlist_len(flags) < 1) {
+        control_write_endreply(conn, 512, "Invalid 'Flags' argument");
+        bad = 1;
+      }
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
+      {
+        if (!strcasecmp(flag, discard_flag)) {
+          discard_pk = 1;
+        } else if (!strcasecmp(flag, detach_flag)) {
+          detach = 1;
+        } else if (!strcasecmp(flag, max_s_close_flag)) {
+          max_streams_close_circuit = 1;
+        } else if (!strcasecmp(flag, basicauth_flag)) {
+          auth_type = REND_BASIC_AUTH;
+        } else if (!strcasecmp(flag, non_anonymous_flag)) {
+          non_anonymous = 1;
+        } else {
+          control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s",
+                                  escaped(flag));
+          bad = 1;
+          break;
+        }
+      } SMARTLIST_FOREACH_END(flag);
+      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
+      smartlist_free(flags);
+      if (bad)
+        goto out;
+
+    } else if (!strcasecmp(arg->key, "ClientAuth")) {
+      int created = 0;
+      rend_authorized_client_t *client =
+        add_onion_helper_clientauth(arg->value, &created, conn);
+      if (!client) {
+        goto out;
+      }
+
+      if (auth_clients != NULL) {
+        int bad = 0;
+        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
+          if (strcmp(ac->client_name, client->client_name) == 0) {
+            bad = 1;
+            break;
+          }
+        } SMARTLIST_FOREACH_END(ac);
+        if (bad) {
+          control_write_endreply(conn, 512, "Duplicate name in ClientAuth");
+          rend_authorized_client_free(client);
+          goto out;
+        }
+      } else {
+        auth_clients = smartlist_new();
+        auth_created_clients = smartlist_new();
+      }
+      smartlist_add(auth_clients, client);
+      if (created) {
+        smartlist_add(auth_created_clients, client);
+      }
+    } else {
+      tor_assert_nonfatal_unreached();
+      goto out;
+    }
+  }
+  if (smartlist_len(port_cfgs) == 0) {
+    control_write_endreply(conn, 512, "Missing 'Port' argument");
+    goto out;
+  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
+    control_write_endreply(conn, 512, "No auth type specified");
+    goto out;
+  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
+    control_write_endreply(conn, 512, "No auth clients specified");
+    goto out;
+  } else if ((auth_type == REND_BASIC_AUTH &&
+              smartlist_len(auth_clients) > 512) ||
+             (auth_type == REND_STEALTH_AUTH &&
+              smartlist_len(auth_clients) > 16)) {
+    control_write_endreply(conn, 512, "Too many auth clients");
+    goto out;
+  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
+                                                              get_options())) {
+    /* If we failed, and the non-anonymous flag is set, Tor must be in
+     * anonymous hidden service mode.
+     * The error message changes based on the current Tor config:
+     * 512 Tor is in anonymous hidden service mode
+     * 512 Tor is in non-anonymous hidden service mode
+     * (I've deliberately written them out in full here to aid searchability.)
+     */
+    control_printf_endreply(conn, 512,
+                            "Tor is in %sanonymous hidden service " "mode",
+                            non_anonymous ? "" : "non-");
+    goto out;
+  }
+
+  /* Parse the "keytype:keyblob" argument. */
+  int hs_version = 0;
+  add_onion_secret_key_t pk = { NULL };
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+
+  const char *onionkey = smartlist_get(args->args, 0);
+  if (add_onion_helper_keyarg(onionkey, discard_pk,
+                              &key_new_alg, &key_new_blob, &pk, &hs_version,
+                              conn) < 0) {
+    goto out;
+  }
+
+  /* Hidden service version 3 don't have client authentication support so if
+   * ClientAuth was given, send back an error. */
+  if (hs_version == HS_VERSION_THREE && auth_clients) {
+    control_write_endreply(conn, 513, "ClientAuth not supported");
+    goto out;
+  }
+
+  /* Create the HS, using private key pk, client authentication auth_type,
+   * the list of auth_clients, and port config port_cfg.
+   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
+   * regardless of success/failure.
+   */
+  char *service_id = NULL;
+  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
+                                         max_streams,
+                                         max_streams_close_circuit, auth_type,
+                                         auth_clients, &service_id);
+  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
+  auth_clients = NULL; /* so is auth_clients */
+  switch (ret) {
+  case RSAE_OKAY:
+  {
+    if (detach) {
+      if (!detached_onion_services)
+        detached_onion_services = smartlist_new();
+      smartlist_add(detached_onion_services, service_id);
+    } else {
+      if (!conn->ephemeral_onion_services)
+        conn->ephemeral_onion_services = smartlist_new();
+      smartlist_add(conn->ephemeral_onion_services, service_id);
+    }
+
+    tor_assert(service_id);
+    control_printf_midreply(conn, 250, "ServiceID=%s", service_id);
+    if (key_new_alg) {
+      tor_assert(key_new_blob);
+      control_printf_midreply(conn, 250, "PrivateKey=%s:%s",
+                              key_new_alg, key_new_blob);
+    }
+    if (auth_created_clients) {
+      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
+        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
+                                                auth_type);
+        tor_assert(encoded);
+        control_printf_midreply(conn, 250, "ClientAuth=%s:%s",
+                                ac->client_name, encoded);
+        memwipe(encoded, 0, strlen(encoded));
+        tor_free(encoded);
+      });
+    }
+
+    send_control_done(conn);
+    break;
+  }
+  case RSAE_BADPRIVKEY:
+    control_write_endreply(conn, 551, "Failed to generate onion address");
+    break;
+  case RSAE_ADDREXISTS:
+    control_write_endreply(conn, 550, "Onion address collision");
+    break;
+  case RSAE_BADVIRTPORT:
+    control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+    break;
+  case RSAE_BADAUTH:
+    control_write_endreply(conn, 512, "Invalid client authorization");
+    break;
+  case RSAE_INTERNAL: FALLTHROUGH;
+  default:
+    control_write_endreply(conn, 551, "Failed to add Onion Service");
+  }
+  if (key_new_blob) {
+    memwipe(key_new_blob, 0, strlen(key_new_blob));
+    tor_free(key_new_blob);
+  }
+
+ out:
+  if (port_cfgs) {
+    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
+                      rend_service_port_config_free(p));
+    smartlist_free(port_cfgs);
+  }
+
+  if (auth_clients) {
+    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
+                      rend_authorized_client_free(ac));
+    smartlist_free(auth_clients);
+  }
+  if (auth_created_clients) {
+    // Do not free entries; they are the same as auth_clients
+    smartlist_free(auth_created_clients);
+  }
+  return 0;
+}
+
+/** Helper function to handle parsing the KeyType:KeyBlob argument to the
+ * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
+ * and the private key not discarded, the algorithm and serialized private key,
+ * or NULL and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned key_new_blob.
+ *
+ * Note: The error messages returned are deliberately vague to avoid echoing
+ * key material.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC int
+add_onion_helper_keyarg(const char *arg, int discard_pk,
+                        const char **key_new_alg_out, char **key_new_blob_out,
+                        add_onion_secret_key_t *decoded_key, int *hs_version,
+                        control_connection_t *conn)
+{
+  smartlist_t *key_args = smartlist_new();
+  crypto_pk_t *pk = NULL;
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+  int ret = -1;
+
+  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(key_args) != 2) {
+    control_write_endreply(conn, 512, "Invalid key type/blob");
+    goto err;
+  }
+
+  /* The format is "KeyType:KeyBlob". */
+  static const char *key_type_new = "NEW";
+  static const char *key_type_best = "BEST";
+  static const char *key_type_rsa1024 = "RSA1024";
+  static const char *key_type_ed25519_v3 = "ED25519-V3";
+
+  const char *key_type = smartlist_get(key_args, 0);
+  const char *key_blob = smartlist_get(key_args, 1);
+
+  if (!strcasecmp(key_type_rsa1024, key_type)) {
+    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
+    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
+    if (!pk) {
+      control_write_endreply(conn, 512, "Failed to decode RSA key");
+      goto err;
+    }
+    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
+      crypto_pk_free(pk);
+      control_write_endreply(conn, 512, "Invalid RSA key size");
+      goto err;
+    }
+    decoded_key->v2 = pk;
+    *hs_version = HS_VERSION_TWO;
+  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
+    /* parsing of private ed25519 key */
+    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
+    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
+                      strlen(key_blob)) != sizeof(sk->seckey)) {
+      tor_free(sk);
+      control_write_endreply(conn, 512, "Failed to decode ED25519-V3 key");
+      goto err;
+    }
+    decoded_key->v3 = sk;
+    *hs_version = HS_VERSION_THREE;
+  } else if (!strcasecmp(key_type_new, key_type)) {
+    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
+    if (!strcasecmp(key_type_rsa1024, key_blob)) {
+      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
+      pk = crypto_pk_new();
+      if (crypto_pk_generate_key(pk)) {
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_rsa1024);
+        goto err;
+      }
+      if (!discard_pk) {
+        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
+          crypto_pk_free(pk);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_rsa1024);
+          goto err;
+        }
+        key_new_alg = key_type_rsa1024;
+      }
+      decoded_key->v2 = pk;
+      *hs_version = HS_VERSION_TWO;
+    } else if (!strcasecmp(key_type_ed25519_v3, key_blob) ||
+               !strcasecmp(key_type_best, key_blob)) {
+      /* "ED25519-V3", ed25519 key, also currently "BEST" by default. */
+      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+      if (ed25519_secret_key_generate(sk, 1) < 0) {
+        tor_free(sk);
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_ed25519_v3);
+        goto err;
+      }
+      if (!discard_pk) {
+        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
+        key_new_blob = tor_malloc_zero(len);
+        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
+                          sizeof(sk->seckey), 0) != (len - 1)) {
+          tor_free(sk);
+          tor_free(key_new_blob);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_ed25519_v3);
+          goto err;
+        }
+        key_new_alg = key_type_ed25519_v3;
+      }
+      decoded_key->v3 = sk;
+      *hs_version = HS_VERSION_THREE;
+    } else {
+      control_write_endreply(conn, 513, "Invalid key type");
+      goto err;
+    }
+  } else {
+    control_write_endreply(conn, 513, "Invalid key type");
+    goto err;
+  }
+
+  /* Succeeded in loading or generating a private key. */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(key_args, char *, cp, {
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+  });
+  smartlist_free(key_args);
+
+  *key_new_alg_out = key_new_alg;
+  *key_new_blob_out = key_new_blob;
+
+  return ret;
+}
+
+/** Helper function to handle parsing a ClientAuth argument to the
+ * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
+ * and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned auth_client.
+ *
+ * If 'created' is specified, it will be set to 1 when a new cookie has
+ * been generated.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC rend_authorized_client_t *
+add_onion_helper_clientauth(const char *arg, int *created,
+                            control_connection_t *conn)
+{
+  int ok = 0;
+
+  tor_assert(arg);
+  tor_assert(created);
+
+  smartlist_t *auth_args = smartlist_new();
+  rend_authorized_client_t *client =
+    tor_malloc_zero(sizeof(rend_authorized_client_t));
+  smartlist_split_string(auth_args, arg, ":", 0, 0);
+  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
+    control_write_endreply(conn, 512, "Invalid ClientAuth syntax");
+    goto err;
+  }
+  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
+  if (smartlist_len(auth_args) == 2) {
+    char *decode_err_msg = NULL;
+    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
+                                client->descriptor_cookie,
+                                NULL, &decode_err_msg) < 0) {
+      tor_assert(decode_err_msg);
+      control_write_endreply(conn, 512, decode_err_msg);
+      tor_free(decode_err_msg);
+      goto err;
+    }
+    *created = 0;
+  } else {
+    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
+    *created = 1;
+  }
+
+  if (!rend_valid_client_name(client->client_name)) {
+    control_write_endreply(conn, 512, "Invalid name in ClientAuth");
+    goto err;
+  }
+
+  ok = 1;
+ err:
+  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
+  smartlist_free(auth_args);
+  if (!ok) {
+    rend_authorized_client_free(client);
+    client = NULL;
+  }
+  return client;
+}
+
+static const control_cmd_syntax_t del_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+};
+
+/** Called when we get a DEL_ONION command; parse the body, and remove
+ * the existing ephemeral Onion Service. */
+static int
+handle_control_del_onion(control_connection_t *conn,
+                         const control_cmd_args_t *cmd_args)
+{
+  int hs_version = 0;
+  smartlist_t *args = cmd_args->args;
+  tor_assert(smartlist_len(args) == 1);
+
+  const char *service_id = smartlist_get(args, 0);
+  if (rend_valid_v2_service_id(service_id)) {
+    hs_version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(service_id)) {
+    hs_version = HS_VERSION_THREE;
+  } else {
+    control_write_endreply(conn, 512, "Malformed Onion Service id");
+    goto out;
+  }
+
+  /* Determine if the onion service belongs to this particular control
+   * connection, or if it is in the global list of detached services.  If it
+   * is in neither, either the service ID is invalid in some way, or it
+   * explicitly belongs to a different control connection, and an error
+   * should be returned.
+   */
+  smartlist_t *services[2] = {
+    conn->ephemeral_onion_services,
+    detached_onion_services
+  };
+  smartlist_t *onion_services = NULL;
+  int idx = -1;
+  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
+    idx = smartlist_string_pos(services[i], service_id);
+    if (idx != -1) {
+      onion_services = services[i];
+      break;
+    }
+  }
+  if (onion_services == NULL) {
+    control_write_endreply(conn, 552, "Unknown Onion Service id");
+  } else {
+    int ret = -1;
+    switch (hs_version) {
+    case HS_VERSION_TWO:
+      ret = rend_service_del_ephemeral(service_id);
+      break;
+    case HS_VERSION_THREE:
+      ret = hs_service_del_ephemeral(service_id);
+      break;
+    default:
+      /* The ret value will be -1 thus hitting the warning below. This should
+       * never happen because of the check at the start of the function. */
+      break;
+    }
+    if (ret < 0) {
+      /* This should *NEVER* fail, since the service is on either the
+       * per-control connection list, or the global one.
+       */
+      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
+               escaped(service_id));
+      tor_fragile_assert();
+    }
+
+    /* Remove/scrub the service_id from the appropriate list. */
+    char *cp = smartlist_get(onion_services, idx);
+    smartlist_del(onion_services, idx);
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+
+    send_control_done(conn);
+  }
+
+ out:
+  return 0;
+}
+
+static const control_cmd_syntax_t obsolete_syntax = {
+  .max_args = UINT_MAX
+};
+
+/**
+ * Called when we get an obsolete command: tell the controller that it is
+ * obsolete.
+ */
+static int
+handle_control_obsolete(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  (void)args;
+  char *command = tor_strdup(conn->current_cmd);
+  tor_strupper(command);
+  control_printf_endreply(conn, 511, "%s is obsolete.", command);
+  tor_free(command);
+  return 0;
+}
+
+/**
+ * Function pointer to a handler function for a controller command.
+ **/
+typedef int (*handler_fn_t) (control_connection_t *conn,
+                             const control_cmd_args_t *args);
+
+/**
+ * Definition for a controller command.
+ */
+typedef struct control_cmd_def_t {
+  /**
+   * The name of the command. If the command is multiline, the name must
+   * begin with "+".  This is not case-sensitive. */
+  const char *name;
+  /**
+   * A function to execute the command.
+   */
+  handler_fn_t handler;
+  /**
+   * Zero or more CMD_FL_* flags, or'd together.
+   */
+  unsigned flags;
+  /**
+   * For parsed command: a syntax description.
+   */
+  const control_cmd_syntax_t *syntax;
+} control_cmd_def_t;
+
+/**
+ * Indicates that the command's arguments are sensitive, and should be
+ * memwiped after use.
+ */
+#define CMD_FL_WIPE (1u<<0)
+
+#ifndef COCCI
+/** Macro: declare a command with a one-line argument, a given set of flags,
+ * and a syntax definition.
+ **/
+#define ONE_LINE(name, flags)                                   \
+  {                                                             \
+    (#name),                                                    \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax,                                           \
+   }
+
+/**
+ * Macro: declare a command with a multi-line argument and a given set of
+ * flags.
+ **/
+#define MULTLINE(name, flags)                                   \
+  { ("+"#name),                                                 \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax                                            \
+  }
+
+/**
+ * Macro: declare an obsolete command. (Obsolete commands give a different
+ * error than non-existent ones.)
+ **/
+#define OBSOLETE(name)                          \
+  { #name,                                      \
+      handle_control_obsolete,                  \
+      0,                                        \
+      &obsolete_syntax,                         \
+  }
+#endif /* !defined(COCCI) */
+
+/**
+ * An array defining all the recognized controller commands.
+ **/
+static const control_cmd_def_t CONTROL_COMMANDS[] =
+{
+  ONE_LINE(setconf, 0),
+  ONE_LINE(resetconf, 0),
+  ONE_LINE(getconf, 0),
+  MULTLINE(loadconf, 0),
+  ONE_LINE(setevents, 0),
+  ONE_LINE(authenticate, CMD_FL_WIPE),
+  ONE_LINE(saveconf, 0),
+  ONE_LINE(signal, 0),
+  ONE_LINE(takeownership, 0),
+  ONE_LINE(dropownership, 0),
+  ONE_LINE(mapaddress, 0),
+  ONE_LINE(getinfo, 0),
+  ONE_LINE(extendcircuit, 0),
+  ONE_LINE(setcircuitpurpose, 0),
+  OBSOLETE(setrouterpurpose),
+  ONE_LINE(attachstream, 0),
+  MULTLINE(postdescriptor, 0),
+  ONE_LINE(redirectstream, 0),
+  ONE_LINE(closestream, 0),
+  ONE_LINE(closecircuit, 0),
+  ONE_LINE(usefeature, 0),
+  ONE_LINE(resolve, 0),
+  ONE_LINE(protocolinfo, 0),
+  ONE_LINE(authchallenge, CMD_FL_WIPE),
+  ONE_LINE(dropguards, 0),
+  ONE_LINE(hsfetch, 0),
+  MULTLINE(hspost, 0),
+  ONE_LINE(add_onion, CMD_FL_WIPE),
+  ONE_LINE(del_onion, CMD_FL_WIPE),
+  ONE_LINE(onion_client_auth_add, CMD_FL_WIPE),
+  ONE_LINE(onion_client_auth_remove, 0),
+  ONE_LINE(onion_client_auth_view, 0),
+};
+
+/**
+ * The number of entries in CONTROL_COMMANDS.
+ **/
+static const size_t N_CONTROL_COMMANDS = ARRAY_LENGTH(CONTROL_COMMANDS);
+
+/**
+ * Run a single control command, as defined by a control_cmd_def_t,
+ * with a given set of arguments.
+ */
+static int
+handle_single_control_command(const control_cmd_def_t *def,
+                              control_connection_t *conn,
+                              uint32_t cmd_data_len,
+                              char *args)
+{
+  int rv = 0;
+
+  control_cmd_args_t *parsed_args;
+  char *err=NULL;
+  tor_assert(def->syntax);
+  parsed_args = control_cmd_parse_args(conn->current_cmd,
+                                       def->syntax,
+                                       cmd_data_len, args,
+                                       &err);
+  if (!parsed_args) {
+    control_printf_endreply(conn, 512, "Bad arguments to %s: %s",
+                            conn->current_cmd, err?err:"");
+    tor_free(err);
+  } else {
+    if (BUG(err))
+      tor_free(err);
+    if (def->handler(conn, parsed_args))
+      rv = 0;
+
+    if (def->flags & CMD_FL_WIPE)
+      control_cmd_args_wipe(parsed_args);
+
+    control_cmd_args_free(parsed_args);
+  }
+
+  if (def->flags & CMD_FL_WIPE)
+    memwipe(args, 0, cmd_data_len);
+
+  return rv;
+}
+
+/**
+ * Run a given controller command, as selected by the current_cmd field of
+ * <b>conn</b>.
+ */
+int
+handle_control_command(control_connection_t *conn,
+                       uint32_t cmd_data_len,
+                       char *args)
+{
+  tor_assert(conn);
+  tor_assert(args);
+  tor_assert(args[cmd_data_len] == '\0');
+
+  for (unsigned i = 0; i < N_CONTROL_COMMANDS; ++i) {
+    const control_cmd_def_t *def = &CONTROL_COMMANDS[i];
+    if (!strcasecmp(conn->current_cmd, def->name)) {
+      return handle_single_control_command(def, conn, cmd_data_len, args);
+    }
+  }
+
+  control_printf_endreply(conn, 510, "Unrecognized command \"%s\"",
+                          conn->current_cmd);
+
+  return 0;
+}
+
+void
+control_cmd_free_all(void)
+{
+  if (detached_onion_services) { /* Free the detached onion services */
+    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
+    smartlist_free(detached_onion_services);
+  }
+}
diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h
new file mode 100644
index 0000000000..0ff0f0755f
--- /dev/null
+++ b/src/feature/control/control_cmd.h
@@ -0,0 +1,113 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.h
+ * \brief Header file for control_cmd.c.
+ **/
+
+#ifndef TOR_CONTROL_CMD_H
+#define TOR_CONTROL_CMD_H
+
+#include "lib/malloc/malloc.h"
+
+int handle_control_command(control_connection_t *conn,
+                           uint32_t cmd_data_len,
+                           char *args);
+void control_cmd_free_all(void);
+
+typedef struct control_cmd_args_t control_cmd_args_t;
+void control_cmd_args_free_(control_cmd_args_t *args);
+void control_cmd_args_wipe(control_cmd_args_t *args);
+
+#define control_cmd_args_free(v) \
+  FREE_AND_NULL(control_cmd_args_t, control_cmd_args_free_, (v))
+
+/**
+ * Definition for the syntax of a controller command, as parsed by
+ * control_cmd_parse_args.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+typedef struct control_cmd_syntax_t {
+  /**
+   * Lowest number of positional arguments that this command accepts.
+   * 0 for "it's okay not to have positional arguments."
+   **/
+  unsigned int min_args;
+  /**
+   * Highest number of positional arguments that this command accepts.
+   * UINT_MAX for no limit.
+   **/
+  unsigned int max_args;
+  /**
+   * If true, we should parse options after the positional arguments
+   * as a set of unordered flags and key=value arguments.
+   *
+   * Requires that max_args is not UINT_MAX.
+   **/
+  bool accept_keywords;
+  /**
+   * If accept_keywords is true, then only the keywords listed in this
+   * (NULL-terminated) array are valid keywords for this command.
+   **/
+  const char **allowed_keywords;
+  /**
+   * If accept_keywords is true, this option is passed to kvline_parse() as
+   * its flags.
+   **/
+  unsigned kvline_flags;
+  /**
+   * True iff this command wants to be followed by a multiline object.
+   **/
+  bool want_cmddata;
+  /**
+   * True iff this command needs access to the raw body of the input.
+   *
+   * This should not be needed for pure commands; it is purely a legacy
+   * option.
+   **/
+  bool store_raw_body;
+} control_cmd_syntax_t;
+
+#ifdef CONTROL_CMD_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/* ADD_ONION secret key to create an ephemeral service. The command supports
+ * multiple versions so this union stores the key and passes it to the HS
+ * subsystem depending on the requested version. */
+typedef union add_onion_secret_key_t {
+  /* Hidden service v2 secret key. */
+  crypto_pk_t *v2;
+  /* Hidden service v3 secret key. */
+  ed25519_secret_key_t *v3;
+} add_onion_secret_key_t;
+
+STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
+                                   const char **key_new_alg_out,
+                                   char **key_new_blob_out,
+                                   add_onion_secret_key_t *decoded_key,
+                                   int *hs_version,
+                                   control_connection_t *conn);
+
+STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg,
+                                   int *created, control_connection_t *conn);
+
+STATIC control_cmd_args_t *control_cmd_parse_args(
+                                   const char *command,
+                                   const control_cmd_syntax_t *syntax,
+                                   size_t body_len,
+                                   const char *body,
+                                   char **error_out);
+
+#endif /* defined(CONTROL_CMD_PRIVATE) */
+
+#ifdef CONTROL_MODULE_PRIVATE
+smartlist_t * get_detached_onion_services(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_CMD_H) */
diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h
new file mode 100644
index 0000000000..e7d064c6fe
--- /dev/null
+++ b/src/feature/control/control_cmd_args_st.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd_args_st.h
+ * \brief Definition for control_cmd_args_t
+ **/
+
+#ifndef TOR_CONTROL_CMD_ST_H
+#define TOR_CONTROL_CMD_ST_H
+
+struct smartlist_t;
+struct config_line_t;
+
+/**
+ * Parsed arguments for a control command.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+struct control_cmd_args_t {
+  /**
+   * The command itself, as provided by the controller.  Not owned by this
+   * structure.
+   **/
+  const char *command;
+  /**
+   * Positional arguments to the command.
+   **/
+  struct smartlist_t *args;
+  /**
+   * Keyword arguments to the command.
+   **/
+  struct config_line_t *kwargs;
+  /**
+   * Number of bytes in <b>cmddata</b>; 0 if <b>cmddata</b> is not set.
+   **/
+  size_t cmddata_len;
+  /**
+   * A multiline object passed with this command.
+   **/
+  char *cmddata;
+  /**
+   * If set, a nul-terminated string containing the raw unparsed arguments.
+   **/
+  const char *raw_body;
+};
+
+#endif /* !defined(TOR_CONTROL_CMD_ST_H) */
diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h
index 177a916257..9e410324e0 100644
--- a/src/feature/control/control_connection_st.h
+++ b/src/feature/control/control_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file control_connection_st.h
+ * @brief Controller connection structure.
+ **/
+
 #ifndef CONTROL_CONNECTION_ST_H
 #define CONTROL_CONNECTION_ST_H
 
@@ -40,7 +45,8 @@ struct control_connection_t {
   /** A control command that we're reading from the inbuf, but which has not
    * yet arrived completely. */
   char *incoming_cmd;
+  /** The control command that we are currently processing. */
+  char *current_cmd;
 };
 
-#endif
-
+#endif /* !defined(CONTROL_CONNECTION_ST_H) */
diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c
new file mode 100644
index 0000000000..916ccea875
--- /dev/null
+++ b/src/feature/control/control_events.c
@@ -0,0 +1,2314 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.c
+ * \brief Implement the event-reporting part of the controller API.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/command.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/reasons.h"
+#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/control/control_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "lib/evloop/compat_libevent.h"
+#include "lib/encoding/confline.h"
+
+static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
+static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/** An event mask of all the events that any controller is interested in
+ * receiving. */
+static event_mask_t global_event_mask = 0;
+
+/** True iff we have disabled log messages from being sent to the controller */
+static int disable_log_messages = 0;
+
+/** Macro: true if any control connection is interested in events of type
+ * <b>e</b>. */
+#define EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & EVENT_MASK_(e)))
+
+/** Macro: true if any event from the bitfield 'e' is interesting. */
+#define ANY_EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & (e)))
+
+static void send_control_event_impl(uint16_t event,
+                                    const char *format, va_list ap)
+  CHECK_PRINTF(2,0);
+static int control_event_status(int type, int severity, const char *format,
+                                va_list args)
+  CHECK_PRINTF(3,0);
+
+static void send_control_event(uint16_t event,
+                               const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+/** Table mapping event values to their names.  Used to implement SETEVENTS
+ * and GETINFO events/names, and to keep they in sync. */
+const struct control_event_t control_event_table[] = {
+  { EVENT_CIRCUIT_STATUS, "CIRC" },
+  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
+  { EVENT_STREAM_STATUS, "STREAM" },
+  { EVENT_OR_CONN_STATUS, "ORCONN" },
+  { EVENT_BANDWIDTH_USED, "BW" },
+  { EVENT_DEBUG_MSG, "DEBUG" },
+  { EVENT_INFO_MSG, "INFO" },
+  { EVENT_NOTICE_MSG, "NOTICE" },
+  { EVENT_WARN_MSG, "WARN" },
+  { EVENT_ERR_MSG, "ERR" },
+  { EVENT_NEW_DESC, "NEWDESC" },
+  { EVENT_ADDRMAP, "ADDRMAP" },
+  { EVENT_DESCCHANGED, "DESCCHANGED" },
+  { EVENT_NS, "NS" },
+  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
+  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
+  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
+  { EVENT_GUARD, "GUARD" },
+  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
+  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
+  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
+  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
+  { EVENT_GOT_SIGNAL, "SIGNAL" },
+  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
+  { EVENT_CONN_BW, "CONN_BW" },
+  { EVENT_CELL_STATS, "CELL_STATS" },
+  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
+  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
+  { EVENT_HS_DESC, "HS_DESC" },
+  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
+  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
+  { 0, NULL },
+};
+
+/** Given a log severity, return the corresponding control event code. */
+static inline int
+log_severity_to_event(int severity)
+{
+  switch (severity) {
+    case LOG_DEBUG: return EVENT_DEBUG_MSG;
+    case LOG_INFO: return EVENT_INFO_MSG;
+    case LOG_NOTICE: return EVENT_NOTICE_MSG;
+    case LOG_WARN: return EVENT_WARN_MSG;
+    case LOG_ERR: return EVENT_ERR_MSG;
+    default: return -1;
+  }
+}
+
+/** Helper: clear bandwidth counters of all origin circuits. */
+static void
+clear_circ_bw_fields(void)
+{
+  origin_circuit_t *ocirc;
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+  }
+  SMARTLIST_FOREACH_END(circ);
+}
+
+/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
+ * connection's event_mask field. */
+void
+control_update_global_event_mask(void)
+{
+  smartlist_t *conns = get_connection_array();
+  event_mask_t old_mask, new_mask;
+  old_mask = global_event_mask;
+  int any_old_per_sec_events = control_any_per_second_event_enabled();
+
+  global_event_mask = 0;
+  SMARTLIST_FOREACH(conns, connection_t *, _conn,
+  {
+    if (_conn->type == CONN_TYPE_CONTROL &&
+        STATE_IS_OPEN(_conn->state)) {
+      control_connection_t *conn = TO_CONTROL_CONN(_conn);
+      global_event_mask |= conn->event_mask;
+    }
+  });
+
+  new_mask = global_event_mask;
+
+  /* Handle the aftermath.  Set up the log callback to tell us only what
+   * we want to hear...*/
+  control_adjust_event_log_severity();
+
+  /* Macro: true if ev was false before and is true now. */
+#define NEWLY_ENABLED(ev) \
+  (! (old_mask & (ev)) && (new_mask & (ev)))
+
+  /* ...then, if we've started logging stream or circ bw, clear the
+   * appropriate fields. */
+  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
+    SMARTLIST_FOREACH(conns, connection_t *, conn,
+    {
+      if (conn->type == CONN_TYPE_AP) {
+        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
+        edge_conn->n_written = edge_conn->n_read = 0;
+      }
+    });
+  }
+  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
+    clear_circ_bw_fields();
+  }
+  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
+    uint64_t r, w;
+    control_get_bytes_rw_last_sec(&r, &w);
+  }
+  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
+    rescan_periodic_events(get_options());
+  }
+
+#undef NEWLY_ENABLED
+}
+
+/** Given a control event code for a message event, return the corresponding
+ * log severity. */
+static inline int
+event_to_log_severity(int event)
+{
+  switch (event) {
+    case EVENT_DEBUG_MSG: return LOG_DEBUG;
+    case EVENT_INFO_MSG: return LOG_INFO;
+    case EVENT_NOTICE_MSG: return LOG_NOTICE;
+    case EVENT_WARN_MSG: return LOG_WARN;
+    case EVENT_ERR_MSG: return LOG_ERR;
+    default: return -1;
+  }
+}
+
+/** Adjust the log severities that result in control_event_logmsg being called
+ * to match the severity of log messages that any controllers are interested
+ * in. */
+void
+control_adjust_event_log_severity(void)
+{
+  int i;
+  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
+
+  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      min_log_event = i;
+      break;
+    }
+  }
+  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      max_log_event = i;
+      break;
+    }
+  }
+  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
+    if (min_log_event > EVENT_NOTICE_MSG)
+      min_log_event = EVENT_NOTICE_MSG;
+    if (max_log_event < EVENT_ERR_MSG)
+      max_log_event = EVENT_ERR_MSG;
+  }
+  if (min_log_event <= max_log_event)
+    change_callback_log_severity(event_to_log_severity(min_log_event),
+                                 event_to_log_severity(max_log_event),
+                                 control_event_logmsg);
+  else
+    change_callback_log_severity(LOG_ERR, LOG_ERR,
+                                 control_event_logmsg);
+}
+
+/** Return true iff the event with code <b>c</b> is being sent to any current
+ * control connection.  This is useful if the amount of work needed to prepare
+ * to call the appropriate control_event_...() function is high.
+ */
+int
+control_event_is_interesting(int event)
+{
+  return EVENT_IS_INTERESTING(event);
+}
+
+/** Return true if any event that needs to fire once a second is enabled. */
+int
+control_any_per_second_event_enabled(void)
+{
+  return ANY_EVENT_IS_INTERESTING(
+      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CELL_STATS) |
+      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CONN_BW) |
+      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
+  );
+}
+
+/* The value of 'get_bytes_read()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_read = 0;
+/* The value of 'get_bytes_written()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_written = 0;
+
+/**
+ * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
+ * and written by Tor since the last call to this function.
+ *
+ * Call this only from the main thread.
+ */
+static void
+control_get_bytes_rw_last_sec(uint64_t *n_read,
+                              uint64_t *n_written)
+{
+  const uint64_t stats_n_bytes_read = get_bytes_read();
+  const uint64_t stats_n_bytes_written = get_bytes_written();
+
+  *n_read = stats_n_bytes_read - stats_prev_n_read;
+  *n_written = stats_n_bytes_written - stats_prev_n_written;
+  stats_prev_n_read = stats_n_bytes_read;
+  stats_prev_n_written = stats_n_bytes_written;
+}
+
+/**
+ * Run all the controller events (if any) that are scheduled to trigger once
+ * per second.
+ */
+void
+control_per_second_events(void)
+{
+  if (!control_any_per_second_event_enabled())
+    return;
+
+  uint64_t bytes_read, bytes_written;
+  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
+  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
+
+  control_event_stream_bandwidth_used();
+  control_event_conn_bandwidth_used();
+  control_event_circ_bandwidth_used();
+  control_event_circuit_cell_stats();
+}
+
+/** Represents an event that's queued to be sent to one or more
+ * controllers. */
+typedef struct queued_event_t {
+  uint16_t event;
+  char *msg;
+} queued_event_t;
+
+/** Pointer to int. If this is greater than 0, we don't allow new events to be
+ * queued. */
+static tor_threadlocal_t block_event_queue_flag;
+
+/** Holds a smartlist of queued_event_t objects that may need to be sent
+ * to one or more controllers */
+static smartlist_t *queued_control_events = NULL;
+
+/** True if the flush_queued_events_event is pending. */
+static int flush_queued_event_pending = 0;
+
+/** Lock to protect the above fields. */
+static tor_mutex_t *queued_control_events_lock = NULL;
+
+/** An event that should fire in order to flush the contents of
+ * queued_control_events. */
+static mainloop_event_t *flush_queued_events_event = NULL;
+
+void
+control_initialize_event_queue(void)
+{
+  if (queued_control_events == NULL) {
+    queued_control_events = smartlist_new();
+  }
+
+  if (flush_queued_events_event == NULL) {
+    struct event_base *b = tor_libevent_get_base();
+    if (b) {
+      flush_queued_events_event =
+        mainloop_event_new(flush_queued_events_cb, NULL);
+      tor_assert(flush_queued_events_event);
+    }
+  }
+
+  if (queued_control_events_lock == NULL) {
+    queued_control_events_lock = tor_mutex_new();
+    tor_threadlocal_init(&block_event_queue_flag);
+  }
+}
+
+static int *
+get_block_event_queue(void)
+{
+  int *val = tor_threadlocal_get(&block_event_queue_flag);
+  if (PREDICT_UNLIKELY(val == NULL)) {
+    val = tor_malloc_zero(sizeof(int));
+    tor_threadlocal_set(&block_event_queue_flag, val);
+  }
+  return val;
+}
+
+/** Helper: inserts an event on the list of events queued to be sent to
+ * one or more controllers, and schedules the events to be flushed if needed.
+ *
+ * This function takes ownership of <b>msg</b>, and may free it.
+ *
+ * We queue these events rather than send them immediately in order to break
+ * the dependency in our callgraph from code that generates events for the
+ * controller, and the network layer at large.  Otherwise, nearly every
+ * interesting part of Tor would potentially call every other interesting part
+ * of Tor.
+ */
+MOCK_IMPL(STATIC void,
+queue_control_event_string,(uint16_t event, char *msg))
+{
+  /* This is redundant with checks done elsewhere, but it's a last-ditch
+   * attempt to avoid queueing something we shouldn't have to queue. */
+  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
+    tor_free(msg);
+    return;
+  }
+
+  int *block_event_queue = get_block_event_queue();
+  if (*block_event_queue) {
+    tor_free(msg);
+    return;
+  }
+
+  queued_event_t *ev = tor_malloc(sizeof(*ev));
+  ev->event = event;
+  ev->msg = msg;
+
+  /* No queueing an event while queueing an event */
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  tor_assert(queued_control_events);
+  smartlist_add(queued_control_events, ev);
+
+  int activate_event = 0;
+  if (! flush_queued_event_pending && in_main_thread()) {
+    activate_event = 1;
+    flush_queued_event_pending = 1;
+  }
+
+  tor_mutex_release(queued_control_events_lock);
+
+  --*block_event_queue;
+
+  /* We just put an event on the queue; mark the queue to be
+   * flushed.  We only do this from the main thread for now; otherwise,
+   * we'd need to incur locking overhead in Libevent or use a socket.
+   */
+  if (activate_event) {
+    tor_assert(flush_queued_events_event);
+    mainloop_event_activate(flush_queued_events_event);
+  }
+}
+
+#define queued_event_free(ev) \
+  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
+
+/** Release all storage held by <b>ev</b>. */
+static void
+queued_event_free_(queued_event_t *ev)
+{
+  if (ev == NULL)
+    return;
+
+  tor_free(ev->msg);
+  tor_free(ev);
+}
+
+/** Send every queued event to every controller that's interested in it,
+ * and remove the events from the queue.  If <b>force</b> is true,
+ * then make all controllers send their data out immediately, since we
+ * may be about to shut down. */
+static void
+queued_events_flush_all(int force)
+{
+  /* Make sure that we get all the pending log events, if there are any. */
+  flush_pending_log_callbacks();
+
+  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
+    return;
+  }
+  smartlist_t *all_conns = get_connection_array();
+  smartlist_t *controllers = smartlist_new();
+  smartlist_t *queued_events;
+
+  int *block_event_queue = get_block_event_queue();
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  /* No queueing an event while flushing events. */
+  flush_queued_event_pending = 0;
+  queued_events = queued_control_events;
+  queued_control_events = smartlist_new();
+  tor_mutex_release(queued_control_events_lock);
+
+  /* Gather all the controllers that will care... */
+  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_CONTROL &&
+        !conn->marked_for_close &&
+        conn->state == CONTROL_CONN_STATE_OPEN) {
+      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
+
+      smartlist_add(controllers, control_conn);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
+    const event_mask_t bit = ((event_mask_t)1) << ev->event;
+    const size_t msg_len = strlen(ev->msg);
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      if (control_conn->event_mask & bit) {
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
+      }
+    } SMARTLIST_FOREACH_END(control_conn);
+
+    queued_event_free(ev);
+  } SMARTLIST_FOREACH_END(ev);
+
+  if (force) {
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      connection_flush(TO_CONN(control_conn));
+    } SMARTLIST_FOREACH_END(control_conn);
+  }
+
+  smartlist_free(queued_events);
+  smartlist_free(controllers);
+
+  --*block_event_queue;
+}
+
+/** Libevent callback: Flushes pending events to controllers that are
+ * interested in them. */
+static void
+flush_queued_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void) event;
+  (void) arg;
+  queued_events_flush_all(0);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is given by <b>msg</b>.
+ *
+ * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
+ * respect to the EXTENDED_EVENTS feature. */
+MOCK_IMPL(STATIC void,
+send_control_event_string,(uint16_t event,
+                           const char *msg))
+{
+  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
+  queue_control_event_string(event, tor_strdup(msg));
+}
+
+/** Helper for send_control_event and control_event_status:
+ * Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event_impl(uint16_t event,
+                        const char *format, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, format, ap);
+  if (len < 0) {
+    log_warn(LD_BUG, "Unable to format event for controller.");
+    return;
+  }
+
+  queue_control_event_string(event, buf);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event(uint16_t event,
+                   const char *format, ...)
+{
+  va_list ap;
+  va_start(ap, format);
+  send_control_event_impl(event, format, ap);
+  va_end(ap);
+}
+
+/** Something major has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+int
+control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                             int reason_code)
+{
+  const char *status;
+  char reasons[64] = "";
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
+    return 0;
+  tor_assert(circ);
+
+  switch (tp)
+    {
+    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case CIRC_EVENT_BUILT: status = "BUILT"; break;
+    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
+    case CIRC_EVENT_FAILED: status = "FAILED"; break;
+    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
+    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
+    char unk_reason_buf[16];
+    if (!reason_str) {
+      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
+      reason_str = unk_reason_buf;
+    }
+    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
+    } else {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=%s", reason_str);
+    }
+  }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS,
+                                "650 CIRC %lu %s%s%s%s\r\n",
+                                (unsigned long)circ->global_identifier,
+                                status, sp,
+                                circdesc,
+                                reasons);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/** Something minor has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+static int
+control_event_circuit_status_minor(origin_circuit_t *circ,
+                                   circuit_status_minor_event_t e,
+                                   int purpose, const struct timeval *tv)
+{
+  const char *event_desc;
+  char event_tail[160] = "";
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
+    return 0;
+  tor_assert(circ);
+
+  switch (e)
+    {
+    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
+      event_desc = "PURPOSE_CHANGED";
+
+      {
+        /* event_tail can currently be up to 68 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "");
+      }
+
+      break;
+    case CIRC_MINOR_EVENT_CANNIBALIZED:
+      event_desc = "CANNIBALIZED";
+
+      {
+        /* event_tail can currently be up to 130 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        const struct timeval *old_timestamp_began = tv;
+        char tbuf[ISO_TIME_USEC_LEN+1];
+        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
+
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "",
+                     tbuf);
+      }
+
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
+                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
+                       (unsigned long)circ->global_identifier,
+                       event_desc, sp,
+                       circdesc,
+                       event_tail);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
+ * interested controllers.
+ */
+int
+control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                      int old_purpose)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+                                            old_purpose,
+                                            NULL);
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
+ * created-time from <b>old_tv_created</b>: tell any interested controllers.
+ */
+int
+control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                   int old_purpose,
+                                   const struct timeval *old_tv_created)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_CANNIBALIZED,
+                                            old_purpose,
+                                            old_tv_created);
+}
+
+/** Something has happened to the stream associated with AP connection
+ * <b>conn</b>: tell any interested control connections. */
+int
+control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
+                            int reason_code)
+{
+  char reason_buf[64];
+  char addrport_buf[64];
+  const char *status;
+  circuit_t *circ;
+  origin_circuit_t *origin_circ = NULL;
+  char buf[256];
+  const char *purpose = "";
+  tor_assert(conn->socks_request);
+
+  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
+    return 0;
+
+  if (tp == STREAM_EVENT_CLOSED &&
+      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
+    return 0;
+
+  write_stream_target_to_buf(conn, buf, sizeof(buf));
+
+  reason_buf[0] = '\0';
+  switch (tp)
+    {
+    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
+    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
+    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
+    case STREAM_EVENT_FAILED: status = "FAILED"; break;
+    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
+    case STREAM_EVENT_NEW: status = "NEW"; break;
+    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
+    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
+    case STREAM_EVENT_REMAP: status = "REMAP"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (reason_code && (tp == STREAM_EVENT_FAILED ||
+                      tp == STREAM_EVENT_CLOSED ||
+                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
+    const char *reason_str = stream_end_reason_to_control_string(reason_code);
+    char *r = NULL;
+    if (!reason_str) {
+      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
+      reason_str = r;
+    }
+    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=END REMOTE_REASON=%s", reason_str);
+    else
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=%s", reason_str);
+    tor_free(r);
+  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
+    switch (reason_code) {
+    case REMAP_STREAM_SOURCE_CACHE:
+      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
+      break;
+    case REMAP_STREAM_SOURCE_EXIT:
+      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
+      break;
+    default:
+      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
+                   reason_code);
+      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
+      break;
+    }
+  }
+
+  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
+    /*
+     * When the control conn is an AF_UNIX socket and we have no address,
+     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
+     * dnsserv.c.
+     */
+    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
+      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
+                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
+    } else {
+      /*
+       * else leave it blank so control on AF_UNIX doesn't need to make
+       * something up.
+       */
+      addrport_buf[0] = '\0';
+    }
+  } else {
+    addrport_buf[0] = '\0';
+  }
+
+  if (tp == STREAM_EVENT_NEW_RESOLVE) {
+    purpose = " PURPOSE=DNS_REQUEST";
+  } else if (tp == STREAM_EVENT_NEW) {
+    if (conn->use_begindir) {
+      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
+      int linked_dir_purpose = -1;
+      if (linked && linked->type == CONN_TYPE_DIR)
+        linked_dir_purpose = linked->purpose;
+      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
+        purpose = " PURPOSE=DIR_UPLOAD";
+      else
+        purpose = " PURPOSE=DIR_FETCH";
+    } else
+      purpose = " PURPOSE=USER";
+  }
+
+  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+  if (circ && CIRCUIT_IS_ORIGIN(circ))
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+
+  {
+    char *conndesc = entry_connection_describe_status_for_controller(conn);
+    const char *sp = strlen(conndesc) ? " " : "";
+    send_control_event(EVENT_STREAM_STATUS,
+                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s%s%s\r\n",
+                     (ENTRY_TO_CONN(conn)->global_identifier),
+                     status,
+                        origin_circ?
+                           (unsigned long)origin_circ->global_identifier : 0ul,
+                        buf, reason_buf, addrport_buf, purpose, sp, conndesc);
+    tor_free(conndesc);
+  }
+
+  /* XXX need to specify its intended exit, etc? */
+
+  return 0;
+}
+
+/** Called when the status of an OR connection <b>conn</b> changes: tell any
+ * interested control connections. <b>tp</b> is the new status for the
+ * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
+ * may be the reason why.
+ */
+int
+control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
+                             int reason)
+{
+  int ncircs = 0;
+  const char *status;
+  char name[128];
+  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
+
+  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
+    return 0;
+
+  switch (tp)
+    {
+    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
+    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
+    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
+    case OR_CONN_EVENT_NEW: status = "NEW"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (conn->chan) {
+    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
+  } else {
+    ncircs = 0;
+  }
+  ncircs += connection_or_get_num_circuits(conn);
+  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
+    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
+  }
+
+  orconn_target_get_name(name, sizeof(name), conn);
+  send_control_event(EVENT_OR_CONN_STATUS,
+                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
+                              name, status,
+                              reason ? " REASON=" : "",
+                              orconn_end_reason_to_control_string(reason),
+                              ncircs_buf,
+                              (conn->base_.global_identifier));
+
+  return 0;
+}
+
+/**
+ * Print out STREAM_BW event for a single conn
+ */
+int
+control_event_stream_bandwidth(edge_connection_t *edge_conn)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    if (!edge_conn->n_read && !edge_conn->n_written)
+      return 0;
+
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
+    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                       (edge_conn->base_.global_identifier),
+                       (unsigned long)edge_conn->n_read,
+                       (unsigned long)edge_conn->n_written,
+                       tbuf);
+
+    edge_conn->n_written = edge_conn->n_read = 0;
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth streams have used. */
+int
+control_event_stream_bandwidth_used(void)
+{
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    smartlist_t *conns = get_connection_array();
+    edge_connection_t *edge_conn;
+    struct timeval now;
+    char tbuf[ISO_TIME_USEC_LEN+1];
+
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
+    {
+        if (conn->type != CONN_TYPE_AP)
+          continue;
+        edge_conn = TO_EDGE_CONN(conn);
+        if (!edge_conn->n_read && !edge_conn->n_written)
+          continue;
+
+        tor_gettimeofday(&now);
+        format_iso_time_nospace_usec(tbuf, &now);
+        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                           (edge_conn->base_.global_identifier),
+                           (unsigned long)edge_conn->n_read,
+                           (unsigned long)edge_conn->n_written,
+                           tbuf);
+
+        edge_conn->n_written = edge_conn->n_read = 0;
+    }
+    SMARTLIST_FOREACH_END(conn);
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control connections
+ * how much bandwidth origin circuits have used. */
+int
+control_event_circ_bandwidth_used(void)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
+  }
+  SMARTLIST_FOREACH_END(circ);
+
+  return 0;
+}
+
+/**
+ * Emit a CIRC_BW event line for a specific circuit.
+ *
+ * This function sets the values it emits to 0, and does not emit
+ * an event if there is no new data to report since the last call.
+ *
+ * Therefore, it may be called at any frequency.
+ */
+int
+control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+
+  tor_assert(ocirc);
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  /* n_read_circ_bw and n_written_circ_bw are always updated
+   * when there is any new cell on a circuit, and set to 0 after
+   * the event, below.
+   *
+   * Therefore, checking them is sufficient to determine if there
+   * is new data to report. */
+  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
+    return 0;
+
+  tor_gettimeofday(&now);
+  format_iso_time_nospace_usec(tbuf, &now);
+  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
+                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
+                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
+                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
+                     ocirc->global_identifier,
+                     (unsigned long)ocirc->n_read_circ_bw,
+                     (unsigned long)ocirc->n_written_circ_bw,
+                     tbuf,
+                     (unsigned long)ocirc->n_delivered_read_circ_bw,
+                     (unsigned long)ocirc->n_overhead_read_circ_bw,
+                     (unsigned long)ocirc->n_delivered_written_circ_bw,
+                     (unsigned long)ocirc->n_overhead_written_circ_bw);
+  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+
+  return 0;
+}
+
+/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
+  * bandwidth counters. */
+int
+control_event_conn_bandwidth(connection_t *conn)
+{
+  const char *conn_type_str;
+  if (!get_options()->TestingEnableConnBwEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
+    return 0;
+  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
+    return 0;
+  switch (conn->type) {
+    case CONN_TYPE_OR:
+      conn_type_str = "OR";
+      break;
+    case CONN_TYPE_DIR:
+      conn_type_str = "DIR";
+      break;
+    case CONN_TYPE_EXIT:
+      conn_type_str = "EXIT";
+      break;
+    default:
+      return 0;
+  }
+  send_control_event(EVENT_CONN_BW,
+                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
+                     "READ=%lu WRITTEN=%lu\r\n",
+                     (conn->global_identifier),
+                     conn_type_str,
+                     (unsigned long)conn->n_read_conn_bw,
+                     (unsigned long)conn->n_written_conn_bw);
+  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth connections have used. */
+int
+control_event_conn_bandwidth_used(void)
+{
+  if (get_options()->TestingEnableConnBwEvent &&
+      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
+    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
+                      control_event_conn_bandwidth(conn));
+  }
+  return 0;
+}
+
+/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
+ * cells, removed cells, and waiting times by cell command and direction.
+ * Store results in <b>cell_stats</b>.  Free cell statistics of the
+ * circuit afterwards. */
+void
+sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
+{
+  memset(cell_stats, 0, sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
+                          const testing_cell_stats_entry_t *, ent) {
+    tor_assert(ent->command <= CELL_COMMAND_MAX_);
+    if (!ent->removed && !ent->exitward) {
+      cell_stats->added_cells_appward[ent->command] += 1;
+    } else if (!ent->removed && ent->exitward) {
+      cell_stats->added_cells_exitward[ent->command] += 1;
+    } else if (!ent->exitward) {
+      cell_stats->removed_cells_appward[ent->command] += 1;
+      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
+    } else {
+      cell_stats->removed_cells_exitward[ent->command] += 1;
+      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+  circuit_clear_testing_cell_stats(circ);
+}
+
+/** Helper: append a cell statistics string to <code>event_parts</code>,
+ * prefixed with <code>key</code>=.  Statistics consist of comma-separated
+ * key:value pairs with lower-case command strings as keys and cell
+ * numbers or total waiting times as values.  A key:value pair is included
+ * if the entry in <code>include_if_non_zero</code> is not zero, but with
+ * the (possibly zero) entry from <code>number_to_include</code>.  Both
+ * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
+ * entry in <code>include_if_non_zero</code> is positive, no string will
+ * be added to <code>event_parts</code>. */
+void
+append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
+                             const uint64_t *include_if_non_zero,
+                             const uint64_t *number_to_include)
+{
+  smartlist_t *key_value_strings = smartlist_new();
+  int i;
+  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
+    if (include_if_non_zero[i] > 0) {
+      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
+                             cell_command_to_string(i),
+                             (number_to_include[i]));
+    }
+  }
+  if (smartlist_len(key_value_strings) > 0) {
+    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
+    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
+    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
+    tor_free(joined);
+  }
+  smartlist_free(key_value_strings);
+}
+
+/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
+ * CELL_STATS event and write result string to <b>event_string</b>. */
+void
+format_cell_stats(char **event_string, circuit_t *circ,
+                  cell_stats_t *cell_stats)
+{
+  smartlist_t *event_parts = smartlist_new();
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "ID=%lu",
+                 (unsigned long)ocirc->global_identifier);
+  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
+                 (unsigned long)or_circ->p_circ_id);
+    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
+                 (or_circ->p_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "InboundAdded",
+                                 cell_stats->added_cells_appward,
+                                 cell_stats->added_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundRemoved",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->removed_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundTime",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->total_time_appward);
+  }
+  if (circ->n_chan) {
+    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
+                     (unsigned long)circ->n_circ_id);
+    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
+                 (circ->n_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "OutboundAdded",
+                                 cell_stats->added_cells_exitward,
+                                 cell_stats->added_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundRemoved",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->removed_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundTime",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->total_time_exitward);
+  }
+  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
+  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
+  smartlist_free(event_parts);
+}
+
+/** A second or more has elapsed: tell any interested control connection
+ * how many cells have been processed for a given circuit. */
+int
+control_event_circuit_cell_stats(void)
+{
+  cell_stats_t *cell_stats;
+  char *event_string;
+  if (!get_options()->TestingEnableCellStatsEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
+    return 0;
+  cell_stats = tor_malloc(sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!circ->testing_cell_stats)
+      continue;
+    sum_up_cell_stats_by_command(circ, cell_stats);
+    format_cell_stats(&event_string, circ, cell_stats);
+    send_control_event(EVENT_CELL_STATS,
+                       "650 CELL_STATS %s\r\n", event_string);
+    tor_free(event_string);
+  }
+  SMARTLIST_FOREACH_END(circ);
+  tor_free(cell_stats);
+  return 0;
+}
+
+/* about 5 minutes worth. */
+#define N_BW_EVENTS_TO_CACHE 300
+/* Index into cached_bw_events to next write. */
+static int next_measurement_idx = 0;
+/* number of entries set in n_measurements */
+static int n_measurements = 0;
+static struct cached_bw_event_t {
+  uint32_t n_read;
+  uint32_t n_written;
+} cached_bw_events[N_BW_EVENTS_TO_CACHE];
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth we used. */
+int
+control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
+{
+  cached_bw_events[next_measurement_idx].n_read = n_read;
+  cached_bw_events[next_measurement_idx].n_written = n_written;
+  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
+    next_measurement_idx = 0;
+  if (n_measurements < N_BW_EVENTS_TO_CACHE)
+    ++n_measurements;
+
+  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
+    send_control_event(EVENT_BANDWIDTH_USED,
+                       "650 BW %lu %lu\r\n",
+                       (unsigned long)n_read,
+                       (unsigned long)n_written);
+  }
+
+  return 0;
+}
+
+char *
+get_bw_samples(void)
+{
+  int i;
+  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
+    % N_BW_EVENTS_TO_CACHE;
+  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+
+  smartlist_t *elements = smartlist_new();
+
+  for (i = 0; i < n_measurements; ++i) {
+    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+    const struct cached_bw_event_t *bwe = &cached_bw_events[idx];
+
+    smartlist_add_asprintf(elements, "%u,%u",
+                           (unsigned)bwe->n_read,
+                           (unsigned)bwe->n_written);
+
+    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/** Called when we are sending a log message to the controllers: suspend
+ * sending further log messages to the controllers until we're done.  Used by
+ * CONN_LOG_PROTECT. */
+void
+disable_control_logging(void)
+{
+  ++disable_log_messages;
+}
+
+/** We're done sending a log message to the controllers: re-enable controller
+ * logging.  Used by CONN_LOG_PROTECT. */
+void
+enable_control_logging(void)
+{
+  if (--disable_log_messages < 0)
+    tor_assert(0);
+}
+
+/** We got a log message: tell any interested control connections. */
+void
+control_event_logmsg(int severity, log_domain_mask_t domain, const char *msg)
+{
+  int event;
+
+  /* Don't even think of trying to add stuff to a buffer from a cpuworker
+   * thread. (See #25987 for plan to fix.) */
+  if (! in_main_thread())
+    return;
+
+  if (disable_log_messages)
+    return;
+
+  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
+      severity <= LOG_NOTICE) {
+    char *esc = esc_for_log(msg);
+    ++disable_log_messages;
+    control_event_general_status(severity, "BUG REASON=%s", esc);
+    --disable_log_messages;
+    tor_free(esc);
+  }
+
+  event = log_severity_to_event(severity);
+  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
+    char *b = NULL;
+    const char *s;
+    if (strchr(msg, '\n')) {
+      char *cp;
+      b = tor_strdup(msg);
+      for (cp = b; *cp; ++cp)
+        if (*cp == '\r' || *cp == '\n')
+          *cp = ' ';
+    }
+    switch (severity) {
+      case LOG_DEBUG: s = "DEBUG"; break;
+      case LOG_INFO: s = "INFO"; break;
+      case LOG_NOTICE: s = "NOTICE"; break;
+      case LOG_WARN: s = "WARN"; break;
+      case LOG_ERR: s = "ERR"; break;
+      default: s = "UnknownLogSeverity"; break;
+    }
+    ++disable_log_messages;
+    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
+    if (severity == LOG_ERR) {
+      /* Force a flush, since we may be about to die horribly */
+      queued_events_flush_all(1);
+    }
+    --disable_log_messages;
+    tor_free(b);
+  }
+}
+
+/**
+ * Logging callback: called when there is a queued pending log callback.
+ */
+void
+control_event_logmsg_pending(void)
+{
+  if (! in_main_thread()) {
+    /* We can't handle this case yet, since we're using a
+     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
+     * use a different mechanism instead: see #25987.
+     **/
+    return;
+  }
+  tor_assert(flush_queued_events_event);
+  mainloop_event_activate(flush_queued_events_event);
+}
+
+/** Called whenever we receive new router descriptors: tell any
+ * interested control connections.  <b>routers</b> is a list of
+ * routerinfo_t's.
+ */
+int
+control_event_descriptors_changed(smartlist_t *routers)
+{
+  char *msg;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
+    return 0;
+
+  {
+    smartlist_t *names = smartlist_new();
+    char *ids;
+    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
+        router_get_verbose_nickname(b, ri);
+        smartlist_add(names, b);
+      });
+    ids = smartlist_join_strings(names, " ", 0, NULL);
+    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
+    send_control_event_string(EVENT_NEW_DESC,  msg);
+    tor_free(ids);
+    tor_free(msg);
+    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
+    smartlist_free(names);
+  }
+  return 0;
+}
+
+/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
+ * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
+ * <b>error</b> is non-NULL, it is an error code describing the failure
+ * mode of the mapping.
+ */
+int
+control_event_address_mapped(const char *from, const char *to, time_t expires,
+                             const char *error, const int cached)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
+    return 0;
+
+  if (expires < 3 || expires == TIME_MAX)
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s NEVER %s%s"
+                                "CACHED=\"%s\"\r\n",
+                                  from, to, error?error:"", error?" ":"",
+                                cached?"YES":"NO");
+  else {
+    char buf[ISO_TIME_LEN+1];
+    char buf2[ISO_TIME_LEN+1];
+    format_local_iso_time(buf,expires);
+    format_iso_time(buf2,expires);
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s \"%s\""
+                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
+                                from, to, buf,
+                                error?error:"", error?" ":"",
+                                buf2, cached?"YES":"NO");
+  }
+
+  return 0;
+}
+/** The network liveness has changed; this is called from circuitstats.c
+ * whenever we receive a cell, or when timeout expires and we assume the
+ * network is down. */
+int
+control_event_network_liveness_update(int liveness)
+{
+  if (liveness > 0) {
+    if (get_cached_network_liveness() <= 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(1);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS UP\r\n");
+    }
+    /* else was already live, no-op */
+  } else {
+    if (get_cached_network_liveness() > 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(0);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS DOWN\r\n");
+    }
+    /* else was already dead, no-op */
+  }
+
+  return 0;
+}
+
+/** Helper function for NS-style events. Constructs and sends an event
+ * of type <b>event</b> with string <b>event_string</b> out of the set of
+ * networkstatuses <b>statuses</b>. Currently it is used for NS events
+ * and NEWCONSENSUS events. */
+static int
+control_event_networkstatus_changed_helper(smartlist_t *statuses,
+                                           uint16_t event,
+                                           const char *event_string)
+{
+  smartlist_t *strs;
+  char *s, *esc = NULL;
+  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
+    return 0;
+
+  strs = smartlist_new();
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
+  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
+    {
+      s = networkstatus_getinfo_helper_single(rs);
+      if (!s) continue;
+      smartlist_add(strs, s);
+    });
+
+  s = smartlist_join_strings(strs, "", 0, NULL);
+  write_escaped_data(s, strlen(s), &esc);
+  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
+  smartlist_free(strs);
+  tor_free(s);
+  send_control_event_string(event,  esc);
+  send_control_event_string(event,
+                            "650 OK\r\n");
+
+  tor_free(esc);
+  return 0;
+}
+
+/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
+ * an NS event to any controller that cares. */
+int
+control_event_networkstatus_changed(smartlist_t *statuses)
+{
+  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
+}
+
+/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
+ * event consisting of an NS-style line for each relay in the consensus. */
+int
+control_event_newconsensus(const networkstatus_t *consensus)
+{
+  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
+    return 0;
+  return control_event_networkstatus_changed_helper(
+           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
+}
+
+/** Called when we compute a new circuitbuildtimeout */
+int
+control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                               const char *args)
+{
+  const char *type_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
+    return 0;
+
+  switch (type) {
+    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
+      type_string = "COMPUTED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESET:
+      type_string = "RESET";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
+      type_string = "SUSPENDED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_DISCARD:
+      type_string = "DISCARD";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESUME:
+      type_string = "RESUME";
+      break;
+    default:
+      type_string = "UNKNOWN";
+      break;
+  }
+
+  send_control_event(EVENT_BUILDTIMEOUT_SET,
+                     "650 BUILDTIMEOUT_SET %s %s\r\n",
+                     type_string, args);
+
+  return 0;
+}
+
+/** Called when a signal has been processed from signal_callback */
+int
+control_event_signal(uintptr_t signal_num)
+{
+  const char *signal_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
+    return 0;
+
+  for (unsigned i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if ((int)signal_num == signal_table[i].sig) {
+      signal_string = signal_table[i].signal_name;
+      break;
+    }
+  }
+
+  if (signal_string == NULL) {
+    log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
+             (unsigned long)signal_num);
+    return -1;
+  }
+
+  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
+                     signal_string);
+  return 0;
+}
+
+/** Called when a single local_routerstatus_t has changed: Sends an NS event
+ * to any controller that cares. */
+int
+control_event_networkstatus_changed_single(const routerstatus_t *rs)
+{
+  smartlist_t *statuses;
+  int r;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NS))
+    return 0;
+
+  statuses = smartlist_new();
+  smartlist_add(statuses, (void*)rs);
+  r = control_event_networkstatus_changed(statuses);
+  smartlist_free(statuses);
+  return r;
+}
+
+/** Our own router descriptor has changed; tell any controllers that care.
+ */
+int
+control_event_my_descriptor_changed(void)
+{
+  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
+  return 0;
+}
+
+/** Helper: sends a status event where <b>type</b> is one of
+ * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
+ * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
+ * string corresponding to <b>args</b>. */
+static int
+control_event_status(int type, int severity, const char *format, va_list args)
+{
+  char *user_buf = NULL;
+  char format_buf[160];
+  const char *status, *sev;
+
+  switch (type) {
+    case EVENT_STATUS_GENERAL:
+      status = "STATUS_GENERAL";
+      break;
+    case EVENT_STATUS_CLIENT:
+      status = "STATUS_CLIENT";
+      break;
+    case EVENT_STATUS_SERVER:
+      status = "STATUS_SERVER";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status type %d", type);
+      return -1;
+  }
+  switch (severity) {
+    case LOG_NOTICE:
+      sev = "NOTICE";
+      break;
+    case LOG_WARN:
+      sev = "WARN";
+      break;
+    case LOG_ERR:
+      sev = "ERR";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
+      return -1;
+  }
+  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
+                   status, sev)<0) {
+    log_warn(LD_BUG, "Format string too long.");
+    return -1;
+  }
+  if (tor_vasprintf(&user_buf, format, args)<0) {
+    log_warn(LD_BUG, "Failed to create user buffer.");
+    return -1;
+  }
+
+  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
+  tor_free(user_buf);
+  return 0;
+}
+
+#ifndef COCCI
+#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
+  int r;                                                        \
+  do {                                                          \
+    va_list ap;                                                 \
+    if (!EVENT_IS_INTERESTING(event))                           \
+      return 0;                                                 \
+                                                                \
+    va_start(ap, format);                                       \
+    r = control_event_status((event), (sev), format, ap);       \
+    va_end(ap);                                                 \
+  } while (0)
+#endif /* !defined(COCCI) */
+
+/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_general_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_general_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_client_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_client_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_server_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_server_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Called when the status of an entry guard with the given <b>nickname</b>
+ * and identity <b>digest</b> has changed to <b>status</b>: tells any
+ * controllers that care. */
+int
+control_event_guard(const char *nickname, const char *digest,
+                    const char *status)
+{
+  char hbuf[HEX_DIGEST_LEN+1];
+  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
+  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
+    return 0;
+
+  {
+    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
+    const node_t *node = node_get_by_id(digest);
+    if (node) {
+      node_get_verbose_nickname(node, buf);
+    } else {
+      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
+    }
+    send_control_event(EVENT_GUARD,
+                       "650 GUARD ENTRY %s %s\r\n", buf, status);
+  }
+  return 0;
+}
+
+/** Called when a configuration option changes. This is generally triggered
+ * by SETCONF requests and RELOAD/SIGHUP signals. The <b>changes</b> are
+ * a linked list of configuration key-values.
+ * <b>changes</b> can be NULL, meaning "no changes".
+ */
+void
+control_event_conf_changed(const config_line_t *changes)
+{
+  char *result;
+  smartlist_t *lines;
+  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) || !changes) {
+    return;
+  }
+  lines = smartlist_new();
+  for (const config_line_t *line = changes; line; line = line->next) {
+    if (line->value == NULL) {
+      smartlist_add_asprintf(lines, "650-%s", line->key);
+    } else {
+      smartlist_add_asprintf(lines, "650-%s=%s", line->key, line->value);
+    }
+  }
+  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
+  send_control_event(EVENT_CONF_CHANGED,
+    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
+  tor_free(result);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+}
+
+/** We just generated a new summary of which countries we've seen clients
+ * from recently. Send a copy to the controller in case it wants to
+ * display it for the user. */
+void
+control_event_clients_seen(const char *controller_str)
+{
+  send_control_event(EVENT_CLIENTS_SEEN,
+    "650 CLIENTS_SEEN %s\r\n", controller_str);
+}
+
+/** A new pluggable transport called <b>transport_name</b> was
+ *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
+ *  "server" or "client" depending on the mode of the pluggable
+ *  transport.
+ *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
+ */
+void
+control_event_transport_launched(const char *mode, const char *transport_name,
+                                 tor_addr_t *addr, uint16_t port)
+{
+  send_control_event(EVENT_TRANSPORT_LAUNCHED,
+                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
+                     mode, transport_name, fmt_addr(addr), port);
+}
+
+/** A pluggable transport called <b>pt_name</b> has emitted a log message
+ * found in <b>message</b> at <b>severity</b> log level. */
+void
+control_event_pt_log(const char *log)
+{
+  send_control_event(EVENT_PT_LOG,
+                     "650 PT_LOG %s\r\n",
+                     log);
+}
+
+/** A pluggable transport has emitted a STATUS message found in
+ * <b>status</b>. */
+void
+control_event_pt_status(const char *status)
+{
+  send_control_event(EVENT_PT_STATUS,
+                     "650 PT_STATUS %s\r\n",
+                     status);
+}
+
+/** Convert rendezvous auth type to string for HS_DESC control events
+ */
+const char *
+rend_auth_type_to_string(rend_auth_type_t auth_type)
+{
+  const char *str;
+
+  switch (auth_type) {
+    case REND_NO_AUTH:
+      str = "NO_AUTH";
+      break;
+    case REND_BASIC_AUTH:
+      str = "BASIC_AUTH";
+      break;
+    case REND_STEALTH_AUTH:
+      str = "STEALTH_AUTH";
+      break;
+    default:
+      str = "UNKNOWN";
+  }
+
+  return str;
+}
+
+/** Return either the onion address if the given pointer is a non empty
+ * string else the unknown string. */
+static const char *
+rend_hsaddress_str_or_unknown(const char *onion_address)
+{
+  static const char *str_unknown = "UNKNOWN";
+  const char *str_ret = str_unknown;
+
+  /* No valid pointer, unknown it is. */
+  if (!onion_address) {
+    goto end;
+  }
+  /* Empty onion address thus we don't know, unknown it is. */
+  if (onion_address[0] == '\0') {
+    goto end;
+  }
+  /* All checks are good so return the given onion address. */
+  str_ret = onion_address;
+
+ end:
+  return str_ret;
+}
+
+/** send HS_DESC requested event.
+ *
+ * <b>rend_query</b> is used to fetch requested onion address and auth type.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
+ */
+void
+control_event_hs_descriptor_requested(const char *onion_address,
+                                      rend_auth_type_t auth_type,
+                                      const char *id_digest,
+                                      const char *desc_id,
+                                      const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC CREATED event when a local service generates a descriptor.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>desc_id</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number. If it is negative, it
+ * is ignored.
+ */
+void
+control_event_hs_descriptor_created(const char *onion_address,
+                                    const char *desc_id,
+                                    int replica)
+{
+  char *replica_field = NULL;
+
+  if (BUG(!onion_address || !desc_id)) {
+    return;
+  }
+
+  if (replica >= 0) {
+    tor_asprintf(&replica_field, " REPLICA=%d", replica);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
+                     onion_address, desc_id,
+                     replica_field ? replica_field : "");
+  tor_free(replica_field);
+}
+
+/** send HS_DESC upload event.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id</b> is the ID of requested hs descriptor.
+ */
+void
+control_event_hs_descriptor_upload(const char *onion_address,
+                                   const char *id_digest,
+                                   const char *desc_id,
+                                   const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!onion_address || !id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
+                     onion_address,
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hsv2_descriptor_received
+ * control_event_hsv2_descriptor_failed
+ * control_event_hsv3_descriptor_failed
+ *
+ * So do not call this function directly.
+ */
+static void
+event_hs_descriptor_receive_end(const char *action,
+                                const char *onion_address,
+                                const char *desc_id,
+                                rend_auth_type_t auth_type,
+                                const char *hsdir_id_digest,
+                                const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !onion_address)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s %s %s%s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     desc_id ? desc_id : "",
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hs_descriptor_uploaded
+ * control_event_hs_descriptor_upload_failed
+ *
+ * So do not call this function directly.
+ */
+void
+control_event_hs_descriptor_upload_end(const char *action,
+                                       const char *onion_address,
+                                       const char *id_digest,
+                                       const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !id_digest)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     node_describe_longname_by_id(id_digest),
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** For an HS descriptor query <b>rend_data</b>, using the
+ * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
+ * which descriptor ID in the query is the right one.
+ *
+ * Return a pointer of the binary descriptor ID found in the query's object
+ * or NULL if not found. */
+static const char *
+get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
+{
+  int replica;
+  const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
+
+  /* Possible if the fetch was done using a descriptor ID. This means that
+   * the HSFETCH command was used. */
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
+    goto end;
+  }
+
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
+  /* OK, we have an onion address so now let's find which descriptor ID
+   * is the one associated with the HSDir fingerprint. */
+  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
+       replica++) {
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
+
+    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
+      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
+        /* Found it! This descriptor ID is the right one. */
+        desc_id = digest;
+        goto end;
+      }
+    } SMARTLIST_FOREACH_END(fingerprint);
+  }
+
+ end:
+  return desc_id;
+}
+
+/** send HS_DESC RECEIVED event
+ *
+ * called when we successfully received a hidden service descriptor.
+ */
+void
+control_event_hsv2_descriptor_received(const char *onion_address,
+                                       const rend_data_t *rend_data,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/* Send HS_DESC RECEIVED event
+ *
+ * Called when we successfully received a hidden service descriptor. */
+void
+control_event_hsv3_descriptor_received(const char *onion_address,
+                                       const char *desc_id,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/** send HS_DESC UPLOADED event
+ *
+ * called when we successfully uploaded a hidden service descriptor.
+ */
+void
+control_event_hs_descriptor_uploaded(const char *id_digest,
+                                     const char *onion_address)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
+                                         id_digest, NULL);
+}
+
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
+ */
+void
+control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
+                                  desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC event to inform controller that the query to
+ * <b>onion_address</b> failed to retrieve hidden service descriptor
+ * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
+ * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
+ * field. */
+void
+control_event_hsv3_descriptor_failed(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !reason)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC_CONTENT event after completion of a successful fetch
+ * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
+ * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
+ * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
+ * not trigger the control event. */
+void
+control_event_hs_descriptor_content(const char *onion_address,
+                                    const char *desc_id,
+                                    const char *hsdir_id_digest,
+                                    const char *content)
+{
+  static const char *event_name = "HS_DESC_CONTENT";
+  char *esc_content = NULL;
+
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
+    return;
+  }
+
+  if (content == NULL) {
+    /* Point it to empty content so it can still be escaped. */
+    content = "";
+  }
+  write_escaped_data(content, strlen(content), &esc_content);
+
+  send_control_event(EVENT_HS_DESC_CONTENT,
+                     "650+%s %s %s %s\r\n%s650 OK\r\n",
+                     event_name,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     desc_id,
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     esc_content);
+  tor_free(esc_content);
+}
+
+/** Send HS_DESC event to inform controller upload of hidden service
+ * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
+ * is not NULL, add it to REASON= field.
+ */
+void
+control_event_hs_descriptor_upload_failed(const char *id_digest,
+                                          const char *onion_address,
+                                          const char *reason)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+  control_event_hs_descriptor_upload_end("FAILED", onion_address,
+                                         id_digest, reason);
+}
+
+void
+control_events_free_all(void)
+{
+  smartlist_t *queued_events = NULL;
+
+  stats_prev_n_read = stats_prev_n_written = 0;
+
+  if (queued_control_events_lock) {
+    tor_mutex_acquire(queued_control_events_lock);
+    flush_queued_event_pending = 0;
+    queued_events = queued_control_events;
+    queued_control_events = NULL;
+    tor_mutex_release(queued_control_events_lock);
+  }
+  if (queued_events) {
+    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
+                      queued_event_free(ev));
+    smartlist_free(queued_events);
+  }
+  if (flush_queued_events_event) {
+    mainloop_event_free(flush_queued_events_event);
+    flush_queued_events_event = NULL;
+  }
+  global_event_mask = 0;
+  disable_log_messages = 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* For testing: change the value of global_event_mask */
+void
+control_testing_set_global_event_mask(uint64_t mask)
+{
+  global_event_mask = mask;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control_events.h b/src/feature/control/control_events.h
new file mode 100644
index 0000000000..4a5492b510
--- /dev/null
+++ b/src/feature/control/control_events.h
@@ -0,0 +1,352 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.h
+ * \brief Header file for control_events.c.
+ **/
+
+#ifndef TOR_CONTROL_EVENTS_H
+#define TOR_CONTROL_EVENTS_H
+
+#include "lib/cc/ctassert.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+
+struct config_line_t;
+
+/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
+ * The various types are defined in control-spec.txt . */
+typedef enum circuit_status_minor_event_t {
+  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+  CIRC_MINOR_EVENT_CANNIBALIZED,
+} circuit_status_minor_event_t;
+
+/** Used to indicate the type of a stream event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum stream_status_event_t {
+  STREAM_EVENT_SENT_CONNECT = 0,
+  STREAM_EVENT_SENT_RESOLVE = 1,
+  STREAM_EVENT_SUCCEEDED    = 2,
+  STREAM_EVENT_FAILED       = 3,
+  STREAM_EVENT_CLOSED       = 4,
+  STREAM_EVENT_NEW          = 5,
+  STREAM_EVENT_NEW_RESOLVE  = 6,
+  STREAM_EVENT_FAILED_RETRIABLE = 7,
+  STREAM_EVENT_REMAP        = 8
+} stream_status_event_t;
+
+/** Used to indicate the type of a buildtime event */
+typedef enum buildtimeout_set_event_t {
+  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
+  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
+  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
+  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
+  BUILDTIMEOUT_SET_EVENT_RESUME = 4
+} buildtimeout_set_event_t;
+
+/** Enum describing various stages of bootstrapping, for use with controller
+ * bootstrap status events. The values range from 0 to 100. */
+typedef enum {
+  BOOTSTRAP_STATUS_UNDEF=-1,
+  BOOTSTRAP_STATUS_STARTING=0,
+
+  /* Initial connection to any relay */
+
+  BOOTSTRAP_STATUS_CONN_PT=1,
+  BOOTSTRAP_STATUS_CONN_DONE_PT=2,
+  BOOTSTRAP_STATUS_CONN_PROXY=3,
+  BOOTSTRAP_STATUS_CONN_DONE_PROXY=4,
+  BOOTSTRAP_STATUS_CONN=5,
+  BOOTSTRAP_STATUS_CONN_DONE=10,
+  BOOTSTRAP_STATUS_HANDSHAKE=14,
+  BOOTSTRAP_STATUS_HANDSHAKE_DONE=15,
+
+  /* Loading directory info */
+
+  BOOTSTRAP_STATUS_ONEHOP_CREATE=20,
+  BOOTSTRAP_STATUS_REQUESTING_STATUS=25,
+  BOOTSTRAP_STATUS_LOADING_STATUS=30,
+  BOOTSTRAP_STATUS_LOADING_KEYS=40,
+  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
+  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
+  BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75,
+
+  /* Connecting to a relay for AP circuits */
+
+  BOOTSTRAP_STATUS_AP_CONN_PT=76,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77,
+  BOOTSTRAP_STATUS_AP_CONN_PROXY=78,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79,
+  BOOTSTRAP_STATUS_AP_CONN=80,
+  BOOTSTRAP_STATUS_AP_CONN_DONE=85,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE=89,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90,
+
+  /* Creating AP circuits */
+
+  BOOTSTRAP_STATUS_CIRCUIT_CREATE=95,
+  BOOTSTRAP_STATUS_DONE=100
+} bootstrap_status_t;
+
+/** Reason for remapping an AP connection's address: we have a cached
+ * answer. */
+#define REMAP_STREAM_SOURCE_CACHE 1
+/** Reason for remapping an AP connection's address: the exit node told us an
+ * answer. */
+#define REMAP_STREAM_SOURCE_EXIT 2
+
+void control_initialize_event_queue(void);
+
+void control_update_global_event_mask(void);
+void control_adjust_event_log_severity(void);
+
+#define EVENT_NS 0x000F
+int control_event_is_interesting(int event);
+
+void control_per_second_events(void);
+int control_any_per_second_event_enabled(void);
+
+int control_event_circuit_status(origin_circuit_t *circ,
+                                 circuit_status_event_t e, int reason);
+int control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                          int old_purpose);
+int control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                       int old_purpose,
+                                       const struct timeval *old_tv_created);
+int control_event_stream_status(entry_connection_t *conn,
+                                stream_status_event_t e,
+                                int reason);
+int control_event_or_conn_status(or_connection_t *conn,
+                                 or_conn_status_event_t e, int reason);
+int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
+int control_event_stream_bandwidth(edge_connection_t *edge_conn);
+int control_event_stream_bandwidth_used(void);
+int control_event_circ_bandwidth_used(void);
+int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
+int control_event_conn_bandwidth(connection_t *conn);
+int control_event_conn_bandwidth_used(void);
+int control_event_circuit_cell_stats(void);
+void control_event_logmsg(int severity, log_domain_mask_t domain,
+                          const char *msg);
+void control_event_logmsg_pending(void);
+int control_event_descriptors_changed(smartlist_t *routers);
+int control_event_address_mapped(const char *from, const char *to,
+                                 time_t expires, const char *error,
+                                 const int cached);
+int control_event_my_descriptor_changed(void);
+int control_event_network_liveness_update(int liveness);
+int control_event_networkstatus_changed(smartlist_t *statuses);
+
+int control_event_newconsensus(const networkstatus_t *consensus);
+int control_event_networkstatus_changed_single(const routerstatus_t *rs);
+int control_event_general_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_client_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_server_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+int control_event_general_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_client_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_server_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+
+int control_event_guard(const char *nickname, const char *digest,
+                        const char *status);
+void control_event_conf_changed(const struct config_line_t *changes);
+int control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                                   const char *args);
+int control_event_signal(uintptr_t signal);
+
+void control_event_bootstrap(bootstrap_status_t status, int progress);
+int control_get_bootstrap_percent(void);
+MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
+                                                 int reason,
+                                                 or_connection_t *or_conn));
+void control_event_boot_dir(bootstrap_status_t status, int progress);
+void control_event_boot_first_orconn(void);
+void control_event_bootstrap_problem(const char *warn, const char *reason,
+                                     const connection_t *conn, int dowarn);
+char *control_event_boot_last_msg(void);
+void control_event_bootstrap_reset(void);
+
+void control_event_clients_seen(const char *controller_str);
+void control_event_transport_launched(const char *mode,
+                                      const char *transport_name,
+                                      tor_addr_t *addr, uint16_t port);
+void control_event_pt_log(const char *log);
+void control_event_pt_status(const char *status);
+
+void control_event_hs_descriptor_requested(const char *onion_address,
+                                           rend_auth_type_t auth_type,
+                                           const char *id_digest,
+                                           const char *desc_id,
+                                           const char *hsdir_index);
+void control_event_hs_descriptor_created(const char *onion_address,
+                                         const char *desc_id,
+                                         int replica);
+void control_event_hs_descriptor_upload(const char *onion_address,
+                                        const char *desc_id,
+                                        const char *hs_dir,
+                                        const char *hsdir_index);
+void control_event_hs_descriptor_upload_end(const char *action,
+                                            const char *onion_address,
+                                            const char *hs_dir,
+                                            const char *reason);
+void control_event_hs_descriptor_uploaded(const char *hs_dir,
+                                          const char *onion_address);
+/* Hidden service v2 HS_DESC specific. */
+void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                          const char *id_digest,
+                                          const char *reason);
+void control_event_hsv2_descriptor_received(const char *onion_address,
+                                            const rend_data_t *rend_data,
+                                            const char *id_digest);
+/* Hidden service v3 HS_DESC specific. */
+void control_event_hsv3_descriptor_failed(const char *onion_address,
+                                          const char *desc_id,
+                                          const char *hsdir_id_digest,
+                                          const char *reason);
+void control_event_hsv3_descriptor_received(const char *onion_address,
+                                            const char *desc_id,
+                                            const char *hsdir_id_digest);
+void control_event_hs_descriptor_upload_failed(const char *hs_dir,
+                                               const char *onion_address,
+                                               const char *reason);
+void control_event_hs_descriptor_content(const char *onion_address,
+                                         const char *desc_id,
+                                         const char *hsdir_fp,
+                                         const char *content);
+
+void control_events_free_all(void);
+
+#ifdef CONTROL_MODULE_PRIVATE
+char *get_bw_samples(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#ifdef CONTROL_EVENTS_PRIVATE
+/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
+ * connection is interested in events of type <b>e</b>.  We use this
+ * so that we can decide to skip generating event messages that nobody
+ * has interest in without having to walk over the global connection
+ * list to find out.
+ **/
+typedef uint64_t event_mask_t;
+
+/* Recognized asynchronous event types.  It's okay to expand this list
+ * because it is used both as a list of v0 event types, and as indices
+ * into the bitfield to determine which controllers want which events.
+ */
+/* This bitfield has no event zero    0x0000 */
+#define EVENT_MIN_                    0x0001
+#define EVENT_CIRCUIT_STATUS          0x0001
+#define EVENT_STREAM_STATUS           0x0002
+#define EVENT_OR_CONN_STATUS          0x0003
+#define EVENT_BANDWIDTH_USED          0x0004
+#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
+#define EVENT_NEW_DESC                0x0006
+#define EVENT_DEBUG_MSG               0x0007
+#define EVENT_INFO_MSG                0x0008
+#define EVENT_NOTICE_MSG              0x0009
+#define EVENT_WARN_MSG                0x000A
+#define EVENT_ERR_MSG                 0x000B
+#define EVENT_ADDRMAP                 0x000C
+/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
+   can reclaim 0x000D. */
+#define EVENT_DESCCHANGED             0x000E
+/* Exposed above */
+// #define EVENT_NS                   0x000F
+#define EVENT_STATUS_CLIENT           0x0010
+#define EVENT_STATUS_SERVER           0x0011
+#define EVENT_STATUS_GENERAL          0x0012
+#define EVENT_GUARD                   0x0013
+#define EVENT_STREAM_BANDWIDTH_USED   0x0014
+#define EVENT_CLIENTS_SEEN            0x0015
+#define EVENT_NEWCONSENSUS            0x0016
+#define EVENT_BUILDTIMEOUT_SET        0x0017
+#define EVENT_GOT_SIGNAL              0x0018
+#define EVENT_CONF_CHANGED            0x0019
+#define EVENT_CONN_BW                 0x001A
+#define EVENT_CELL_STATS              0x001B
+/* UNUSED :                           0x001C */
+#define EVENT_CIRC_BANDWIDTH_USED     0x001D
+#define EVENT_TRANSPORT_LAUNCHED      0x0020
+#define EVENT_HS_DESC                 0x0021
+#define EVENT_HS_DESC_CONTENT         0x0022
+#define EVENT_NETWORK_LIVENESS        0x0023
+#define EVENT_PT_LOG                  0x0024
+#define EVENT_PT_STATUS               0x0025
+#define EVENT_MAX_                    0x0025
+
+/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
+#define EVENT_CAPACITY_               0x0040
+
+/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
+ * different structure, as it can only handle a maximum left shift of 1<<63. */
+CTASSERT(EVENT_MAX_ < EVENT_CAPACITY_);
+
+#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
+
+#define EVENT_MASK_NONE_             ((uint64_t)0x0)
+
+#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
+#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
+                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
+                                          - EVENT_MIN_))
+
+#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
+                                      & EVENT_MASK_BELOW_MAX_)
+
+/** Helper structure: temporarily stores cell statistics for a circuit. */
+typedef struct cell_stats_t {
+  /** Number of cells added in app-ward direction by command. */
+  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells added in exit-ward direction by command. */
+  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in app-ward direction by command. */
+  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in exit-ward direction by command. */
+  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in app-ward direction by command. */
+  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in exit-ward direction by command. */
+  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
+} cell_stats_t;
+
+void sum_up_cell_stats_by_command(circuit_t *circ,
+                                  cell_stats_t *cell_stats);
+void append_cell_stats_by_command(smartlist_t *event_parts,
+                                  const char *key,
+                                  const uint64_t *include_if_non_zero,
+                                  const uint64_t *number_to_include);
+void format_cell_stats(char **event_string, circuit_t *circ,
+                       cell_stats_t *cell_stats);
+
+/** Helper structure: maps event values to their names. */
+struct control_event_t {
+  uint16_t event_code;
+  const char *event_name;
+};
+
+extern const struct control_event_t control_event_table[];
+
+#ifdef TOR_UNIT_TESTS
+MOCK_DECL(STATIC void,
+          send_control_event_string,(uint16_t event, const char *msg));
+
+MOCK_DECL(STATIC void,
+          queue_control_event_string,(uint16_t event, char *msg));
+
+void control_testing_set_global_event_mask(uint64_t mask);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(CONTROL_EVENTS_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_EVENTS_H) */
diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c
new file mode 100644
index 0000000000..d76e6ad8dd
--- /dev/null
+++ b/src/feature/control/control_fmt.c
@@ -0,0 +1,274 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+
+/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
+ * <b>buf</b>, determine the address:port combination requested on
+ * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
+ * failure. */
+int
+write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
+{
+  char buf2[256];
+  if (conn->chosen_exit_name)
+    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
+      return -1;
+  if (!conn->socks_request)
+    return -1;
+  if (tor_snprintf(buf, len, "%s%s%s:%d",
+               conn->socks_request->address,
+               conn->chosen_exit_name ? buf2 : "",
+               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
+                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
+               conn->socks_request->port)<0)
+    return -1;
+  return 0;
+}
+
+/** Figure out the best name for the target router of an OR connection
+ * <b>conn</b>, and write it into the <b>len</b>-character buffer
+ * <b>name</b>. */
+void
+orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
+{
+  const node_t *node = node_get_by_id(conn->identity_digest);
+  if (node) {
+    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
+    node_get_verbose_nickname(node, name);
+  } else if (! tor_digest_is_zero(conn->identity_digest)) {
+    name[0] = '$';
+    base16_encode(name+1, len-1, conn->identity_digest,
+                  DIGEST_LEN);
+  } else {
+    tor_snprintf(name, len, "%s:%d",
+                 conn->base_.address, conn->base_.port);
+  }
+}
+
+/** Allocate and return a description of <b>circ</b>'s current status,
+ * including its path (if any). */
+char *
+circuit_describe_status_for_controller(origin_circuit_t *circ)
+{
+  char *rv;
+  smartlist_t *descparts = smartlist_new();
+
+  {
+    char *vpath = circuit_list_path_for_controller(circ);
+    if (*vpath) {
+      smartlist_add(descparts, vpath);
+    } else {
+      tor_free(vpath); /* empty path; don't put an extra space in the result */
+    }
+  }
+
+  {
+    cpath_build_state_t *build_state = circ->build_state;
+    smartlist_t *flaglist = smartlist_new();
+    char *flaglist_joined;
+
+    if (build_state->onehop_tunnel)
+      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
+    if (build_state->is_internal)
+      smartlist_add(flaglist, (void *)"IS_INTERNAL");
+    if (build_state->need_capacity)
+      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
+    if (build_state->need_uptime)
+      smartlist_add(flaglist, (void *)"NEED_UPTIME");
+
+    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
+    if (smartlist_len(flaglist)) {
+      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
+
+      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
+
+      tor_free(flaglist_joined);
+    }
+
+    smartlist_free(flaglist);
+  }
+
+  smartlist_add_asprintf(descparts, "PURPOSE=%s",
+                    circuit_purpose_to_controller_string(circ->base_.purpose));
+
+  {
+    const char *hs_state =
+      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
+
+    if (hs_state != NULL) {
+      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
+    }
+  }
+
+  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
+    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    const char *onion_address;
+    if (circ->rend_data) {
+      onion_address = rend_data_get_address(circ->rend_data);
+    } else {
+      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
+      onion_address = addr;
+    }
+    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
+  }
+
+  {
+    char tbuf[ISO_TIME_USEC_LEN+1];
+    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
+
+    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
+  }
+
+  // Show username and/or password if available.
+  if (circ->socks_username_len > 0) {
+    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
+                                     (size_t) circ->socks_username_len);
+    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
+                           socks_username_escaped);
+    tor_free(socks_username_escaped);
+  }
+  if (circ->socks_password_len > 0) {
+    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
+                                     (size_t) circ->socks_password_len);
+    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
+                           socks_password_escaped);
+    tor_free(socks_password_escaped);
+  }
+
+  rv = smartlist_join_strings(descparts, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
+  smartlist_free(descparts);
+
+  return rv;
+}
+
+/** Allocate and return a description of <b>conn</b>'s current status. */
+char *
+entry_connection_describe_status_for_controller(const entry_connection_t *conn)
+{
+  char *rv;
+  smartlist_t *descparts = smartlist_new();
+
+  if (conn->socks_request != NULL) {
+    // Show username and/or password if available; used by IsolateSOCKSAuth.
+    if (conn->socks_request->usernamelen > 0) {
+      char* username_escaped = esc_for_log_len(conn->socks_request->username,
+                                 (size_t) conn->socks_request->usernamelen);
+      smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
+                             username_escaped);
+      tor_free(username_escaped);
+    }
+    if (conn->socks_request->passwordlen > 0) {
+      char* password_escaped = esc_for_log_len(conn->socks_request->password,
+                                 (size_t) conn->socks_request->passwordlen);
+      smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
+                             password_escaped);
+      tor_free(password_escaped);
+    }
+
+    const char *client_protocol;
+    // Show the client protocol; used by IsolateClientProtocol.
+    switch (conn->socks_request->listener_type)
+      {
+      case CONN_TYPE_AP_LISTENER:
+        switch (conn->socks_request->socks_version)
+          {
+          case 4: client_protocol = "SOCKS4"; break;
+          case 5: client_protocol = "SOCKS5"; break;
+          default: client_protocol = "UNKNOWN";
+          }
+        break;
+      case CONN_TYPE_AP_TRANS_LISTENER: client_protocol = "TRANS"; break;
+      case CONN_TYPE_AP_NATD_LISTENER: client_protocol = "NATD"; break;
+      case CONN_TYPE_AP_DNS_LISTENER: client_protocol = "DNS"; break;
+      case CONN_TYPE_AP_HTTP_CONNECT_LISTENER:
+        client_protocol = "HTTPCONNECT"; break;
+      default: client_protocol = "UNKNOWN";
+      }
+    smartlist_add_asprintf(descparts, "CLIENT_PROTOCOL=%s",
+                           client_protocol);
+  }
+
+  // Show newnym epoch; used for stream isolation when NEWNYM is used.
+  smartlist_add_asprintf(descparts, "NYM_EPOCH=%u",
+                         conn->nym_epoch);
+
+  // Show session group; used for stream isolation of multiple listener ports.
+  smartlist_add_asprintf(descparts, "SESSION_GROUP=%d",
+                         conn->entry_cfg.session_group);
+
+  // Show isolation flags.
+  smartlist_t *isoflaglist = smartlist_new();
+  char *isoflaglist_joined;
+  if (conn->entry_cfg.isolation_flags & ISO_DESTPORT) {
+    smartlist_add(isoflaglist, (void *)"DESTPORT");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_DESTADDR) {
+    smartlist_add(isoflaglist, (void *)"DESTADDR");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_SOCKSAUTH) {
+    smartlist_add(isoflaglist, (void *)"SOCKS_USERNAME");
+    smartlist_add(isoflaglist, (void *)"SOCKS_PASSWORD");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_CLIENTPROTO) {
+    smartlist_add(isoflaglist, (void *)"CLIENT_PROTOCOL");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_CLIENTADDR) {
+    smartlist_add(isoflaglist, (void *)"CLIENTADDR");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_SESSIONGRP) {
+    smartlist_add(isoflaglist, (void *)"SESSION_GROUP");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_NYM_EPOCH) {
+    smartlist_add(isoflaglist, (void *)"NYM_EPOCH");
+  }
+  isoflaglist_joined = smartlist_join_strings(isoflaglist, ",", 0, NULL);
+  smartlist_add_asprintf(descparts, "ISO_FIELDS=%s", isoflaglist_joined);
+  tor_free(isoflaglist_joined);
+  smartlist_free(isoflaglist);
+
+  rv = smartlist_join_strings(descparts, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
+  smartlist_free(descparts);
+
+  return rv;
+}
+
+/** Return a longname the node whose identity is <b>id_digest</b>. If
+ * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
+ * returned instead.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+MOCK_IMPL(const char *,
+node_describe_longname_by_id,(const char *id_digest))
+{
+  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
+  node_get_verbose_nickname_by_id(id_digest, longname);
+  return longname;
+}
diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h
new file mode 100644
index 0000000000..f3357cfc4e
--- /dev/null
+++ b/src/feature/control/control_fmt.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.h
+ * \brief Header file for control_fmt.c.
+ **/
+
+#ifndef TOR_CONTROL_FMT_H
+#define TOR_CONTROL_FMT_H
+
+int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
+                               size_t len);
+void orconn_target_get_name(char *buf, size_t len,
+                            or_connection_t *conn);
+char *circuit_describe_status_for_controller(origin_circuit_t *circ);
+char *entry_connection_describe_status_for_controller(const
+                                                     entry_connection_t *conn);
+
+MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
+
+#endif /* !defined(TOR_CONTROL_FMT_H) */
diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c
new file mode 100644
index 0000000000..0823acbe07
--- /dev/null
+++ b/src/feature/control/control_getinfo.c
@@ -0,0 +1,1773 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_getinfo.c
+ * \brief Implementation for miscellaneous controller getinfo commands.
+ */
+
+#define CONTROL_EVENTS_PRIVATE
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/policies.h"
+#include "core/or/versions.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/control/fmt_serverstatus.h"
+#include "feature/control/getinfo_geoip.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/relay_find_addr.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendcache.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/version/torversion.h"
+#include "lib/encoding/kvline.h"
+
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifndef _WIN32
+#include <pwd.h>
+#endif
+
+static char *list_getinfo_options(void);
+static char *download_status_to_string(const download_status_t *dl);
+
+/** Implementation helper for GETINFO: knows the answers for various
+ * trivial-to-implement questions. */
+static int
+getinfo_helper_misc(control_connection_t *conn, const char *question,
+                    char **answer, const char **errmsg)
+{
+  (void) conn;
+  if (!strcmp(question, "version")) {
+    *answer = tor_strdup(get_version());
+  } else if (!strcmp(question, "bw-event-cache")) {
+    *answer = get_bw_samples();
+  } else if (!strcmp(question, "config-file")) {
+    const char *a = get_torrc_fname(0);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-defaults-file")) {
+    const char *a = get_torrc_fname(1);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-text")) {
+    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
+  } else if (!strcmp(question, "info/names")) {
+    *answer = list_getinfo_options();
+  } else if (!strcmp(question, "dormant")) {
+    int dormant = rep_hist_circbuilding_dormant(time(NULL));
+    *answer = tor_strdup(dormant ? "1" : "0");
+  } else if (!strcmp(question, "events/names")) {
+    int i;
+    smartlist_t *event_names = smartlist_new();
+
+    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+      smartlist_add(event_names, (char *)control_event_table[i].event_name);
+    }
+
+    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
+
+    smartlist_free(event_names);
+  } else if (!strcmp(question, "signal/names")) {
+    smartlist_t *signal_names = smartlist_new();
+    int j;
+    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
+      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
+    }
+
+    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
+
+    smartlist_free(signal_names);
+  } else if (!strcmp(question, "features/names")) {
+    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
+  } else if (!strcmp(question, "address")) {
+    uint32_t addr;
+    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
+      *errmsg = "Address unknown";
+      return -1;
+    }
+    *answer = tor_dup_ip(addr);
+    tor_assert_nonfatal(*answer);
+  } else if (!strcmp(question, "traffic/read")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
+  } else if (!strcmp(question, "traffic/written")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
+  } else if (!strcmp(question, "uptime")) {
+    long uptime_secs = get_uptime();
+    tor_asprintf(answer, "%ld", uptime_secs);
+  } else if (!strcmp(question, "process/pid")) {
+    int myPid = -1;
+
+#ifdef _WIN32
+      myPid = _getpid();
+#else
+      myPid = getpid();
+#endif
+
+    tor_asprintf(answer, "%d", myPid);
+  } else if (!strcmp(question, "process/uid")) {
+#ifdef _WIN32
+      *answer = tor_strdup("-1");
+#else
+      int myUid = geteuid();
+      tor_asprintf(answer, "%d", myUid);
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/user")) {
+#ifdef _WIN32
+      *answer = tor_strdup("");
+#else
+      int myUid = geteuid();
+      const struct passwd *myPwEntry = tor_getpwuid(myUid);
+
+      if (myPwEntry) {
+        *answer = tor_strdup(myPwEntry->pw_name);
+      } else {
+        *answer = tor_strdup("");
+      }
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/descriptor-limit")) {
+    int max_fds = get_max_sockets();
+    tor_asprintf(answer, "%d", max_fds);
+  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
+    tor_asprintf(answer, "%"PRIu64,
+                 (get_options()->MaxMemInQueues));
+  } else if (!strcmp(question, "fingerprint")) {
+    crypto_pk_t *server_key;
+    if (!server_mode(get_options())) {
+      *errmsg = "Not running in server mode";
+      return -1;
+    }
+    server_key = get_server_identity_key();
+    *answer = tor_malloc(HEX_DIGEST_LEN+1);
+    crypto_pk_get_fingerprint(server_key, *answer, 0);
+  }
+  return 0;
+}
+
+/** Awful hack: return a newly allocated string based on a routerinfo and
+ * (possibly) an extrainfo, sticking the read-history and write-history from
+ * <b>ei</b> into the resulting string.  The thing you get back won't
+ * necessarily have a valid signature.
+ *
+ * New code should never use this; it's for backward compatibility.
+ *
+ * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
+ * not be NUL-terminated. */
+static char *
+munge_extrainfo_into_routerinfo(const char *ri_body,
+                                const signed_descriptor_t *ri,
+                                const signed_descriptor_t *ei)
+{
+  char *out = NULL, *outp;
+  int i;
+  const char *router_sig;
+  const char *ei_body = signed_descriptor_get_body(ei);
+  size_t ri_len = ri->signed_descriptor_len;
+  size_t ei_len = ei->signed_descriptor_len;
+  if (!ei_body)
+    goto bail;
+
+  outp = out = tor_malloc(ri_len+ei_len+1);
+  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
+    goto bail;
+  ++router_sig;
+  memcpy(out, ri_body, router_sig-ri_body);
+  outp += router_sig-ri_body;
+
+  for (i=0; i < 2; ++i) {
+    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
+    const char *cp, *eol;
+    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
+      continue;
+    ++cp;
+    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
+      continue;
+    memcpy(outp, cp, eol-cp+1);
+    outp += eol-cp+1;
+  }
+  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
+  *outp++ = '\0';
+  tor_assert(outp-out < (int)(ri_len+ei_len+1));
+
+  return out;
+ bail:
+  tor_free(out);
+  return tor_strndup(ri_body, ri->signed_descriptor_len);
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * which ports are bound. */
+static int
+getinfo_helper_listeners(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  int type;
+  smartlist_t *res;
+
+  (void)control_conn;
+  (void)errmsg;
+
+  if (!strcmp(question, "net/listeners/or"))
+    type = CONN_TYPE_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/extor"))
+    type = CONN_TYPE_EXT_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/dir"))
+    type = CONN_TYPE_DIR_LISTENER;
+  else if (!strcmp(question, "net/listeners/socks"))
+    type = CONN_TYPE_AP_LISTENER;
+  else if (!strcmp(question, "net/listeners/trans"))
+    type = CONN_TYPE_AP_TRANS_LISTENER;
+  else if (!strcmp(question, "net/listeners/natd"))
+    type = CONN_TYPE_AP_NATD_LISTENER;
+  else if (!strcmp(question, "net/listeners/httptunnel"))
+    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  else if (!strcmp(question, "net/listeners/dns"))
+    type = CONN_TYPE_AP_DNS_LISTENER;
+  else if (!strcmp(question, "net/listeners/control"))
+    type = CONN_TYPE_CONTROL_LISTENER;
+  else
+    return 0; /* unknown key */
+
+  res = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    struct sockaddr_storage ss;
+    socklen_t ss_len = sizeof(ss);
+
+    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
+      continue;
+
+    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
+      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
+    } else {
+      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
+      smartlist_add(res, esc_for_log(tmp));
+      tor_free(tmp);
+    }
+
+  } SMARTLIST_FOREACH_END(conn);
+
+  *answer = smartlist_join_strings(res, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
+  smartlist_free(res);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * the current time in both local and UTC forms. */
+STATIC int
+getinfo_helper_current_time(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  if (!strcmp(question, "current-time/local"))
+    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else if (!strcmp(question, "current-time/utc"))
+    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else
+    return 0;
+
+  *answer = tor_strdup(timebuf);
+  return 0;
+}
+
+/** GETINFO helper for dumping different consensus flavors
+ * returns: 0 on success -1 on error. */
+STATIC int
+getinfo_helper_current_consensus(consensus_flavor_t flavor,
+                                 char** answer,
+                                 const char** errmsg)
+{
+  const char *flavor_name = networkstatus_get_flavor_name(flavor);
+  if (BUG(!strcmp(flavor_name, "??"))) {
+    *errmsg = "Internal error: unrecognized flavor name.";
+    return -1;
+  }
+  if (we_want_to_fetch_flavor(get_options(), flavor)) {
+    /** Check from the cache */
+    const cached_dir_t *consensus = dirserv_get_consensus(flavor_name);
+    if (consensus) {
+      *answer = tor_strdup(consensus->dir);
+    }
+  }
+  if (!*answer) { /* try loading it from disk */
+
+    tor_mmap_t *mapped = networkstatus_map_cached_consensus(flavor_name);
+    if (mapped) {
+      *answer = tor_memdup_nulterm(mapped->data, mapped->size);
+      tor_munmap_file(mapped);
+    }
+    if (!*answer) { /* generate an error */
+      *errmsg = "Could not open cached consensus. "
+        "Make sure FetchUselessDescriptors is set to 1.";
+      return -1;
+    }
+  }
+  return 0;
+}
+
+/** Helper for getinfo_helper_dir.
+ *
+ * Add a signed_descriptor_t to <b>descs_out</b> for each router matching
+ * <b>key</b>.  The key should be either
+ *   - "/tor/server/authority" for our own routerinfo;
+ *   - "/tor/server/all" for all the routerinfos we have, concatenated;
+ *   - "/tor/server/fp/FP" where FP is a plus-separated sequence of
+ *     hex identity digests; or
+ *   - "/tor/server/d/D" where D is a plus-separated sequence
+ *     of server descriptor digests, in hex.
+ *
+ * Return 0 if we found some matching descriptors, or -1 if we do not
+ * have any descriptors, no matching descriptors, or if we did not
+ * recognize the key (URL).
+ * If -1 is returned *<b>msg</b> will be set to an appropriate error
+ * message.
+ */
+static int
+controller_get_routerdescs(smartlist_t *descs_out, const char *key,
+                        const char **msg)
+{
+  *msg = NULL;
+
+  if (!strcmp(key, "/tor/server/all")) {
+    routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
+                      smartlist_add(descs_out, &(r->cache_info)));
+  } else if (!strcmp(key, "/tor/server/authority")) {
+    const routerinfo_t *ri = router_get_my_routerinfo();
+    if (ri)
+      smartlist_add(descs_out, (void*) &(ri->cache_info));
+  } else if (!strcmpstart(key, "/tor/server/d/")) {
+    smartlist_t *digests = smartlist_new();
+    key += strlen("/tor/server/d/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH(digests, const char *, d,
+       {
+         signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
+         if (sd)
+           smartlist_add(descs_out,sd);
+       });
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else if (!strcmpstart(key, "/tor/server/fp/")) {
+    smartlist_t *digests = smartlist_new();
+    time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    key += strlen("/tor/server/fp/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
+         if (router_digest_is_me(d)) {
+           /* calling router_get_my_routerinfo() to make sure it exists */
+           const routerinfo_t *ri = router_get_my_routerinfo();
+           if (ri)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         } else {
+           const routerinfo_t *ri = router_get_by_id_digest(d);
+           /* Don't actually serve a descriptor that everyone will think is
+            * expired.  This is an (ugly) workaround to keep buggy 0.1.1.10
+            * Tors from downloading descriptors that they will throw away.
+            */
+           if (ri && ri->cache_info.published_on > cutoff)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         }
+    } SMARTLIST_FOREACH_END(d);
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else {
+    *msg = "Key not recognized";
+    return -1;
+  }
+
+  if (!smartlist_len(descs_out)) {
+    *msg = "Servers unavailable";
+    return -1;
+  }
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * directory information. */
+STATIC int
+getinfo_helper_dir(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  (void) control_conn;
+  if (!strcmpstart(question, "desc/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/id/* "
+                "instead of desc/id/*.";
+      return 0;
+    }
+  } else if (!strcmpstart(question, "desc/name/")) {
+    const routerinfo_t *ri = NULL;
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node =
+      node_get_by_nickname(question+strlen("desc/name/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/name/* "
+                "instead of desc/name/*.";
+      return 0;
+    }
+  } else if (!strcmp(question, "desc/download-enabled")) {
+    int r = we_fetch_router_descriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmp(question, "desc/all-recent")) {
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
+      {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        if (body)
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+      });
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
+    /* XXXX Remove this once Torstat asks for extrainfos. */
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+        if (ei && body) {
+          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
+                                                        &ri->cache_info, ei));
+        } else if (body) {
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+        }
+      } SMARTLIST_FOREACH_END(ri);
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/client/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      rend_cache_entry_t *e = NULL;
+      if (!rend_cache_lookup_entry(question, -1, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      const char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_cache_lookup_encoded_as_client(&service_pk);
+      if (desc) {
+        *answer = tor_strdup(desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/service/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+    rend_cache_entry_t *e = NULL;
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_service_lookup_current_desc(&service_pk);
+      if (desc) {
+        /* Newly allocated string, we have ownership. */
+        *answer = desc;
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "md/all")) {
+    const smartlist_t *nodes = nodelist_get_list();
+    tor_assert(nodes);
+
+    if (smartlist_len(nodes) == 0) {
+      *answer = tor_strdup("");
+      return 0;
+    }
+
+    smartlist_t *microdescs = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
+      if (n->md && n->md->body) {
+        char *copy = tor_strndup(n->md->body, n->md->bodylen);
+        smartlist_add(microdescs, copy);
+      }
+    } SMARTLIST_FOREACH_END(n);
+
+    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
+    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
+    smartlist_free(microdescs);
+  } else if (!strcmpstart(question, "md/id/")) {
+    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmpstart(question, "md/name/")) {
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
+    /* XXXX duplicated code */
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmp(question, "md/download-enabled")) {
+    int r = we_fetch_microdescriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmpstart(question, "desc-annotations/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node =
+      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *annotations =
+        signed_descriptor_get_annotations(&ri->cache_info);
+      if (annotations)
+        *answer = tor_strndup(annotations,
+                              ri->cache_info.annotations_len);
+    }
+  } else if (!strcmpstart(question, "dir/server/")) {
+    size_t answer_len = 0;
+    char *url = NULL;
+    smartlist_t *descs = smartlist_new();
+    const char *msg;
+    int res;
+    char *cp;
+    tor_asprintf(&url, "/tor/%s", question+4);
+    res = controller_get_routerdescs(descs, url, &msg);
+    if (res) {
+      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
+      smartlist_free(descs);
+      tor_free(url);
+      *errmsg = msg;
+      return -1;
+    }
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      answer_len += sd->signed_descriptor_len);
+    cp = *answer = tor_malloc(answer_len+1);
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      {
+                        memcpy(cp, signed_descriptor_get_body(sd),
+                               sd->signed_descriptor_len);
+                        cp += sd->signed_descriptor_len;
+                      });
+    *cp = '\0';
+    tor_free(url);
+    smartlist_free(descs);
+  } else if (!strcmpstart(question, "dir/status/")) {
+    *answer = tor_strdup("");
+  } else if (!strcmp(question, "dir/status-vote/current/consensus")) {
+    int consensus_result = getinfo_helper_current_consensus(FLAV_NS,
+                                                            answer, errmsg);
+    if (consensus_result < 0) {
+      return -1;
+    }
+  } else if (!strcmp(question,
+                     "dir/status-vote/current/consensus-microdesc")) {
+    int consensus_result = getinfo_helper_current_consensus(FLAV_MICRODESC,
+                                                            answer, errmsg);
+    if (consensus_result < 0) {
+      return -1;
+    }
+  } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
+    routerlist_t *routerlist = router_get_routerlist();
+    if (!routerlist || !routerlist->routers ||
+        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
+      return -1;
+    }
+  } else if (!strcmpstart(question, "extra-info/digest/")) {
+    question += strlen("extra-info/digest/");
+    if (strlen(question) == HEX_DIGEST_LEN) {
+      char d[DIGEST_LEN];
+      signed_descriptor_t *sd = NULL;
+      if (base16_decode(d, sizeof(d), question, strlen(question))
+                        == sizeof(d)) {
+        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
+         * but I don't want to risk affecting other parts of the code,
+         * especially since the rules for using our own extrainfo (including
+         * when it might be freed) are different from those for using one
+         * we have downloaded. */
+        if (router_extrainfo_digest_is_me(d))
+          sd = &(router_get_my_extrainfo()->cache_info);
+        else
+          sd = extrainfo_get_by_descriptor_digest(d);
+      }
+      if (sd) {
+        const char *body = signed_descriptor_get_body(sd);
+        if (body)
+          *answer = tor_strndup(body, sd->signed_descriptor_len);
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** Given a smartlist of 20-byte digests, return a newly allocated string
+ * containing each of those digests in order, formatted in HEX, and terminated
+ * with a newline. */
+static char *
+digest_list_to_string(const smartlist_t *sl)
+{
+  int len;
+  char *result, *s;
+
+  /* Allow for newlines, and a \0 at the end */
+  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
+  result = tor_malloc_zero(len);
+
+  s = result;
+  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
+    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
+    s[HEX_DIGEST_LEN] = '\n';
+    s += HEX_DIGEST_LEN + 1;
+  } SMARTLIST_FOREACH_END(digest);
+  *s = '\0';
+
+  return result;
+}
+
+/** Turn a download_status_t into a human-readable description in a newly
+ * allocated string.  The format is specified in control-spec.txt, under
+ * the documentation for "GETINFO download/..." .  */
+static char *
+download_status_to_string(const download_status_t *dl)
+{
+  char *rv = NULL;
+  char tbuf[ISO_TIME_LEN+1];
+  const char *schedule_str, *want_authority_str;
+  const char *increment_on_str, *backoff_str;
+
+  if (dl) {
+    /* Get some substrings of the eventual output ready */
+    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
+
+    switch (dl->schedule) {
+      case DL_SCHED_GENERIC:
+        schedule_str = "DL_SCHED_GENERIC";
+        break;
+      case DL_SCHED_CONSENSUS:
+        schedule_str = "DL_SCHED_CONSENSUS";
+        break;
+      case DL_SCHED_BRIDGE:
+        schedule_str = "DL_SCHED_BRIDGE";
+        break;
+      default:
+        schedule_str = "unknown";
+        break;
+    }
+
+    switch (dl->want_authority) {
+      case DL_WANT_ANY_DIRSERVER:
+        want_authority_str = "DL_WANT_ANY_DIRSERVER";
+        break;
+      case DL_WANT_AUTHORITY:
+        want_authority_str = "DL_WANT_AUTHORITY";
+        break;
+      default:
+        want_authority_str = "unknown";
+        break;
+    }
+
+    switch (dl->increment_on) {
+      case DL_SCHED_INCREMENT_FAILURE:
+        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
+        break;
+      case DL_SCHED_INCREMENT_ATTEMPT:
+        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
+        break;
+      default:
+        increment_on_str = "unknown";
+        break;
+    }
+
+    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
+
+    /* Now assemble them */
+    tor_asprintf(&rv,
+                 "next-attempt-at %s\n"
+                 "n-download-failures %u\n"
+                 "n-download-attempts %u\n"
+                 "schedule %s\n"
+                 "want-authority %s\n"
+                 "increment-on %s\n"
+                 "backoff %s\n"
+                 "last-backoff-position %u\n"
+                 "last-delay-used %d\n",
+                 tbuf,
+                 dl->n_download_failures,
+                 dl->n_download_attempts,
+                 schedule_str,
+                 want_authority_str,
+                 increment_on_str,
+                 backoff_str,
+                 dl->last_backoff_position,
+                 dl->last_delay_used);
+  }
+
+  return rv;
+}
+
+/** Handle the consensus download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_networkstatus(const char *flavor,
+                                       download_status_t **dl_to_emit,
+                                       const char **errmsg)
+{
+  /*
+   * We get the one for the current bootstrapped status by default, or
+   * take an extra /bootstrap or /running suffix
+   */
+  if (strcmp(flavor, "ns") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
+  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
+  } else if (strcmp(flavor, "ns/running") == 0 ) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
+  } else if (strcmp(flavor, "microdesc") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/running") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
+  } else {
+    *errmsg = "Unknown flavor";
+  }
+}
+
+/** Handle the cert download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_cert(const char *fp_sk_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  const char *sk_req;
+  char id_digest[DIGEST_LEN];
+  char sk_digest[DIGEST_LEN];
+
+  /*
+   * We have to handle four cases; fp_sk_req is the request with
+   * a prefix of "downloads/cert/" snipped off.
+   *
+   * Case 1: fp_sk_req = "fps"
+   *  - We should emit a digest_list with a list of all the identity
+   *    fingerprints that can be queried for certificate download status;
+   *    get it by calling list_authority_ids_with_downloads().
+   *
+   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
+   *  - We want the default certificate for this identity fingerprint's
+   *    download status; this is the download we get from URLs starting
+   *    in /fp/ on the directory server.  We can get it with
+   *    id_only_download_status_for_authority_id().
+   *
+   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
+   *  - We want a list of all signing key digests for this identity
+   *    fingerprint which can be queried for certificate download status.
+   *    Get it with list_sk_digests_for_authority_id().
+   *
+   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
+   *         signing key digest sk
+   *   - We want the download status for the certificate for this specific
+   *     signing key and fingerprint.  These correspond to the ones we get
+   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
+   *     list_sk_digests_for_authority_id().
+   */
+
+  if (strcmp(fp_sk_req, "fps") == 0) {
+    *digest_list = list_authority_ids_with_downloads();
+    if (!(*digest_list)) {
+      *errmsg = "Failed to get list of authority identity digests (!)";
+    }
+  } else if (!strcmpstart(fp_sk_req, "fp/")) {
+    fp_sk_req += strlen("fp/");
+    /* Okay, look for another / to tell the fp from fp-sk cases */
+    sk_req = strchr(fp_sk_req, '/');
+    if (sk_req) {
+      /* okay, split it here and try to parse <fp> */
+      if (base16_decode(id_digest, DIGEST_LEN,
+                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
+        /* Skip past the '/' */
+        ++sk_req;
+        if (strcmp(sk_req, "sks") == 0) {
+          /* We're asking for the list of signing key fingerprints */
+          *digest_list = list_sk_digests_for_authority_id(id_digest);
+          if (!(*digest_list)) {
+            *errmsg = "Failed to get list of signing key digests for this "
+                      "authority identity digest";
+          }
+        } else {
+          /* We've got a signing key digest */
+          if (base16_decode(sk_digest, DIGEST_LEN,
+                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
+            *dl_to_emit =
+              download_status_for_authority_id_and_sk(id_digest, sk_digest);
+            if (!(*dl_to_emit)) {
+              *errmsg = "Failed to get download status for this identity/"
+                        "signing key digest pair";
+            }
+          } else {
+            *errmsg = "That didn't look like a signing key digest";
+          }
+        }
+      } else {
+        *errmsg = "That didn't look like an identity digest";
+      }
+    } else {
+      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
+      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
+        if (base16_decode(id_digest, DIGEST_LEN,
+                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
+          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
+          if (!(*dl_to_emit)) {
+            *errmsg = "Failed to get download status for this authority "
+                      "identity digest";
+          }
+        } else {
+          *errmsg = "That didn't look like a digest";
+        }
+      } else {
+        *errmsg = "That didn't look like a digest";
+      }
+    }
+  } else {
+    *errmsg = "Unknown certificate download status query";
+  }
+}
+
+/** Handle the routerdesc download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_desc(const char *desc_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  char desc_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: desc_req = "descs"
+   *   - Emit a list of all router descriptor digests, which we get by
+   *     calling router_get_descriptor_digests(); this can return NULL
+   *     if we have no current ns-flavor consensus.
+   *
+   * Case 2: desc_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using router_get_dl_status_by_descriptor_digest().
+   */
+
+  if (strcmp(desc_req, "descs") == 0) {
+    *digest_list = router_get_descriptor_digests();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
+    }
+    /*
+     * Microdescs don't use the download_status_t mechanism, so we don't
+     * answer queries about their downloads here; see microdesc.c.
+     */
+  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(desc_digest, DIGEST_LEN,
+                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such descriptor digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown router descriptor download status query";
+  }
+}
+
+/** Handle the bridge download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_bridge(const char *bridge_req,
+                                download_status_t **dl_to_emit,
+                                smartlist_t **digest_list,
+                                const char **errmsg)
+{
+  char bridge_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: bridge_req = "bridges"
+   *   - Emit a list of all bridge identity digests, which we get by
+   *     calling list_bridge_identities(); this can return NULL if we are
+   *     not using bridges.
+   *
+   * Case 2: bridge_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using get_bridge_dl_status_by_id().
+   */
+
+  if (strcmp(bridge_req, "bridges") == 0) {
+    *digest_list = list_bridge_identities();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to be using bridges";
+    }
+  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(bridge_digest, DIGEST_LEN,
+                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such bridge identity digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown bridge descriptor download status query";
+  }
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * download status information. */
+STATIC int
+getinfo_helper_downloads(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  download_status_t *dl_to_emit = NULL;
+  smartlist_t *digest_list = NULL;
+
+  /* Assert args are sane */
+  tor_assert(control_conn != NULL);
+  tor_assert(question != NULL);
+  tor_assert(answer != NULL);
+  tor_assert(errmsg != NULL);
+
+  /* We check for this later to see if we should supply a default */
+  *errmsg = NULL;
+
+  /* Are we after networkstatus downloads? */
+  if (!strcmpstart(question, "downloads/networkstatus/")) {
+    getinfo_helper_downloads_networkstatus(
+        question + strlen("downloads/networkstatus/"),
+        &dl_to_emit, errmsg);
+  /* Certificates? */
+  } else if (!strcmpstart(question, "downloads/cert/")) {
+    getinfo_helper_downloads_cert(
+        question + strlen("downloads/cert/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Router descriptors? */
+  } else if (!strcmpstart(question, "downloads/desc/")) {
+    getinfo_helper_downloads_desc(
+        question + strlen("downloads/desc/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Bridge descriptors? */
+  } else if (!strcmpstart(question, "downloads/bridge/")) {
+    getinfo_helper_downloads_bridge(
+        question + strlen("downloads/bridge/"),
+        &dl_to_emit, &digest_list, errmsg);
+  } else {
+    *errmsg = "Unknown download status query";
+  }
+
+  if (dl_to_emit) {
+    *answer = download_status_to_string(dl_to_emit);
+
+    return 0;
+  } else if (digest_list) {
+    *answer = digest_list_to_string(digest_list);
+    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
+    smartlist_free(digest_list);
+
+    return 0;
+  } else {
+    if (!(*errmsg)) {
+      *errmsg = "Unknown error";
+    }
+
+    return -1;
+  }
+}
+
+/** Implementation helper for GETINFO: knows how to generate summaries of the
+ * current states of things we send events about. */
+static int
+getinfo_helper_events(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  const or_options_t *options = get_options();
+  (void) control_conn;
+  if (!strcmp(question, "circuit-status")) {
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
+      origin_circuit_t *circ;
+      char *circdesc;
+      const char *state;
+      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
+        continue;
+      circ = TO_ORIGIN_CIRCUIT(circ_);
+
+      if (circ->base_.state == CIRCUIT_STATE_OPEN)
+        state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
+      else if (circ->cpath)
+        state = "EXTENDED";
+      else
+        state = "LAUNCHED";
+
+      circdesc = circuit_describe_status_for_controller(circ);
+
+      smartlist_add_asprintf(status, "%lu %s%s%s",
+                   (unsigned long)circ->global_identifier,
+                   state, *circdesc ? " " : "", circdesc);
+      tor_free(circdesc);
+    }
+    SMARTLIST_FOREACH_END(circ_);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "stream-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    char buf[256];
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      entry_connection_t *conn;
+      circuit_t *circ;
+      origin_circuit_t *origin_circ = NULL;
+      if (base_conn->type != CONN_TYPE_AP ||
+          base_conn->marked_for_close ||
+          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
+          base_conn->state == AP_CONN_STATE_NATD_WAIT)
+        continue;
+      conn = TO_ENTRY_CONN(base_conn);
+      switch (base_conn->state)
+        {
+        case AP_CONN_STATE_CONTROLLER_WAIT:
+        case AP_CONN_STATE_CIRCUIT_WAIT:
+          if (conn->socks_request &&
+              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
+            state = "NEWRESOLVE";
+          else
+            state = "NEW";
+          break;
+        case AP_CONN_STATE_RENDDESC_WAIT:
+        case AP_CONN_STATE_CONNECT_WAIT:
+          state = "SENTCONNECT"; break;
+        case AP_CONN_STATE_RESOLVE_WAIT:
+          state = "SENTRESOLVE"; break;
+        case AP_CONN_STATE_OPEN:
+          state = "SUCCEEDED"; break;
+        default:
+          log_warn(LD_BUG, "Asked for stream in unknown state %d",
+                   base_conn->state);
+          continue;
+        }
+      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+      if (circ && CIRCUIT_IS_ORIGIN(circ))
+        origin_circ = TO_ORIGIN_CIRCUIT(circ);
+      write_stream_target_to_buf(conn, buf, sizeof(buf));
+      smartlist_add_asprintf(status, "%lu %s %lu %s",
+                   (unsigned long) base_conn->global_identifier,state,
+                   origin_circ?
+                         (unsigned long)origin_circ->global_identifier : 0ul,
+                   buf);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "orconn-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      char name[128];
+      or_connection_t *conn;
+      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
+        continue;
+      conn = TO_OR_CONN(base_conn);
+      if (conn->base_.state == OR_CONN_STATE_OPEN)
+        state = "CONNECTED";
+      else if (conn->nickname)
+        state = "LAUNCHED";
+      else
+        state = "NEW";
+      orconn_target_get_name(name, sizeof(name), conn);
+      smartlist_add_asprintf(status, "%s %s", name, state);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmpstart(question, "address-mappings/")) {
+    time_t min_e, max_e;
+    smartlist_t *mappings;
+    question += strlen("address-mappings/");
+    if (!strcmp(question, "all")) {
+      min_e = 0; max_e = TIME_MAX;
+    } else if (!strcmp(question, "cache")) {
+      min_e = 2; max_e = TIME_MAX;
+    } else if (!strcmp(question, "config")) {
+      min_e = 0; max_e = 0;
+    } else if (!strcmp(question, "control")) {
+      min_e = 1; max_e = 1;
+    } else {
+      return 0;
+    }
+    mappings = smartlist_new();
+    addressmap_get_mappings(mappings, min_e, max_e, 1);
+    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
+    smartlist_free(mappings);
+  } else if (!strcmpstart(question, "status/")) {
+    /* Note that status/ is not a catch-all for events; there's only supposed
+     * to be a status GETINFO if there's a corresponding STATUS event. */
+    if (!strcmp(question, "status/circuit-established")) {
+      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
+    } else if (!strcmp(question, "status/enough-dir-info")) {
+      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
+    } else if (!strcmp(question, "status/good-server-descriptor") ||
+               !strcmp(question, "status/accepted-server-descriptor")) {
+      /* They're equivalent for now, until we can figure out how to make
+       * good-server-descriptor be what we want. See comment in
+       * control-spec.txt. */
+      *answer = tor_strdup(directories_have_accepted_server_descriptor()
+                           ? "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
+      *answer = tor_strdup(check_whether_orport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
+      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded")) {
+      tor_asprintf(answer, "OR=%d DIR=%d",
+                   check_whether_orport_reachable(options) ? 1 : 0,
+                   check_whether_dirport_reachable(options) ? 1 : 0);
+    } else if (!strcmp(question, "status/bootstrap-phase")) {
+      *answer = control_event_boot_last_msg();
+    } else if (!strcmpstart(question, "status/version/")) {
+      int is_server = server_mode(options);
+      networkstatus_t *c = networkstatus_get_latest_consensus();
+      version_status_t status;
+      const char *recommended;
+      if (c) {
+        recommended = is_server ? c->server_versions : c->client_versions;
+        status = tor_version_is_obsolete(VERSION, recommended);
+      } else {
+        recommended = "?";
+        status = VS_UNKNOWN;
+      }
+
+      if (!strcmp(question, "status/version/recommended")) {
+        *answer = tor_strdup(recommended);
+        return 0;
+      }
+      if (!strcmp(question, "status/version/current")) {
+        switch (status)
+          {
+          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
+          case VS_OLD: *answer = tor_strdup("obsolete"); break;
+          case VS_NEW: *answer = tor_strdup("new"); break;
+          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
+          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
+          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
+          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
+          default: tor_fragile_assert();
+          }
+      }
+    } else if (!strcmp(question, "status/clients-seen")) {
+      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
+      if (!bridge_stats) {
+        *errmsg = "No bridge-client stats available";
+        return -1;
+      }
+      *answer = bridge_stats;
+    } else if (!strcmp(question, "status/fresh-relay-descs")) {
+      if (!server_mode(options)) {
+        *errmsg = "Only relays have descriptors";
+        return -1;
+      }
+      routerinfo_t *r;
+      extrainfo_t *e;
+      int result;
+      if ((result = router_build_fresh_descriptor(&r, &e)) < 0) {
+        switch (result) {
+          case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+            *errmsg = "Cannot get relay address while generating descriptor";
+            break;
+          case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+            *errmsg = "Key digest failed";
+            break;
+          case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+            *errmsg = "Cannot generate router descriptor";
+            break;
+          default:
+            *errmsg = "Error generating descriptor";
+            break;
+        }
+        return -1;
+      }
+      size_t size = r->cache_info.signed_descriptor_len + 1;
+      if (e) {
+        size += e->cache_info.signed_descriptor_len + 1;
+      }
+      tor_assert(r->cache_info.signed_descriptor_len);
+      char *descs = tor_malloc(size);
+      char *cp = descs;
+      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
+             r->cache_info.signed_descriptor_len);
+      cp += r->cache_info.signed_descriptor_len - 1;
+      if (e) {
+        if (cp[0] == '\0') {
+          cp[0] = '\n';
+        } else if (cp[0] != '\n') {
+          cp[1] = '\n';
+          cp++;
+        }
+        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
+               e->cache_info.signed_descriptor_len);
+        cp += e->cache_info.signed_descriptor_len - 1;
+      }
+      if (cp[0] == '\n') {
+        cp[0] = '\0';
+      } else if (cp[0] != '\0') {
+        cp[1] = '\0';
+      }
+      *answer = descs;
+      routerinfo_free(r);
+      extrainfo_free(e);
+    } else {
+      return 0;
+    }
+  }
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows how to enumerate hidden services
+ * created via the control port. */
+STATIC int
+getinfo_helper_onions(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
+
+  if (control_conn && !strcmp(question, "onions/current")) {
+    onion_list = control_conn->ephemeral_onion_services;
+  } else if (!strcmp(question, "onions/detached")) {
+    onion_list = get_detached_onion_services();
+  } else {
+    return 0;
+  }
+  if (!onion_list || smartlist_len(onion_list) == 0) {
+    if (answer) {
+      *answer = tor_strdup("");
+    }
+  } else {
+    if (answer) {
+      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about network
+ * liveness. */
+static int
+getinfo_helper_liveness(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+  if (strcmp(question, "network-liveness") == 0) {
+    if (get_cached_network_liveness()) {
+      *answer = tor_strdup("up");
+    } else {
+      *answer = tor_strdup("down");
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
+/** Callback function for GETINFO: on a given control connection, try to
+ * answer the question <b>q</b> and store the newly-allocated answer in
+ * *<b>a</b>. If an internal error occurs, return -1 and optionally set
+ * *<b>error_out</b> to point to an error message to be delivered to the
+ * controller. On success, _or if the key is not recognized_, return 0. Do not
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
+ */
+typedef int (*getinfo_helper_t)(control_connection_t *,
+                                const char *q, char **a,
+                                const char **error_out);
+
+/** A single item for the GETINFO question-to-answer-function table. */
+typedef struct getinfo_item_t {
+  const char *varname; /**< The value (or prefix) of the question. */
+  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
+                        * this entry is documentation-only. */
+  const char *desc; /**< Description of the variable. */
+  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
+} getinfo_item_t;
+
+#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
+#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
+#define DOC(name, desc) { name, NULL, desc, 0 }
+
+/** Table mapping questions accepted by GETINFO to the functions that know how
+ * to answer them. */
+static const getinfo_item_t getinfo_items[] = {
+  ITEM("version", misc, "The current version of Tor."),
+  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
+  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
+  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
+  ITEM("config-text", misc,
+       "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
+  ITEM("accounting/bytes", accounting,
+       "Number of bytes read/written so far in the accounting interval."),
+  ITEM("accounting/bytes-left", accounting,
+      "Number of bytes left to write/read so far in the accounting interval."),
+  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
+  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
+  ITEM("accounting/interval-start", accounting,
+       "Time when the accounting period starts."),
+  ITEM("accounting/interval-end", accounting,
+       "Time when the accounting period ends."),
+  ITEM("accounting/interval-wake", accounting,
+       "Time to wake up in this accounting period."),
+  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
+  ITEM("entry-guards", entry_guards,
+       "Which nodes are we using as entry guards?"),
+  ITEM("fingerprint", misc, NULL),
+  PREFIX("config/", config, "Current configuration values."),
+  DOC("config/names",
+      "List of configuration options, types, and documentation."),
+  DOC("config/defaults",
+      "List of default values for configuration options. "
+      "See also config/names"),
+  PREFIX("current-time/", current_time, "Current time."),
+  DOC("current-time/local", "Current time on the local system."),
+  DOC("current-time/utc", "Current UTC time."),
+  PREFIX("downloads/networkstatus/", downloads,
+         "Download statuses for networkstatus objects"),
+  DOC("downloads/networkstatus/ns",
+      "Download status for current-mode networkstatus download"),
+  DOC("downloads/networkstatus/ns/bootstrap",
+      "Download status for bootstrap-time networkstatus download"),
+  DOC("downloads/networkstatus/ns/running",
+      "Download status for run-time networkstatus download"),
+  DOC("downloads/networkstatus/microdesc",
+      "Download status for current-mode microdesc download"),
+  DOC("downloads/networkstatus/microdesc/bootstrap",
+      "Download status for bootstrap-time microdesc download"),
+  DOC("downloads/networkstatus/microdesc/running",
+      "Download status for run-time microdesc download"),
+  PREFIX("downloads/cert/", downloads,
+         "Download statuses for certificates, by id fingerprint and "
+         "signing key"),
+  DOC("downloads/cert/fps",
+      "List of authority fingerprints for which any download statuses "
+      "exist"),
+  DOC("downloads/cert/fp/<fp>",
+      "Download status for <fp> with the default signing key; corresponds "
+      "to /fp/ URLs on directory server."),
+  DOC("downloads/cert/fp/<fp>/sks",
+      "List of signing keys for which specific download statuses are "
+      "available for this id fingerprint"),
+  DOC("downloads/cert/fp/<fp>/<sk>",
+      "Download status for <fp> with signing key <sk>; corresponds "
+      "to /fp-sk/ URLs on directory server."),
+  PREFIX("downloads/desc/", downloads,
+         "Download statuses for router descriptors, by descriptor digest"),
+  DOC("downloads/desc/descs",
+      "Return a list of known router descriptor digests"),
+  DOC("downloads/desc/<desc>",
+      "Return a download status for a given descriptor digest"),
+  PREFIX("downloads/bridge/", downloads,
+         "Download statuses for bridge descriptors, by bridge identity "
+         "digest"),
+  DOC("downloads/bridge/bridges",
+      "Return a list of configured bridge identity digests with download "
+      "statuses"),
+  DOC("downloads/bridge/<desc>",
+      "Return a download status for a given bridge identity digest"),
+  ITEM("info/names", misc,
+       "List of GETINFO options, types, and documentation."),
+  ITEM("events/names", misc,
+       "Events that the controller can ask for with SETEVENTS."),
+  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
+  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
+  PREFIX("desc/id/", dir, "Router descriptors by ID."),
+  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
+  ITEM("desc/all-recent", dir,
+       "All non-expired, non-superseded router descriptors."),
+  ITEM("desc/download-enabled", dir,
+       "Do we try to download router descriptors?"),
+  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
+  ITEM("md/all", dir, "All known microdescriptors."),
+  PREFIX("md/id/", dir, "Microdescriptors by ID"),
+  PREFIX("md/name/", dir, "Microdescriptors by name"),
+  ITEM("md/download-enabled", dir,
+       "Do we try to download microdescriptors?"),
+  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
+  PREFIX("hs/client/desc/id", dir,
+         "Hidden Service descriptor in client's cache by onion."),
+  PREFIX("hs/service/desc/id/", dir,
+         "Hidden Service descriptor in services's cache by onion."),
+  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
+  ITEM("ns/all", networkstatus,
+       "Brief summary of router status (v2 directory format)"),
+  PREFIX("ns/id/", networkstatus,
+         "Brief summary of router status by ID (v2 directory format)."),
+  PREFIX("ns/name/", networkstatus,
+         "Brief summary of router status by nickname (v2 directory format)."),
+  PREFIX("ns/purpose/", networkstatus,
+         "Brief summary of router status by purpose (v2 directory format)."),
+  PREFIX("consensus/", networkstatus,
+         "Information about and from the ns consensus."),
+  ITEM("network-status", dir,
+       "Brief summary of router status (v1 directory format)"),
+  ITEM("network-liveness", liveness,
+       "Current opinion on whether the network is live"),
+  ITEM("circuit-status", events, "List of current circuits originating here."),
+  ITEM("stream-status", events,"List of current streams."),
+  ITEM("orconn-status", events, "A list of current OR connections."),
+  ITEM("dormant", misc,
+       "Is Tor dormant (not building circuits because it's idle)?"),
+  PREFIX("address-mappings/", events, NULL),
+  DOC("address-mappings/all", "Current address mappings."),
+  DOC("address-mappings/cache", "Current cached DNS replies."),
+  DOC("address-mappings/config",
+      "Current address mappings from configuration."),
+  DOC("address-mappings/control", "Current address mappings from controller."),
+  PREFIX("status/", events, NULL),
+  DOC("status/circuit-established",
+      "Whether we think client functionality is working."),
+  DOC("status/enough-dir-info",
+      "Whether we have enough up-to-date directory information to build "
+      "circuits."),
+  DOC("status/bootstrap-phase",
+      "The last bootstrap phase status event that Tor sent."),
+  DOC("status/clients-seen",
+      "Breakdown of client countries seen by a bridge."),
+  DOC("status/fresh-relay-descs",
+      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
+  DOC("status/version/recommended", "List of currently recommended versions."),
+  DOC("status/version/current", "Status of the current version."),
+  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
+  ITEM("traffic/read", misc,"Bytes read since the process was started."),
+  ITEM("traffic/written", misc,
+       "Bytes written since the process was started."),
+  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
+  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
+  ITEM("process/uid", misc, "User id running the tor process."),
+  ITEM("process/user", misc,
+       "Username under which the tor process is running."),
+  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
+  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
+  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
+  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
+  PREFIX("dir/status/", dir,
+         "v2 networkstatus docs as retrieved from a DirPort."),
+  ITEM("dir/status-vote/current/consensus", dir,
+       "v3 Networkstatus consensus as retrieved from a DirPort."),
+  ITEM("dir/status-vote/current/consensus-microdesc", dir,
+       "v3 Microdescriptor consensus as retrieved from a DirPort."),
+  ITEM("exit-policy/default", policies,
+       "The default value appended to the configured exit policy."),
+  ITEM("exit-policy/reject-private/default", policies,
+       "The default rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate."),
+  ITEM("exit-policy/reject-private/relay", policies,
+       "The relay-specific rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
+  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
+  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
+  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
+  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
+  ITEM("onions/current", onions,
+       "Onion services owned by the current control connection."),
+  ITEM("onions/detached", onions,
+       "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
+  { NULL, NULL, NULL, 0 }
+};
+
+/** Allocate and return a list of recognized GETINFO options. */
+static char *
+list_getinfo_options(void)
+{
+  int i;
+  smartlist_t *lines = smartlist_new();
+  char *ans;
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    if (!getinfo_items[i].desc)
+      continue;
+
+    smartlist_add_asprintf(lines, "%s%s -- %s\n",
+                 getinfo_items[i].varname,
+                 getinfo_items[i].is_prefix ? "*" : "",
+                 getinfo_items[i].desc);
+  }
+  smartlist_sort_strings(lines);
+
+  ans = smartlist_join_strings(lines, "", 0, NULL);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+
+  return ans;
+}
+
+/** Lookup the 'getinfo' entry <b>question</b>, and return
+ * the answer in <b>*answer</b> (or NULL if key not recognized).
+ * Return 0 if success or unrecognized, or -1 if recognized but
+ * internal error. */
+static int
+handle_getinfo_helper(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **err_out)
+{
+  int i;
+  *answer = NULL; /* unrecognized key by default */
+
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    int match;
+    if (getinfo_items[i].is_prefix)
+      match = !strcmpstart(question, getinfo_items[i].varname);
+    else
+      match = !strcmp(question, getinfo_items[i].varname);
+    if (match) {
+      tor_assert(getinfo_items[i].fn);
+      return getinfo_items[i].fn(control_conn, question, answer, err_out);
+    }
+  }
+
+  return 0; /* unrecognized */
+}
+
+const control_cmd_syntax_t getinfo_syntax = {
+  .max_args = UINT_MAX,
+};
+
+/** Called when we receive a GETINFO command.  Try to fetch all requested
+ * information, and reply with information or error message. */
+int
+handle_control_getinfo(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *ans = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    const char *errmsg = NULL;
+
+    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
+      if (!errmsg)
+        errmsg = "Internal error";
+      control_write_endreply(conn, 551, errmsg);
+      goto done;
+    }
+    if (!ans) {
+      if (errmsg) {
+        /* use provided error message */
+        control_reply_add_str(unrecognized, 552, errmsg);
+      } else {
+        /* use default error message */
+        control_reply_add_printf(unrecognized, 552,
+                                 "Unrecognized key \"%s\"", q);
+      }
+    } else {
+      control_reply_add_one_kv(answers, 250, KV_RAW, q, ans);
+      tor_free(ans);
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  control_reply_add_done(answers);
+
+  if (smartlist_len(unrecognized)) {
+    control_write_reply_lines(conn, unrecognized);
+    /* If there were any unrecognized queries, don't write real answers */
+    goto done;
+  }
+
+  control_write_reply_lines(conn, answers);
+
+ done:
+  control_reply_free(answers);
+  control_reply_free(unrecognized);
+
+  return 0;
+}
diff --git a/src/feature/control/control_getinfo.h b/src/feature/control/control_getinfo.h
new file mode 100644
index 0000000000..0ada49258e
--- /dev/null
+++ b/src/feature/control/control_getinfo.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control.h
+ * \brief Header file for control.c.
+ **/
+
+#ifndef TOR_CONTROL_GETINFO_H
+#define TOR_CONTROL_GETINFO_H
+
+struct control_cmd_syntax_t;
+struct control_cmd_args_t;
+extern const struct control_cmd_syntax_t getinfo_syntax;
+
+int handle_control_getinfo(control_connection_t *conn,
+                           const struct control_cmd_args_t *args);
+
+#ifdef CONTROL_GETINFO_PRIVATE
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_networkstatus(
+    const char *flavor,
+    download_status_t **dl_to_emit,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_cert(
+    const char *fp_sk_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_desc(
+    const char *desc_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_bridge(
+    const char *bridge_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC int getinfo_helper_downloads(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_consensus(
+    consensus_flavor_t flavor,
+    char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_time(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+#endif /* defined(CONTROL_GETINFO_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_GETINFO_H) */
diff --git a/src/feature/control/control_hs.c b/src/feature/control/control_hs.c
new file mode 100644
index 0000000000..54b767cd0d
--- /dev/null
+++ b/src/feature/control/control_hs.c
@@ -0,0 +1,354 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_hs.c
+ *
+ * \brief Implement commands for Tor's control-socket interface that are
+ *        related to onion services.
+ **/
+
+#include "core/or/or.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_hs.h"
+#include "feature/control/control_proto.h"
+#include "feature/hs/hs_client.h"
+#include "lib/encoding/confline.h"
+
+#include "feature/control/control_cmd_args_st.h"
+
+/** Parse the 'KeyType ":" PrivateKey' from <b>client_privkey_str</b> and store
+ *  it into <b>privkey</b>. Use <b>conn</b> to output any errors if needed.
+ *
+ *  Return 0 if all went well, -1 otherwise. */
+static int
+parse_private_key_from_control_port(const char *client_privkey_str,
+                                    curve25519_secret_key_t *privkey,
+                                    control_connection_t *conn)
+{
+  int retval = -1;
+  smartlist_t *key_args = smartlist_new();
+
+  tor_assert(privkey);
+
+  smartlist_split_string(key_args, client_privkey_str, ":",
+                         SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(key_args) != 2) {
+    control_printf_endreply(conn, 512, "Invalid key type/blob");
+    goto err;
+  }
+
+  const char *key_type = smartlist_get(key_args, 0);
+  const char *key_blob = smartlist_get(key_args, 1);
+
+  if (strcasecmp(key_type, "x25519")) {
+    control_printf_endreply(conn, 552,
+                            "Unrecognized key type \"%s\"", key_type);
+    goto err;
+  }
+
+  if (base64_decode((char*)privkey->secret_key, sizeof(privkey->secret_key),
+                    key_blob,
+                    strlen(key_blob)) != sizeof(privkey->secret_key)) {
+    control_printf_endreply(conn, 512, "Failed to decode x25519 private key");
+    goto err;
+  }
+
+  if (fast_mem_is_zero((const char*)privkey->secret_key,
+                       sizeof(privkey->secret_key))) {
+    control_printf_endreply(conn, 553,
+                            "Invalid private key \"%s\"", key_blob);
+    goto err;
+  }
+
+  retval = 0;
+
+ err:
+  SMARTLIST_FOREACH(key_args, char *, c, tor_free(c));
+  smartlist_free(key_args);
+  return retval;
+}
+
+/** Syntax details for ONION_CLIENT_AUTH_ADD */
+const control_cmd_syntax_t onion_client_auth_add_syntax = {
+  .max_args = 2,
+  .accept_keywords = true,
+};
+
+/** Called when we get an ONION_CLIENT_AUTH_ADD command; parse the body, and
+ *  register the new client-side client auth credentials:
+ *  "ONION_CLIENT_AUTH_ADD" SP HSAddress
+ *                          SP KeyType ":" PrivateKeyBlob
+ *                          [SP "Type=" TYPE] CRLF
+ */
+int
+handle_control_onion_client_auth_add(control_connection_t *conn,
+                                     const control_cmd_args_t *args)
+{
+  int retval = -1;
+  smartlist_t *flags = smartlist_new();
+  hs_client_service_authorization_t *creds = NULL;
+
+  tor_assert(args);
+
+  int argc = smartlist_len(args->args);
+  /* We need at least 'HSAddress' and 'PrivateKeyBlob' */
+  if (argc < 2) {
+    control_printf_endreply(conn, 512,
+                            "Incomplete ONION_CLIENT_AUTH_ADD command");
+    goto err;
+  }
+
+  creds = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+
+  const char *hsaddress = smartlist_get(args->args, 0);
+  if (!hs_address_is_valid(hsaddress)) {
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",hsaddress);
+    goto err;
+  }
+  strlcpy(creds->onion_address, hsaddress, sizeof(creds->onion_address));
+
+  /* Parse the client private key */
+  const char *client_privkey = smartlist_get(args->args, 1);
+  if (parse_private_key_from_control_port(client_privkey,
+                                          &creds->enc_seckey, conn) < 0) {
+    goto err;
+  }
+
+  /* Now let's parse the remaining arguments (variable size) */
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strcasecmpstart(line->key, "Flags")) {
+      smartlist_split_string(flags, line->value, ",", SPLIT_IGNORE_BLANK, 0);
+      if (smartlist_len(flags) < 1) {
+        control_write_endreply(conn, 512, "Invalid 'Flags' argument");
+        goto err;
+      }
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag) {
+        if (!strcasecmp(flag, "Permanent")) {
+          creds->flags |= CLIENT_AUTH_FLAG_IS_PERMANENT;
+        } else {
+          control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s",
+                                  escaped(flag));
+          goto err;
+        }
+      } SMARTLIST_FOREACH_END(flag);
+    }
+    if (!strcasecmp(line->key, "ClientName")) {
+      if (strlen(line->value) > REND_CLIENTNAME_MAX_LEN) {
+        control_printf_endreply(conn, 512, "ClientName longer than %d chars",
+                                REND_CLIENTNAME_MAX_LEN);
+      }
+      creds->client_name = tor_strdup(line->value);
+    }
+  }
+
+  hs_client_register_auth_status_t register_status;
+  /* Register the credential (register func takes ownership of cred.) */
+  register_status = hs_client_register_auth_credentials(creds);
+  switch (register_status) {
+  case REGISTER_FAIL_BAD_ADDRESS:
+    /* It's a bug because the service addr has already been validated above */
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"", hsaddress);
+    break;
+  case REGISTER_FAIL_PERMANENT_STORAGE:
+    control_printf_endreply(conn, 553, "Unable to store creds for \"%s\"",
+                            hsaddress);
+    break;
+  case REGISTER_SUCCESS_ALREADY_EXISTS:
+    control_printf_endreply(conn, 251,"Client for onion existed and replaced");
+    break;
+  case REGISTER_SUCCESS_AND_DECRYPTED:
+    control_printf_endreply(conn, 252,"Registered client and decrypted desc");
+    break;
+  case REGISTER_SUCCESS:
+    control_printf_endreply(conn, 250, "OK");
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+  }
+
+  retval = 0;
+  goto done;
+
+ err:
+  client_service_authorization_free(creds);
+
+ done:
+  SMARTLIST_FOREACH(flags, char *, s, tor_free(s));
+  smartlist_free(flags);
+  return retval;
+}
+
+/** Syntax details for ONION_CLIENT_AUTH_REMOVE */
+const control_cmd_syntax_t onion_client_auth_remove_syntax = {
+  .max_args = 1,
+  .accept_keywords = true,
+};
+
+/** Called when we get an ONION_CLIENT_AUTH_REMOVE command; parse the body, and
+ *  register the new client-side client auth credentials.
+ *    "ONION_CLIENT_AUTH_REMOVE" SP HSAddress
+ */
+int
+handle_control_onion_client_auth_remove(control_connection_t *conn,
+                                        const control_cmd_args_t *args)
+{
+  int retval = -1;
+
+  tor_assert(args);
+
+  int argc = smartlist_len(args->args);
+  if (argc < 1) {
+    control_printf_endreply(conn, 512,
+                            "Incomplete ONION_CLIENT_AUTH_REMOVE command");
+    goto err;
+  }
+
+  const char *hsaddress = smartlist_get(args->args, 0);
+  if (!hs_address_is_valid(hsaddress)) {
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",hsaddress);
+    goto err;
+  }
+
+  hs_client_removal_auth_status_t removal_status;
+  removal_status = hs_client_remove_auth_credentials(hsaddress);
+  switch (removal_status) {
+  case REMOVAL_BAD_ADDRESS:
+    /* It's a bug because the service addr has already been validated above */
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",hsaddress);
+    break;
+  case REMOVAL_SUCCESS_NOT_FOUND:
+    control_printf_endreply(conn, 251, "No credentials for \"%s\"",hsaddress);
+    break;
+  case REMOVAL_SUCCESS:
+    control_printf_endreply(conn, 250, "OK");
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+  }
+
+  retval = 0;
+
+ err:
+  return retval;
+}
+
+/** Helper: Return a newly allocated string with the encoding of client
+ *  authorization credentials */
+static char *
+encode_client_auth_cred_for_control_port(
+                                       hs_client_service_authorization_t *cred)
+{
+  smartlist_t *control_line = smartlist_new();
+  char x25519_b64[128];
+  char *msg_str = NULL;
+
+  tor_assert(cred);
+
+  if (base64_encode(x25519_b64, sizeof(x25519_b64),
+                    (char *)cred->enc_seckey.secret_key,
+                    sizeof(cred->enc_seckey.secret_key), 0) < 0) {
+    tor_assert_nonfatal_unreached();
+    goto err;
+  }
+
+  smartlist_add_asprintf(control_line, "CLIENT %s x25519:%s",
+                         cred->onion_address, x25519_b64);
+
+  if (cred->flags) { /* flags are also optional */
+    if (cred->flags & CLIENT_AUTH_FLAG_IS_PERMANENT) {
+      smartlist_add_asprintf(control_line, " Flags=Permanent");
+    }
+  }
+
+  if (cred->client_name) {
+    smartlist_add_asprintf(control_line, " ClientName=%s", cred->client_name);
+  }
+
+  /* Join all the components into a single string */
+  msg_str = smartlist_join_strings(control_line, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(control_line, char *, cp, tor_free(cp));
+  smartlist_free(control_line);
+
+  return msg_str;
+}
+
+/** Syntax details for ONION_CLIENT_AUTH_VIEW */
+const control_cmd_syntax_t onion_client_auth_view_syntax = {
+  .max_args = 1,
+  .accept_keywords = true,
+};
+
+/** Called when we get an ONION_CLIENT_AUTH_VIEW command; parse the body, and
+ *  register the new client-side client auth credentials.
+ *        "ONION_CLIENT_AUTH_VIEW" [SP HSAddress] CRLF
+ */
+int
+handle_control_onion_client_auth_view(control_connection_t *conn,
+                                      const control_cmd_args_t *args)
+{
+  int retval = -1;
+  const char *hsaddress = NULL;
+  /* We are gonna put all the credential strings into a smartlist, and sort it
+     before printing, so that we can get a guaranteed order of printing. */
+  smartlist_t *creds_str_list = smartlist_new();
+
+  tor_assert(args);
+
+  int argc = smartlist_len(args->args);
+  if (argc >= 1) {
+    hsaddress = smartlist_get(args->args, 0);
+    if (!hs_address_is_valid(hsaddress)) {
+      control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",
+                              hsaddress);
+      goto err;
+    }
+  }
+
+  if (hsaddress) {
+    control_printf_midreply(conn, 250, "ONION_CLIENT_AUTH_VIEW %s", hsaddress);
+  } else {
+    control_printf_midreply(conn, 250, "ONION_CLIENT_AUTH_VIEW");
+  }
+
+  /* Create an iterator out of the digest256map */
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  digest256map_iter_t *itr = digest256map_iter_init(client_auths);
+  while (!digest256map_iter_done(itr)) {
+    const uint8_t *service_pubkey;
+    void *valp;
+    digest256map_iter_get(itr, &service_pubkey, &valp);
+    tor_assert(valp);
+    hs_client_service_authorization_t *cred = valp;
+
+    /* If a specific HS address was requested, only print creds for that one */
+    if (hsaddress && strcmp(cred->onion_address, hsaddress)) {
+      itr = digest256map_iter_next(client_auths, itr);
+      continue;
+    }
+
+    char *encoding_str = encode_client_auth_cred_for_control_port(cred);
+    tor_assert_nonfatal(encoding_str);
+    smartlist_add(creds_str_list, encoding_str);
+
+    itr = digest256map_iter_next(client_auths, itr);
+  }
+
+  /* We got everything: Now sort the strings and print them */
+  smartlist_sort_strings(creds_str_list);
+  SMARTLIST_FOREACH_BEGIN(creds_str_list, char *, c) {
+    control_printf_midreply(conn, 250, "%s", c);
+  } SMARTLIST_FOREACH_END(c);
+
+  send_control_done(conn);
+
+  retval = 0;
+
+ err:
+  SMARTLIST_FOREACH(creds_str_list, char *, cp, tor_free(cp));
+  smartlist_free(creds_str_list);
+  return retval;
+}
diff --git a/src/feature/control/control_hs.h b/src/feature/control/control_hs.h
new file mode 100644
index 0000000000..8a0cd6818d
--- /dev/null
+++ b/src/feature/control/control_hs.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_hs.c
+ *
+ * \brief Header file for control_hs.c.
+ **/
+
+#ifndef TOR_CONTROL_HS_H
+#define TOR_CONTROL_HS_H
+
+struct control_connection_t;
+struct control_cmd_syntax_t;
+struct control_cmd_args_t;
+
+extern const struct control_cmd_syntax_t onion_client_auth_add_syntax;
+extern const struct control_cmd_syntax_t onion_client_auth_remove_syntax;
+extern const struct control_cmd_syntax_t onion_client_auth_view_syntax;
+
+int
+handle_control_onion_client_auth_add(struct control_connection_t *conn,
+                                     const struct control_cmd_args_t *args);
+
+int
+handle_control_onion_client_auth_remove(struct control_connection_t *conn,
+                                        const struct control_cmd_args_t *args);
+
+int
+handle_control_onion_client_auth_view(struct control_connection_t *conn,
+                                      const struct control_cmd_args_t *args);
+
+#endif /* !defined(TOR_CONTROL_HS_H) */
diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c
new file mode 100644
index 0000000000..98715ad9d5
--- /dev/null
+++ b/src/feature/control/control_proto.c
@@ -0,0 +1,434 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/kvline.h"
+
+/** Append a NUL-terminated string <b>s</b> to the end of
+ * <b>conn</b>-\>outbuf.
+ */
+void
+connection_write_str_to_buf(const char *s, control_connection_t *conn)
+{
+  size_t len = strlen(s);
+  connection_buf_add(s, len, TO_CONN(conn));
+}
+
+/** Acts like sprintf, but writes its formatted string to the end of
+ * <b>conn</b>-\>outbuf. */
+void
+connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
+{
+  va_list ap;
+  char *buf = NULL;
+  int len;
+
+  va_start(ap,format);
+  len = tor_vasprintf(&buf, format, ap);
+  va_end(ap);
+
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
+
+  tor_free(buf);
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
+ * contents of <b>data</b> into *<b>out</b>, adding a period before any period
+ * that appears at the start of a line, and adding a period-CRLF line at
+ * the end. Replace all LF characters sequences with CRLF.  Return the number
+ * of bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+write_escaped_data(const char *data, size_t len, char **out)
+{
+  tor_assert(len < SIZE_MAX - 9);
+  size_t sz_out = len+8+1;
+  char *outp;
+  const char *start = data, *end;
+  size_t i;
+  int start_of_line;
+  for (i=0; i < len; ++i) {
+    if (data[i] == '\n') {
+      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
+      if (sz_out >= SIZE_T_CEILING) {
+        log_warn(LD_BUG, "Input to write_escaped_data was too long");
+        *out = tor_strdup(".\r\n");
+        return 3;
+      }
+    }
+  }
+  *out = outp = tor_malloc(sz_out);
+  end = data+len;
+  start_of_line = 1;
+  while (data < end) {
+    if (*data == '\n') {
+      if (data > start && data[-1] != '\r')
+        *outp++ = '\r';
+      start_of_line = 1;
+    } else if (*data == '.') {
+      if (start_of_line) {
+        start_of_line = 0;
+        *outp++ = '.';
+      }
+    } else {
+      start_of_line = 0;
+    }
+    *outp++ = *data++;
+  }
+  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
+    *outp++ = '\r';
+    *outp++ = '\n';
+  }
+  *outp++ = '.';
+  *outp++ = '\r';
+  *outp++ = '\n';
+  *outp = '\0'; /* NUL-terminate just in case. */
+  tor_assert(outp >= *out);
+  tor_assert((size_t)(outp - *out) <= sz_out);
+  return outp - *out;
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
+ * the contents of <b>data</b> into *<b>out</b>, removing any period
+ * that appears at the start of a line, and replacing all CRLF sequences
+ * with LF.   Return the number of
+ * bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+read_escaped_data(const char *data, size_t len, char **out)
+{
+  char *outp;
+  const char *next;
+  const char *end;
+
+  *out = outp = tor_malloc(len+1);
+
+  end = data+len;
+
+  while (data < end) {
+    /* we're at the start of a line. */
+    if (*data == '.')
+      ++data;
+    next = memchr(data, '\n', end-data);
+    if (next) {
+      size_t n_to_copy = next-data;
+      /* Don't copy a CR that precedes this LF. */
+      if (n_to_copy && *(next-1) == '\r')
+        --n_to_copy;
+      memcpy(outp, data, n_to_copy);
+      outp += n_to_copy;
+      data = next+1; /* This will point at the start of the next line,
+                      * or the end of the string, or a period. */
+    } else {
+      memcpy(outp, data, end-data);
+      outp += (end-data);
+      *outp = '\0';
+      return outp - *out;
+    }
+    *outp++ = '\n';
+  }
+
+  *outp = '\0';
+  return outp - *out;
+}
+
+/** Send a "DONE" message down the control connection <b>conn</b>. */
+void
+send_control_done(control_connection_t *conn)
+{
+  control_write_endreply(conn, 250, "OK");
+}
+
+/** Write a reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param s string reply content
+ */
+MOCK_IMPL(void,
+control_write_reply, (control_connection_t *conn, int code, int c,
+                      const char *s))
+{
+  connection_printf_to_buf(conn, "%03d%c%s\r\n", code, c, s);
+}
+
+/** Write a formatted reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param fmt format string
+ * @param ap va_list from caller
+ */
+void
+control_vprintf_reply(control_connection_t *conn, int code, int c,
+                      const char *fmt, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, fmt, ap);
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+  control_write_reply(conn, code, c, buf);
+  tor_free(buf);
+}
+
+/** Write an EndReplyLine */
+void
+control_write_endreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, ' ', s);
+}
+
+/** Write a formatted EndReplyLine */
+void
+control_printf_endreply(control_connection_t *conn, int code,
+                        const char *fmt, ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, ' ', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a MidReplyLine */
+void
+control_write_midreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '-', s);
+}
+
+/** Write a formatted MidReplyLine */
+void
+control_printf_midreply(control_connection_t *conn, int code, const char *fmt,
+                        ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '-', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a DataReplyLine */
+void
+control_write_datareply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '+', s);
+}
+
+/** Write a formatted DataReplyLine */
+void
+control_printf_datareply(control_connection_t *conn, int code, const char *fmt,
+                         ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '+', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a CmdData */
+void
+control_write_data(control_connection_t *conn, const char *data)
+{
+  char *esc = NULL;
+  size_t esc_len;
+
+  esc_len = write_escaped_data(data, strlen(data), &esc);
+  connection_buf_add(esc, esc_len, TO_CONN(conn));
+  tor_free(esc);
+}
+
+/** Write a single reply line to @a conn.
+ *
+ * @param conn control connection
+ * @param line control reply line to write
+ * @param lastone true if this is the last reply line of a multi-line reply
+ */
+void
+control_write_reply_line(control_connection_t *conn,
+                         const control_reply_line_t *line, bool lastone)
+{
+  const config_line_t *kvline = line->kvline;
+  char *s = NULL;
+
+  if (strpbrk(kvline->value, "\r\n") != NULL) {
+    /* If a key-value pair needs to be encoded as CmdData, it can be
+       the only key-value pair in that reply line */
+    tor_assert(kvline->next == NULL);
+    control_printf_datareply(conn, line->code, "%s=", kvline->key);
+    control_write_data(conn, kvline->value);
+    return;
+  }
+  s = kvline_encode(kvline, line->flags);
+  if (lastone) {
+    control_write_endreply(conn, line->code, s);
+  } else {
+    control_write_midreply(conn, line->code, s);
+  }
+  tor_free(s);
+}
+
+/** Write a set of reply lines to @a conn.
+ *
+ * @param conn control connection
+ * @param lines smartlist of pointers to control_reply_line_t to write
+ */
+void
+control_write_reply_lines(control_connection_t *conn, smartlist_t *lines)
+{
+  bool lastone = false;
+
+  SMARTLIST_FOREACH_BEGIN(lines, control_reply_line_t *, line) {
+    if (line_sl_idx >= line_sl_len - 1)
+      lastone = true;
+    control_write_reply_line(conn, line, lastone);
+  } SMARTLIST_FOREACH_END(line);
+}
+
+/** Add a single key-value pair as a new reply line to a control reply
+ * line list.
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param code numeric control reply code
+ * @param flags kvline encoding flags
+ * @param key key
+ * @param val value
+ */
+void
+control_reply_add_one_kv(smartlist_t *reply, int code, int flags,
+                         const char *key, const char *val)
+{
+  control_reply_line_t *line = tor_malloc_zero(sizeof(*line));
+
+  line->code = code;
+  line->flags = flags;
+  config_line_append(&line->kvline, key, val);
+  smartlist_add(reply, line);
+}
+
+/** Append a single key-value pair to last reply line in a control
+ * reply line list.
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param key key
+ * @param val value
+ */
+void
+control_reply_append_kv(smartlist_t *reply, const char *key, const char *val)
+{
+  int len = smartlist_len(reply);
+  control_reply_line_t *line;
+
+  tor_assert(len > 0);
+
+  line = smartlist_get(reply, len - 1);
+  config_line_append(&line->kvline, key, val);
+}
+
+/** Add new reply line consisting of the string @a s
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param code numeric control reply code
+ * @param s string containing the rest of the reply line
+ */
+void
+control_reply_add_str(smartlist_t *reply, int code, const char *s)
+{
+  control_reply_add_one_kv(reply, code, KV_OMIT_KEYS|KV_RAW, "", s);
+}
+
+/** Format a new reply line
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param code numeric control reply code
+ * @param fmt format string
+ */
+void
+control_reply_add_printf(smartlist_t *reply, int code, const char *fmt, ...)
+{
+  va_list ap;
+  char *buf = NULL;
+
+  va_start(ap, fmt);
+  (void)tor_vasprintf(&buf, fmt, ap);
+  va_end(ap);
+  control_reply_add_str(reply, code, buf);
+  tor_free(buf);
+}
+
+/** Add a "250 OK" line to a set of control reply lines */
+void
+control_reply_add_done(smartlist_t *reply)
+{
+  control_reply_add_str(reply, 250, "OK");
+}
+
+/** Free a control_reply_line_t.  Don't call this directly; use the
+ * control_reply_line_free() macro instead. */
+void
+control_reply_line_free_(control_reply_line_t *line)
+{
+  if (!line)
+    return;
+  config_free_lines(line->kvline);
+  tor_free_(line);
+}
+
+/** Clear a smartlist of control_reply_line_t.  Doesn't free the
+ * smartlist, but does free each individual line. */
+void
+control_reply_clear(smartlist_t *reply)
+{
+  SMARTLIST_FOREACH(reply, control_reply_line_t *, line,
+                    control_reply_line_free(line));
+  smartlist_clear(reply);
+}
+
+/** Free a smartlist of control_reply_line_t. Don't call this
+ * directly; use the control_reply_free() macro instead. */
+void
+control_reply_free_(smartlist_t *reply)
+{
+  control_reply_clear(reply);
+  smartlist_free_(reply);
+}
diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h
new file mode 100644
index 0000000000..4c32b820d1
--- /dev/null
+++ b/src/feature/control/control_proto.h
@@ -0,0 +1,120 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.h
+ * \brief Header file for control_proto.c.
+ *
+ * See @ref replylines for details about the key-value abstraction for
+ * generating reply lines.
+ **/
+
+#ifndef TOR_CONTROL_PROTO_H
+#define TOR_CONTROL_PROTO_H
+
+#include "lib/encoding/confline.h"
+
+/**
+ * @defgroup replylines Control reply lines
+ * @brief Key-value structures for control reply lines
+ *
+ * Control reply lines are config_line_t key-value structures with
+ * some additional information to help formatting, such as the numeric
+ * result code specified in the control protocol and flags affecting
+ * the way kvline_encode() formats the @a kvline.
+ *
+ * Generally, modules implementing control commands will work with
+ * smartlists of these structures, using functions like
+ * control_reply_add_str() for adding a reply line consisting of a
+ * single string, or control_reply_add_one_kv() and
+ * control_reply_append_kv() for composing a line containing one or
+ * more key-value pairs.
+ *
+ * @{
+ */
+/** @brief A reply line for the control protocol.
+ *
+ * This wraps config_line_t with some additional information that's
+ * useful when generating control reply lines.
+ */
+typedef struct control_reply_line_t {
+  int code;                     /**< numeric code */
+  int flags;                    /**< kvline encoding flags */
+  config_line_t *kvline;        /**< kvline */
+} control_reply_line_t;
+
+void control_reply_line_free_(control_reply_line_t *line);
+/**
+ * @brief Free and null a control_reply_line_t
+ *
+ * @param line pointer to control_reply_line_t to free
+ */
+#define control_reply_line_free(line)                   \
+  FREE_AND_NULL(control_reply_line_t,                   \
+                control_reply_line_free_, (line))
+/** @} */
+
+void connection_write_str_to_buf(const char *s, control_connection_t *conn);
+void connection_printf_to_buf(control_connection_t *conn,
+                                     const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+size_t write_escaped_data(const char *data, size_t len, char **out);
+size_t read_escaped_data(const char *data, size_t len, char **out);
+void send_control_done(control_connection_t *conn);
+
+MOCK_DECL(void, control_write_reply, (control_connection_t *conn, int code,
+                                      int c, const char *s));
+void control_vprintf_reply(control_connection_t *conn, int code, int c,
+                           const char *fmt, va_list ap)
+  CHECK_PRINTF(4, 0);
+void control_write_endreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_endreply(control_connection_t *conn, int code,
+                             const char *fmt, ...)
+  CHECK_PRINTF(3, 4);
+void control_write_midreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_midreply(control_connection_t *conn, int code,
+                             const char *fmt,
+                             ...)
+  CHECK_PRINTF(3, 4);
+void control_write_datareply(control_connection_t *conn, int code,
+                             const char *s);
+void control_printf_datareply(control_connection_t *conn, int code,
+                              const char *fmt,
+                              ...)
+  CHECK_PRINTF(3, 4);
+void control_write_data(control_connection_t *conn, const char *data);
+
+/** @addtogroup replylines
+ * @{
+ */
+void control_write_reply_line(control_connection_t *conn,
+                              const control_reply_line_t *line, bool lastone);
+void control_write_reply_lines(control_connection_t *conn, smartlist_t *lines);
+
+void control_reply_add_one_kv(smartlist_t *reply, int code, int flags,
+                              const char *key, const char *val);
+void control_reply_append_kv(smartlist_t *reply, const char *key,
+                             const char *val);
+void control_reply_add_str(smartlist_t *reply, int code, const char *s);
+void control_reply_add_printf(smartlist_t *reply, int code,
+                              const char *fmt, ...)
+  CHECK_PRINTF(3, 4);
+void control_reply_add_done(smartlist_t *reply);
+
+void control_reply_clear(smartlist_t *reply);
+void control_reply_free_(smartlist_t *reply);
+
+/** @brief Free and null a smartlist of control_reply_line_t.
+ *
+ * @param r pointer to smartlist_t of control_reply_line_t to free */
+#define control_reply_free(r) \
+  FREE_AND_NULL(smartlist_t, control_reply_free_, (r))
+/** @} */
+
+#endif /* !defined(TOR_CONTROL_PROTO_H) */
diff --git a/src/feature/control/feature_control.md b/src/feature/control/feature_control.md
new file mode 100644
index 0000000000..9f1681ea91
--- /dev/null
+++ b/src/feature/control/feature_control.md
@@ -0,0 +1,8 @@
+@dir /feature/control
+@brief feature/control: Controller API.
+
+The Controller API is a text-based protocol that another program (or another
+thread, if you're running Tor in-process) can use to configure and control
+Tor while it is running.  The current protocol is documented in
+[control-spec.txt](https://gitweb.torproject.org/torspec.git/tree/control-spec.txt).
+
diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c
index a1ddd2119a..ed9ad95ce2 100644
--- a/src/feature/control/fmt_serverstatus.c
+++ b/src/feature/control/fmt_serverstatus.c
@@ -1,16 +1,21 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file fmt_serverstatus.c
+ * @brief Format relay info for a controller.
+ **/
+
 #include "core/or/or.h"
 #include "feature/control/fmt_serverstatus.h"
 
 #include "app/config/config.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/voteflags.h"// XXXX remove
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -66,11 +71,9 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   smartlist_t *rs_entries;
   time_t now = time(NULL);
   time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  const or_options_t *options = get_options();
   /* We include v2 dir auths here too, because they need to answer
    * controllers. Eventually we'll deprecate this whole function;
    * see also networkstatus_getinfo_by_purpose(). */
-  int authdir = authdir_mode_publishes_statuses(options);
   tor_assert(router_status_out);
 
   rs_entries = smartlist_new();
@@ -78,10 +81,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
     const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
     tor_assert(node);
-    if (authdir) {
-      /* Update router status in routerinfo_t. */
-      dirserv_set_router_is_running(ri, now);
-    }
     if (for_controller) {
       char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
       char *cp = name_buf;
diff --git a/src/feature/control/fmt_serverstatus.h b/src/feature/control/fmt_serverstatus.h
index 4b95e5b59f..9dd9fe125c 100644
--- a/src/feature/control/fmt_serverstatus.h
+++ b/src/feature/control/fmt_serverstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,4 +15,4 @@
 int list_server_status_v1(smartlist_t *routers, char **router_status_out,
                           int for_controller);
 
-#endif
+#endif /* !defined(TOR_FMT_SERVERSTATUS_H) */
diff --git a/src/feature/control/getinfo_geoip.c b/src/feature/control/getinfo_geoip.c
index d188725fa3..33019207e6 100644
--- a/src/feature/control/getinfo_geoip.c
+++ b/src/feature/control/getinfo_geoip.c
@@ -1,3 +1,12 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file getinfo_geoip.c
+ * @brief GEOIP-related contoller GETINFO commands.
+ **/
 
 #include "core/or/or.h"
 #include "core/mainloop/connection.h"
diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h
index fe22137859..5bc4b08414 100644
--- a/src/feature/control/getinfo_geoip.h
+++ b/src/feature/control/getinfo_geoip.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file getinfo_geoip.h
+ * @brief Header for getinfo_geoip.c
+ **/
+
 #ifndef TOR_GETINFO_GEOIP_H
 #define TOR_GETINFO_GEOIP_H
 
@@ -11,4 +16,4 @@ int getinfo_helper_geoip(control_connection_t *control_conn,
                      const char *question, char **answer,
                      const char **errmsg);
 
-#endif
+#endif /* !defined(TOR_GETINFO_GEOIP_H) */
diff --git a/src/feature/control/include.am b/src/feature/control/include.am
new file mode 100644
index 0000000000..07094f23bb
--- /dev/null
+++ b/src/feature/control/include.am
@@ -0,0 +1,39 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/control/btrack.c		\
+	src/feature/control/btrack_circuit.c	\
+	src/feature/control/btrack_orconn.c	\
+	src/feature/control/btrack_orconn_cevent.c	\
+	src/feature/control/btrack_orconn_maps.c	\
+	src/feature/control/control.c		\
+	src/feature/control/control_auth.c	\
+	src/feature/control/control_bootstrap.c	\
+	src/feature/control/control_cmd.c	\
+	src/feature/control/control_hs.c	\
+	src/feature/control/control_events.c	\
+	src/feature/control/control_fmt.c	\
+	src/feature/control/control_getinfo.c	\
+	src/feature/control/control_proto.c	\
+	src/feature/control/fmt_serverstatus.c  \
+	src/feature/control/getinfo_geoip.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/control/btrack_circuit.h		\
+	src/feature/control/btrack_orconn.h		\
+	src/feature/control/btrack_orconn_cevent.h	\
+	src/feature/control/btrack_orconn_maps.h	\
+	src/feature/control/btrack_sys.h		\
+	src/feature/control/control.h			\
+	src/feature/control/control_auth.h		\
+	src/feature/control/control_cmd.h		\
+	src/feature/control/control_hs.h		\
+	src/feature/control/control_cmd_args_st.h	\
+	src/feature/control/control_connection_st.h	\
+	src/feature/control/control_events.h		\
+	src/feature/control/control_fmt.h		\
+	src/feature/control/control_getinfo.h		\
+	src/feature/control/control_proto.h		\
+	src/feature/control/fmt_serverstatus.h		\
+	src/feature/control/getinfo_geoip.h
diff --git a/src/feature/dirauth/.may_include b/src/feature/dirauth/.may_include
new file mode 100644
index 0000000000..a9bb274699
--- /dev/null
+++ b/src/feature/dirauth/.may_include
@@ -0,0 +1,2 @@
+*.h
+feature/dirauth/*.inc
diff --git a/src/feature/dirauth/authmode.c b/src/feature/dirauth/authmode.c
index 29fcc6d1a9..0fde7bc679 100644
--- a/src/feature/dirauth/authmode.c
+++ b/src/feature/dirauth/authmode.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,6 +26,15 @@ authdir_mode(const or_options_t *options)
 {
   return options->AuthoritativeDir != 0;
 }
+
+/* Return true iff we believe ourselves to be a v3 authoritative directory
+ * server. */
+int
+authdir_mode_v3(const or_options_t *options)
+{
+  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
+}
+
 /** Return true iff we are an authoritative directory server that is
  * authoritative about receiving and serving descriptors of type
  * <b>purpose</b> on its dirport.
diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h
index 876a1f947b..6e6ba7f8ae 100644
--- a/src/feature/dirauth/authmode.h
+++ b/src/feature/dirauth/authmode.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,22 +14,16 @@
 #ifdef HAVE_MODULE_DIRAUTH
 
 int authdir_mode(const or_options_t *options);
+int authdir_mode_v3(const or_options_t *options);
 int authdir_mode_handles_descs(const or_options_t *options, int purpose);
 int authdir_mode_publishes_statuses(const or_options_t *options);
 int authdir_mode_tests_reachability(const or_options_t *options);
 int authdir_mode_bridge(const or_options_t *options);
 
-/* Return true iff we believe ourselves to be a v3 authoritative directory
- * server. */
-static inline int
-authdir_mode_v3(const or_options_t *options)
-{
-  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
-}
-
+/* Is the dirauth module enabled? */
 #define have_module_dirauth() (1)
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 #define authdir_mode(options) (((void)(options)),0)
 #define authdir_mode_handles_descs(options,purpose) \
@@ -41,6 +35,6 @@ authdir_mode_v3(const or_options_t *options)
 
 #define have_module_dirauth() (0)
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif /* TOR_MODE_H */
+#endif /* !defined(TOR_DIRAUTH_MODE_H) */
diff --git a/src/feature/dirauth/bridgeauth.c b/src/feature/dirauth/bridgeauth.c
new file mode 100644
index 0000000000..b7bf3e4e04
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.c
@@ -0,0 +1,60 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file bridgeauth.c
+ * @brief Bridge authority code
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/voteflags.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Write out router status entries for all our bridge descriptors. Here, we
+ * also mark routers as running. */
+void
+bridgeauth_dump_bridge_status_to_file(time_t now)
+{
+  char *status;
+  char *fname = NULL;
+  char *thresholds = NULL;
+  char *published_thresholds_and_status = NULL;
+  char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
+
+  dirserv_set_bridges_running(now);
+  status = networkstatus_getinfo_by_purpose("bridge", now);
+
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
+  format_iso_time(published, now);
+  dirserv_compute_bridge_flag_thresholds();
+  thresholds = dirserv_get_flag_thresholds_line();
+  tor_asprintf(&published_thresholds_and_status,
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
+  if (write_str_to_file(fname,published_thresholds_and_status,0)<0) {
+    log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file.");
+  }
+  tor_free(thresholds);
+  tor_free(published_thresholds_and_status);
+  tor_free(fname);
+  tor_free(status);
+  tor_free(fingerprint_line);
+}
diff --git a/src/feature/dirauth/bridgeauth.h b/src/feature/dirauth/bridgeauth.h
new file mode 100644
index 0000000000..382d1cfcb8
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file bridgeauth.h
+ * @brief Header for bridgeauth.c
+ **/
+
+#ifndef TOR_DIRAUTH_BRIDGEAUTH_H
+#define TOR_DIRAUTH_BRIDGEAUTH_H
+
+void bridgeauth_dump_bridge_status_to_file(time_t now);
+
+#endif /* !defined(TOR_DIRAUTH_BRIDGEAUTH_H) */
diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c
index 12f9399e9f..ff0c78f018 100644
--- a/src/feature/dirauth/bwauth.c
+++ b/src/feature/dirauth/bwauth.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,13 +13,16 @@
 #include "feature/dirauth/bwauth.h"
 
 #include "app/config/config.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/dirparse/ns_parse.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
+#include "lib/crypt_ops/crypto_format.h"
 #include "lib/encoding/keyval.h"
 
 /** Total number of routers with measured bandwidth; this is set by
@@ -55,7 +58,7 @@ dirserv_get_last_n_measured_bws(void)
 }
 
 /** Measured bandwidth cache entry */
-typedef struct mbw_cache_entry_s {
+typedef struct mbw_cache_entry_t {
   long mbw_kb;
   time_t as_of;
 } mbw_cache_entry_t;
@@ -181,7 +184,7 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
   /* Check if we have a measured bandwidth, and check the threshold if not */
   if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
                                        &mbw_kb, NULL))) {
-    threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+    threshold = dirauth_get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
     if (routers_with_measured_bw > threshold) {
       /* Return zero for unmeasured bandwidth if we are above threshold */
       bw_kb = 0;
@@ -198,14 +201,38 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
 }
 
 /**
- * Read the measured bandwidth list file, apply it to the list of
- * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
+ * Read the measured bandwidth list <b>from_file</b>:
+ * - store all the headers in <b>bw_file_headers</b>,
+ * - apply bandwidth lines to the list of vote_routerstatus_t in
+ *   <b>routerstatuses</b>,
+ * - cache bandwidth lines for dirserv_get_bandwidth_for_router(),
+ * - expire old entries in the measured bandwidth cache, and
+ * - store the DIGEST_SHA256 of the contents of the file in <b>digest_out</b>.
+ *
  * Returns -1 on error, 0 otherwise.
+ *
+ * If the file can't be read, or is empty:
+ * - <b>bw_file_headers</b> is empty,
+ * - <b>routerstatuses</b> is not modified,
+ * - the measured bandwidth cache is not modified, and
+ * - <b>digest_out</b> is the zero-byte digest.
+ *
+ * Otherwise, if there is an error later in the file:
+ * - <b>bw_file_headers</b> contains all the headers up to the error,
+ * - <b>routerstatuses</b> is updated with all the relay lines up to the error,
+ * - the measured bandwidth cache is updated with all the relay lines up to
+ *   the error,
+ * - if the timestamp is valid and recent, old entries in the  measured
+ *   bandwidth cache are expired, and
+ * - <b>digest_out</b> is the digest up to the first read error (if any).
+ *   The digest is taken over all the readable file contents, even if the
+ *   file is outdated or unparseable.
  */
 int
 dirserv_read_measured_bandwidths(const char *from_file,
                                  smartlist_t *routerstatuses,
-                                 smartlist_t *bw_file_headers)
+                                 smartlist_t *bw_file_headers,
+                                 uint8_t *digest_out)
 {
   FILE *fp = tor_fopen_cloexec(from_file, "r");
   int applied_lines = 0;
@@ -219,8 +246,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   int rv = -1;
   char *line = NULL;
   size_t n = 0;
-
-  /* Initialise line, so that we can't possibly run off the end. */
+  crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256);
 
   if (fp == NULL) {
     log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
@@ -228,16 +254,18 @@ dirserv_read_measured_bandwidths(const char *from_file,
     goto err;
   }
 
-  /* If fgets fails, line is either unmodified, or indeterminate. */
   if (tor_getline(&line,&n,fp) <= 0) {
     log_warn(LD_DIRSERV, "Empty bandwidth file");
     goto err;
   }
+  /* If the line could be gotten, add it to the digest */
+  crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
 
   if (!strlen(line) || line[strlen(line)-1] != '\n') {
     log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    /* Continue adding lines to the digest. */
+    goto continue_digest;
   }
 
   line[strlen(line)-1] = '\0';
@@ -245,14 +273,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
   if (!ok) {
     log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    goto continue_digest;
   }
 
-  now = time(NULL);
+  now = approx_time();
   if ((now - file_time) > MAX_MEASUREMENT_AGE) {
     log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
              (unsigned)(time(NULL) - file_time));
-    goto err;
+    goto continue_digest;
   }
 
   /* If timestamp was correct and bw_file_headers is not NULL,
@@ -267,6 +295,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   while (!feof(fp)) {
     measured_bw_line_t parsed_line;
     if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
       if (measured_bw_line_parse(&parsed_line, line,
                                  line_is_after_headers) != -1) {
         /* This condition will be true when the first complete valid bw line
@@ -305,6 +334,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
            "Applied %d measurements.", applied_lines);
   rv = 0;
 
+ continue_digest:
+  /* Continue parsing lines to return the digest of the Bandwidth File. */
+  while (!feof(fp)) {
+    if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
+    }
+  }
+
  err:
   if (line) {
     // we need to raw_free this buffer because we got it from tor_getdelim()
@@ -312,6 +349,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
   }
   if (fp)
     fclose(fp);
+  if (digest_out)
+    crypto_digest_get_digest(digest, (char *) digest_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
   return rv;
 }
 
@@ -327,6 +367,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
  * the header block yet. If we encounter an incomplete bw line, return -1 but
  * don't warn since there could be additional header lines coming. If we
  * encounter a proper bw line, return 0 (and we got past the headers).
+ *
+ * If the line contains "vote=0", stop parsing it, and return -1, so that the
+ * line is ignored during voting.
  */
 STATIC int
 measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h
index 4507728458..849c58e2fc 100644
--- a/src/feature/dirauth/bwauth.h
+++ b/src/feature/dirauth/bwauth.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,8 +21,8 @@
 
 int dirserv_read_measured_bandwidths(const char *from_file,
                                      smartlist_t *routerstatuses,
-                                     smartlist_t *bw_file_headers);
-
+                                     smartlist_t *bw_file_headers,
+                                     uint8_t *digest_out);
 int dirserv_query_measured_bw_cache_kb(const char *node_id,
                                        long *bw_out,
                                        time_t *as_of_out);
@@ -55,4 +55,4 @@ STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
 STATIC void dirserv_expire_measured_bw_cache(time_t now);
 #endif /* defined(BWAUTH_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_BWAUTH_H) */
diff --git a/src/feature/dirauth/dirauth_config.c b/src/feature/dirauth/dirauth_config.c
new file mode 100644
index 0000000000..a0b6de7eca
--- /dev/null
+++ b/src/feature/dirauth/dirauth_config.c
@@ -0,0 +1,471 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_config.c
+ * @brief Code to interpret the user's configuration of Tor's directory
+ *        authority module.
+ **/
+
+#include "orconfig.h"
+#include "feature/dirauth/dirauth_config.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confmgt.h"
+#include "lib/conf/confdecl.h"
+
+/* Required for dirinfo_type_t in or_options_t */
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/resolve_addr.h"
+
+#include "feature/dirauth/voting_schedule.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/guardfraction.h"
+#include "feature/dirauth/dirauth_options_st.h"
+
+/* Copied from config.c, we will refactor later in 29211. */
+#define REJECT(arg) \
+  STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
+#if defined(__GNUC__) && __GNUC__ <= 3
+#define COMPLAIN(args...) \
+  STMT_BEGIN log_warn(LD_CONFIG, args); STMT_END
+#else
+#define COMPLAIN(args, ...)                                     \
+  STMT_BEGIN log_warn(LD_CONFIG, args, ##__VA_ARGS__); STMT_END
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
+
+#define YES_IF_CHANGED_INT(opt) \
+  if (!CFG_EQ_INT(old_options, new_options, opt)) return 1;
+
+/** Return true iff we are configured to reject request under load for non
+ * relay connections. */
+bool
+dirauth_should_reject_requests_under_load(void)
+{
+  return !!dirauth_get_options()->AuthDirRejectRequestsUnderLoad;
+}
+
+/**
+ * Legacy validation/normalization function for the dirauth mode options in
+ * options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_dirauth_mode(const or_options_t *old_options,
+                              or_options_t *options,
+                              char **msg)
+{
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!authdir_mode(options))
+    return 0;
+
+  /* confirm that our address isn't broken, so we can complain now */
+  uint32_t tmp;
+  if (resolve_my_address(LOG_WARN, options, &tmp, NULL, NULL) < 0)
+    REJECT("Failed to resolve/guess local address. See logs for details.");
+
+  if (!options->ContactInfo && !options->TestingTorNetwork)
+    REJECT("Authoritative directory servers must set ContactInfo");
+
+  if (options->UseEntryGuards) {
+    log_info(LD_CONFIG, "Authoritative directory servers can't set "
+             "UseEntryGuards. Disabling.");
+    options->UseEntryGuards = 0;
+  }
+  if (!options->DownloadExtraInfo && authdir_mode_v3(options)) {
+    log_info(LD_CONFIG, "Authoritative directories always try to download "
+             "extra-info documents. Setting DownloadExtraInfo.");
+    options->DownloadExtraInfo = 1;
+  }
+  if (!(options->BridgeAuthoritativeDir ||
+        options->V3AuthoritativeDir))
+    REJECT("AuthoritativeDir is set, but none of "
+           "(Bridge/V3)AuthoritativeDir is set.");
+
+  /* If we have a v3bandwidthsfile and it's broken, complain on startup */
+  if (options->V3BandwidthsFile && !old_options) {
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                     NULL);
+  }
+  /* same for guardfraction file */
+  if (options->GuardfractionFile && !old_options) {
+    dirserv_read_guardfraction_file(options->GuardfractionFile, NULL);
+  }
+
+  if (!options->DirPort_set)
+    REJECT("Running as authoritative directory, but no DirPort set.");
+
+  if (!options->ORPort_set)
+    REJECT("Running as authoritative directory, but no ORPort set.");
+
+  if (options->ClientOnly)
+    REJECT("Running as authoritative directory, but ClientOnly also set.");
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the dirauth schedule options
+ * in options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_dirauth_schedule(const or_options_t *old_options,
+                                  or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!authdir_mode_v3(options))
+    return 0;
+
+  if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
+      options->V3AuthVotingInterval/2) {
+    REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
+           "V3AuthVotingInterval");
+  }
+
+  if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) {
+        REJECT("V3AuthVoteDelay is way too low.");
+      } else {
+        COMPLAIN("V3AuthVoteDelay is very low. "
+                 "This may lead to failure to vote for a consensus.");
+      }
+    } else {
+      REJECT("V3AuthVoteDelay is way too low.");
+    }
+  }
+
+  if (options->V3AuthDistDelay < MIN_DIST_SECONDS) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) {
+        REJECT("V3AuthDistDelay is way too low.");
+      } else {
+        COMPLAIN("V3AuthDistDelay is very low. "
+                 "This may lead to missing votes in a consensus.");
+      }
+    } else {
+      REJECT("V3AuthDistDelay is way too low.");
+    }
+  }
+
+  if (options->V3AuthNIntervalsValid < 2)
+    REJECT("V3AuthNIntervalsValid must be at least 2.");
+
+  if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) {
+        /* Unreachable, covered by earlier checks */
+        REJECT("V3AuthVotingInterval is insanely low."); /* LCOV_EXCL_LINE */
+      } else {
+        COMPLAIN("V3AuthVotingInterval is very low. "
+                 "This may lead to failure to synchronise for a consensus.");
+      }
+    } else {
+      REJECT("V3AuthVotingInterval is insanely low.");
+    }
+  } else if (options->V3AuthVotingInterval > 24*60*60) {
+    REJECT("V3AuthVotingInterval is insanely high.");
+  } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) {
+    COMPLAIN("V3AuthVotingInterval does not divide evenly into 24 hours.");
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the dirauth testing options
+ * in options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_dirauth_testing(const or_options_t *old_options,
+                                 or_options_t *options,
+                                 char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!authdir_mode(options))
+    return 0;
+
+  if (!authdir_mode_v3(options))
+    return 0;
+
+  if (options->TestingV3AuthInitialVotingInterval
+      < MIN_VOTE_INTERVAL_TESTING_INITIAL) {
+    REJECT("TestingV3AuthInitialVotingInterval is insanely low.");
+  } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) {
+    REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into "
+           "30 minutes.");
+  }
+
+  if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) {
+    REJECT("TestingV3AuthInitialVoteDelay is way too low.");
+  }
+
+  if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) {
+    REJECT("TestingV3AuthInitialDistDelay is way too low.");
+  }
+
+  if (options->TestingV3AuthInitialVoteDelay +
+      options->TestingV3AuthInitialDistDelay >=
+      options->TestingV3AuthInitialVotingInterval) {
+    REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay "
+           "must be less than TestingV3AuthInitialVotingInterval");
+  }
+
+  if (options->TestingV3AuthVotingStartOffset >
+      MIN(options->TestingV3AuthInitialVotingInterval,
+          options->V3AuthVotingInterval)) {
+    REJECT("TestingV3AuthVotingStartOffset is higher than the voting "
+           "interval.");
+  } else if (options->TestingV3AuthVotingStartOffset < 0) {
+    REJECT("TestingV3AuthVotingStartOffset must be non-negative.");
+  }
+
+  return 0;
+}
+
+/**
+ * Return true if changing the configuration from <b>old</b> to <b>new</b>
+ * affects the timing of the voting subsystem
+ */
+static int
+options_transition_affects_dirauth_timing(const or_options_t *old_options,
+                                          const or_options_t *new_options)
+{
+  tor_assert(old_options);
+  tor_assert(new_options);
+
+  if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options))
+    return 1;
+  if (! authdir_mode_v3(new_options))
+    return 0;
+
+  YES_IF_CHANGED_INT(V3AuthVotingInterval);
+  YES_IF_CHANGED_INT(V3AuthVoteDelay);
+  YES_IF_CHANGED_INT(V3AuthDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset);
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirauth actions based on it. All of
+ * the things we do should survive being done repeatedly.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_dirauth(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* We may need to reschedule some dirauth stuff if our status changed. */
+  if (old_options) {
+    if (options_transition_affects_dirauth_timing(old_options, options)) {
+      dirauth_sched_recalculate_timing(options, time(NULL));
+      reschedule_dirvote(options);
+    }
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirauth mtbf actions based on it.
+ * All of the things we do should survive being done repeatedly.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * Must be called immediately after a successful or_state_load().
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_dirauth_mtbf(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  const or_options_t *options = get_options();
+  int running_tor = options->command == CMD_RUN_TOR;
+
+  if (!authdir_mode(options))
+    return 0;
+
+  /* Load dirauth state */
+  if (running_tor) {
+    rep_hist_load_mtbf_data(time(NULL));
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirauth statistics actions based
+ * on it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Sets <b>*print_notice_out</b> if we enabled stats, and need to print
+ * a stats log using options_act_relay_stats_msg().
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_dirauth_stats(const or_options_t *old_options,
+                          bool *print_notice_out)
+{
+  if (BUG(!print_notice_out))
+    return -1;
+
+  const or_options_t *options = get_options();
+
+  if (authdir_mode_bridge(options)) {
+    time_t now = time(NULL);
+    int print_notice = 0;
+
+    if (!old_options || !authdir_mode_bridge(old_options)) {
+      rep_hist_desc_stats_init(now);
+      print_notice = 1;
+    }
+    if (print_notice)
+      *print_notice_out = 1;
+  }
+
+  /* If we used to have statistics enabled but we just disabled them,
+     stop gathering them.  */
+  if (old_options && authdir_mode_bridge(old_options) &&
+      !authdir_mode_bridge(options))
+    rep_hist_desc_stats_term();
+
+  return 0;
+}
+
+/**
+ * Make any necessary modifications to a dirauth_options_t that occur
+ * before validation.  On success return 0; on failure return -1 and
+ * set *<b>msg_out</b> to a newly allocated error string.
+ **/
+static int
+dirauth_options_pre_normalize(void *arg, char **msg_out)
+{
+  dirauth_options_t *options = arg;
+  (void)msg_out;
+
+  if (!options->RecommendedClientVersions)
+    options->RecommendedClientVersions =
+      config_lines_dup(options->RecommendedVersions);
+  if (!options->RecommendedServerVersions)
+    options->RecommendedServerVersions =
+      config_lines_dup(options->RecommendedVersions);
+
+  if (config_ensure_bandwidth_cap(&options->AuthDirFastGuarantee,
+                           "AuthDirFastGuarantee", msg_out) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->AuthDirGuardBWGuarantee,
+                                  "AuthDirGuardBWGuarantee", msg_out) < 0)
+    return -1;
+
+  return 0;
+}
+
+/**
+ * Check whether a dirauth_options_t is correct.
+ *
+ * On success return 0; on failure return -1 and set *<b>msg_out</b> to a
+ * newly allocated error string.
+ **/
+static int
+dirauth_options_validate(const void *arg, char **msg)
+{
+  const dirauth_options_t *options = arg;
+
+  if (options->VersioningAuthoritativeDirectory &&
+      (!options->RecommendedClientVersions ||
+       !options->RecommendedServerVersions)) {
+      REJECT("Versioning authoritative dir servers must set "
+           "Recommended*Versions.");
+  }
+
+  char *t;
+  /* Call these functions to produce warnings only. */
+  t = format_recommended_version_list(options->RecommendedClientVersions, 1);
+  tor_free(t);
+  t = format_recommended_version_list(options->RecommendedServerVersions, 1);
+  tor_free(t);
+
+  if (options->TestingAuthDirTimeToLearnReachability > 2*60*60) {
+    COMPLAIN("TestingAuthDirTimeToLearnReachability is insanely high.");
+  }
+
+  return 0;
+}
+
+/* Declare the options field table for dirauth_options */
+#define CONF_CONTEXT TABLE
+#include "feature/dirauth/dirauth_options.inc"
+#undef CONF_CONTEXT
+
+/** Magic number for dirauth_options_t. */
+#define DIRAUTH_OPTIONS_MAGIC 0x41757448
+
+/**
+ * Declare the configuration options for the dirauth module.
+ **/
+const config_format_t dirauth_options_fmt = {
+  .size = sizeof(dirauth_options_t),
+  .magic = { "dirauth_options_t",
+             DIRAUTH_OPTIONS_MAGIC,
+             offsetof(dirauth_options_t, magic) },
+  .vars = dirauth_options_t_vars,
+
+  .pre_normalize_fn = dirauth_options_pre_normalize,
+  .validate_fn = dirauth_options_validate
+};
diff --git a/src/feature/dirauth/dirauth_config.h b/src/feature/dirauth/dirauth_config.h
new file mode 100644
index 0000000000..9042ff8779
--- /dev/null
+++ b/src/feature/dirauth/dirauth_config.h
@@ -0,0 +1,91 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_config.h
+ * @brief Header for feature/dirauth/dirauth_config.c
+ **/
+
+#ifndef TOR_FEATURE_DIRAUTH_DIRAUTH_CONFIG_H
+#define TOR_FEATURE_DIRAUTH_DIRAUTH_CONFIG_H
+
+struct or_options_t;
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+#include "lib/cc/torint.h"
+
+int options_validate_dirauth_mode(const struct or_options_t *old_options,
+                                  struct or_options_t *options,
+                                  char **msg);
+
+int options_validate_dirauth_schedule(const struct or_options_t *old_options,
+                                      struct or_options_t *options,
+                                      char **msg);
+
+int options_validate_dirauth_testing(const struct or_options_t *old_options,
+                                     struct or_options_t *options,
+                                     char **msg);
+
+int options_act_dirauth(const struct or_options_t *old_options);
+int options_act_dirauth_mtbf(const struct or_options_t *old_options);
+int options_act_dirauth_stats(const struct or_options_t *old_options,
+                              bool *print_notice_out);
+
+bool dirauth_should_reject_requests_under_load(void);
+
+extern const struct config_format_t dirauth_options_fmt;
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+/** When tor is compiled with the dirauth module disabled, it can't be
+ * configured as a directory authority.
+ *
+ * Returns -1 and sets msg to a newly allocated string, if AuthoritativeDir
+ * is set in options. Otherwise returns 0. */
+static inline int
+options_validate_dirauth_mode(const struct or_options_t *old_options,
+                              struct or_options_t *options,
+                              char **msg)
+{
+  (void)old_options;
+
+  /* Only check the primary option for now, #29211 will disable more
+   * options. */
+  if (options->AuthoritativeDir) {
+    /* REJECT() this configuration */
+    *msg = tor_strdup("This tor was built with dirauth mode disabled. "
+                      "It can not be configured with AuthoritativeDir 1.");
+    return -1;
+  }
+
+  return 0;
+}
+
+#define options_validate_dirauth_schedule(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_dirauth_testing(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+
+#define options_act_dirauth(old_options) \
+  (((void)(old_options)),0)
+#define options_act_dirauth_mtbf(old_options) \
+  (((void)(old_options)),0)
+
+static inline int
+options_act_dirauth_stats(const struct or_options_t *old_options,
+                          bool *print_notice_out)
+{
+  (void)old_options;
+  *print_notice_out = 0;
+  return 0;
+}
+
+#define dirauth_should_reject_requests_under_load() (false)
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(TOR_FEATURE_DIRAUTH_DIRAUTH_CONFIG_H) */
diff --git a/src/feature/dirauth/dirauth_options.inc b/src/feature/dirauth/dirauth_options.inc
new file mode 100644
index 0000000000..2aa07a6c88
--- /dev/null
+++ b/src/feature/dirauth/dirauth_options.inc
@@ -0,0 +1,105 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options.inc
+ * @brief Declare configuration options for the crypto_ops module.
+ **/
+
+/** Holds configuration about our directory authority options. */
+BEGIN_CONF_STRUCT(dirauth_options_t)
+
+/** If non-zero, always vote the Fast flag for any relay advertising
+ * this amount of capacity or more. */
+CONF_VAR(AuthDirFastGuarantee, MEMUNIT, 0, "100 KB")
+
+/** If non-zero, this advertised capacity or more is always sufficient
+ * to satisfy the bandwidth requirement for the Guard flag. */
+CONF_VAR(AuthDirGuardBWGuarantee, MEMUNIT, 0, "2 MB")
+
+/** Boolean: are we on IPv6?  */
+CONF_VAR(AuthDirHasIPv6Connectivity, BOOL, 0, "0")
+
+/** True iff we should list bad exits, and vote for all other exits as
+ * good. */
+CONF_VAR(AuthDirListBadExits, BOOL, 0, "0")
+
+/** Do not permit more than this number of servers per IP address. */
+CONF_VAR(AuthDirMaxServersPerAddr, POSINT, 0, "2")
+
+/** Boolean: Do we enforce key-pinning? */
+CONF_VAR(AuthDirPinKeys, BOOL, 0, "1")
+
+/** Bool (default: 1): Switch for the shared random protocol. Only
+ * relevant to a directory authority. If off, the authority won't
+ * participate in the protocol. If on (default), a flag is added to the
+ * vote indicating participation. */
+CONF_VAR(AuthDirSharedRandomness, BOOL, 0, "1")
+
+/** Bool (default: 1): When testing routerinfos as a directory authority,
+ * do we enforce Ed25519 identity match? */
+/* NOTE: remove this option someday. */
+CONF_VAR(AuthDirTestEd25519LinkKeys, BOOL, 0, "1")
+
+/** Authority only: key=value pairs that we add to our networkstatus
+ * consensus vote on the 'params' line. */
+CONF_VAR(ConsensusParams, LINELIST, 0, NULL)
+
+/** Authority only: minimum number of measured bandwidths we must see
+ * before we only believe measured bandwidths to assign flags. */
+CONF_VAR(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, 0, "500")
+
+/** As directory authority, accept hidden service directories after what
+ * time? */
+CONF_VAR(MinUptimeHidServDirectoryV2, INTERVAL, 0, "96 hours")
+
+/** Which versions of tor should we tell users to run? */
+CONF_VAR(RecommendedVersions, LINELIST, 0, NULL)
+
+/** Which versions of tor should we tell users to run on clients? */
+CONF_VAR(RecommendedClientVersions, LINELIST, 0, NULL)
+
+/** Which versions of tor should we tell users to run on relays? */
+CONF_VAR(RecommendedServerVersions, LINELIST, 0, NULL)
+
+/** If an authority has been around for less than this amount of time, it
+ * does not believe its reachability information is accurate.  Only
+ * altered on testing networks. */
+CONF_VAR(TestingAuthDirTimeToLearnReachability, INTERVAL, 0, "30 minutes")
+
+  /** Relays in a testing network which should be voted Exit
+   * regardless of exit policy. */
+CONF_VAR(TestingDirAuthVoteExit, ROUTERSET, 0, NULL)
+CONF_VAR(TestingDirAuthVoteExitIsStrict, BOOL, 0, "0")
+
+/** Relays in a testing network which should be voted Guard
+ * regardless of uptime and bandwidth. */
+CONF_VAR(TestingDirAuthVoteGuard, ROUTERSET, 0, NULL)
+CONF_VAR(TestingDirAuthVoteGuardIsStrict, BOOL, 0, "0")
+
+/** Relays in a testing network which should be voted HSDir
+ * regardless of uptime and DirPort. */
+CONF_VAR(TestingDirAuthVoteHSDir, ROUTERSET, 0, NULL)
+CONF_VAR(TestingDirAuthVoteHSDirIsStrict, BOOL, 0, "0")
+
+/** Minimum value for the Exit flag threshold on testing networks. */
+CONF_VAR(TestingMinExitFlagThreshold, MEMUNIT, 0, "0")
+
+/** Minimum value for the Fast flag threshold on testing networks. */
+CONF_VAR(TestingMinFastFlagThreshold, MEMUNIT, 0, "0")
+
+/** Boolean: is this an authoritative directory that's willing to recommend
+ * versions? */
+CONF_VAR(VersioningAuthoritativeDirectory, BOOL, 0, "0")
+
+/** Boolean: Under bandwidth pressure, if set to 1, the authority will always
+ * answer directory requests from relays but will start sending 503 error code
+ * for the other connections. If set to 0, all connections are considered the
+ * same and the authority will try to answer them all regardless of bandwidth
+ * pressure or not. */
+CONF_VAR(AuthDirRejectRequestsUnderLoad, BOOL, 0, "1")
+
+END_CONF_STRUCT(dirauth_options_t)
diff --git a/src/feature/dirauth/dirauth_options_st.h b/src/feature/dirauth/dirauth_options_st.h
new file mode 100644
index 0000000000..02a498c054
--- /dev/null
+++ b/src/feature/dirauth/dirauth_options_st.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options_st.h
+ * @brief Structure dirauth_options_t to hold directory authority options.
+ **/
+
+#ifndef TOR_FEATURE_DIRAUTH_DIRAUTH_OPTIONS_ST_H
+#define TOR_FEATURE_DIRAUTH_DIRAUTH_OPTIONS_ST_H
+
+#include "lib/conf/confdecl.h"
+#include "feature/nodelist/routerset.h"
+
+#define CONF_CONTEXT STRUCT
+#include "feature/dirauth/dirauth_options.inc"
+#undef CONF_CONTEXT
+
+typedef struct dirauth_options_t dirauth_options_t;
+
+#endif /* !defined(TOR_FEATURE_DIRAUTH_DIRAUTH_OPTIONS_ST_H) */
diff --git a/src/feature/dirauth/dirauth_periodic.c b/src/feature/dirauth/dirauth_periodic.c
new file mode 100644
index 0000000000..19e51c5a05
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.c
@@ -0,0 +1,168 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_periodic.c
+ * @brief Peridoic events for directory authorities.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/or_options_st.h"
+#include "core/mainloop/netstatus.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/mainloop/periodic.h"
+
+#ifndef COCCI
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+#endif /* !defined(COCCI) */
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: if we're an authority, check on our authority
+ * certificate (the one that authenticates our authority signing key).
+ */
+static int
+check_authority_cert_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  (void)options;
+  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
+   * close to expiring and warn the admin if it is. */
+  v3_authority_check_key_expiry();
+#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
+  return CHECK_V3_CERTIFICATE_INTERVAL;
+}
+
+DECLARE_EVENT(check_authority_cert, DIRAUTH, 0);
+
+/**
+ * Scheduled callback: Run directory-authority voting functionality.
+ *
+ * The schedule is a bit complicated here, so dirvote_act() manages the
+ * schedule itself.
+ **/
+static int
+dirvote_callback(time_t now, const or_options_t *options)
+{
+  if (!authdir_mode_v3(options)) {
+    tor_assert_nonfatal_unreached();
+    return 3600;
+  }
+
+  time_t next = dirvote_act(options, now);
+  if (BUG(next == TIME_MAX)) {
+    /* This shouldn't be returned unless we called dirvote_act() without
+     * being an authority.  If it happens, maybe our configuration will
+     * fix itself in an hour or so? */
+    return 3600;
+  }
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(dirvote, DIRAUTH, FL(NEED_NET));
+
+/** Reschedule the directory-authority voting event.  Run this whenever the
+ * schedule has changed. */
+void
+reschedule_dirvote(const or_options_t *options)
+{
+  if (authdir_mode_v3(options)) {
+    periodic_event_reschedule(&dirvote_event);
+  }
+}
+
+/**
+ * Periodic callback: if we're an authority, record our measured stability
+ * information from rephist in an mtbf file.
+ */
+static int
+save_stability_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options)) {
+    if (rep_hist_record_mtbf_data(now, 1)<0) {
+      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+    }
+  }
+#define SAVE_STABILITY_INTERVAL (30*60)
+  return SAVE_STABILITY_INTERVAL;
+}
+
+DECLARE_EVENT(save_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're an authority, make sure we test
+ * the routers on the network for reachability.
+ */
+static int
+launch_reachability_tests_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options) &&
+      !net_is_disabled()) {
+    /* try to determine reachability of the other Tor relays */
+    dirserv_test_reachability(now);
+  }
+  return REACHABILITY_TEST_INTERVAL;
+}
+
+DECLARE_EVENT(launch_reachability_tests, AUTHORITIES, FL(NEED_NET));
+
+/**
+ * Periodic callback: if we're an authority, discount the stability
+ * information (and other rephist information) that's older.
+ */
+static int
+downrate_stability_callback(time_t now, const or_options_t *options)
+{
+  (void)options;
+  /* 1d. Periodically, we discount older stability information so that new
+   * stability info counts more, and save the stability information to disk as
+   * appropriate. */
+  time_t next = rep_hist_downrate_old_runs(now);
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(downrate_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're the bridge authority, write a networkstatus
+ * file to disk.
+ */
+static int
+write_bridge_ns_callback(time_t now, const or_options_t *options)
+{
+  if (options->BridgeAuthoritativeDir) {
+    bridgeauth_dump_bridge_status_to_file(now);
+#define BRIDGE_STATUSFILE_INTERVAL (30*60)
+     return BRIDGE_STATUSFILE_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(write_bridge_ns, BRIDGEAUTH, 0);
+
+void
+dirauth_register_periodic_events(void)
+{
+  periodic_events_register(&downrate_stability_event);
+  periodic_events_register(&launch_reachability_tests_event);
+  periodic_events_register(&save_stability_event);
+  periodic_events_register(&check_authority_cert_event);
+  periodic_events_register(&dirvote_event);
+  periodic_events_register(&write_bridge_ns_event);
+}
diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h
new file mode 100644
index 0000000000..ccdda92a77
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_periodic.h
+ * @brief Header for dirauth_periodic.c
+ **/
+
+#ifndef DIRVOTE_PERIODIC_H
+#define DIRVOTE_PERIODIC_H
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+void dirauth_register_periodic_events(void);
+void reschedule_dirvote(const or_options_t *options);
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+static inline void
+reschedule_dirvote(const or_options_t *options)
+{
+  (void)options;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(DIRVOTE_PERIODIC_H) */
diff --git a/src/feature/dirauth/dirauth_stub.c b/src/feature/dirauth/dirauth_stub.c
new file mode 100644
index 0000000000..9f48ce14fd
--- /dev/null
+++ b/src/feature/dirauth/dirauth_stub.c
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_stub.c
+ * @brief Stub declarations for use when dirauth module is disabled.
+ **/
+
+#include "orconfig.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confdecl.h"
+#include "lib/subsys/subsys.h"
+
+/* Declare the options field table for dirauth_options */
+#define CONF_CONTEXT STUB_TABLE
+#include "feature/dirauth/dirauth_options.inc"
+#undef CONF_CONTEXT
+
+static const config_format_t dirauth_options_stub_fmt = {
+  .vars = dirauth_options_t_vars,
+};
+
+const struct subsys_fns_t sys_dirauth = {
+  .name = "dirauth",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = false,
+  .level = DIRAUTH_SUBSYS_LEVEL,
+
+  .options_format = &dirauth_options_stub_fmt
+};
diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c
new file mode 100644
index 0000000000..07c5743877
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.c
@@ -0,0 +1,71 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_sys.c
+ * @brief Directory authority subsystem declarations
+ **/
+
+#include "core/or/or.h"
+
+#define DIRAUTH_SYS_PRIVATE
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dirauth/dirauth_config.h"
+
+#include "feature/dirauth/dirauth_options_st.h"
+
+#include "lib/subsys/subsys.h"
+
+static const dirauth_options_t *global_dirauth_options;
+
+static int
+subsys_dirauth_initialize(void)
+{
+  dirauth_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_dirauth_shutdown(void)
+{
+  dirserv_free_fingerprint_list();
+  dirvote_free_all();
+  dirserv_clear_measured_bw_cache();
+  keypin_close_journal();
+  global_dirauth_options = NULL;
+}
+
+const dirauth_options_t *
+dirauth_get_options(void)
+{
+  tor_assert(global_dirauth_options);
+  return global_dirauth_options;
+}
+
+STATIC int
+dirauth_set_options(void *arg)
+{
+  dirauth_options_t *opts = arg;
+  global_dirauth_options = opts;
+  return 0;
+}
+
+const struct subsys_fns_t sys_dirauth = {
+  .name = "dirauth",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = DIRAUTH_SUBSYS_LEVEL,
+  .initialize = subsys_dirauth_initialize,
+  .shutdown = subsys_dirauth_shutdown,
+
+  .options_format = &dirauth_options_fmt,
+  .set_options = dirauth_set_options,
+};
diff --git a/src/feature/dirauth/dirauth_sys.h b/src/feature/dirauth/dirauth_sys.h
new file mode 100644
index 0000000000..c512b91b33
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_sys.h
+ * @brief Header for dirauth_sys.c
+ **/
+
+#ifndef DIRAUTH_SYS_H
+#define DIRAUTH_SYS_H
+
+struct dirauth_options_t;
+const struct dirauth_options_t *dirauth_get_options(void);
+
+extern const struct subsys_fns_t sys_dirauth;
+
+/**
+ * Subsystem level for the directory-authority system.
+ *
+ * Defined here so that it can be shared between the real and stub
+ * definitions.
+ **/
+#define DIRAUTH_SUBSYS_LEVEL 70
+
+#ifdef DIRAUTH_SYS_PRIVATE
+STATIC int dirauth_set_options(void *arg);
+#endif
+
+#endif /* !defined(DIRAUTH_SYS_H) */
diff --git a/src/feature/dirauth/dircollate.c b/src/feature/dirauth/dircollate.c
index 7992e3a85f..2657f53853 100644
--- a/src/feature/dirauth/dircollate.c
+++ b/src/feature/dirauth/dircollate.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,8 +32,8 @@ static void dircollator_collate_by_ed25519(dircollator_t *dc);
 
 /** Hashtable entry mapping a pair of digests (actually an ed25519 key and an
  * RSA SHA1 digest) to an array of vote_routerstatus_t. */
-typedef struct ddmap_entry_s {
-  HT_ENTRY(ddmap_entry_s) node;
+typedef struct ddmap_entry_t {
+  HT_ENTRY(ddmap_entry_t) node;
   /** A SHA1-RSA1024 identity digest and Ed25519 identity key,
    * concatenated.  (If there is no ed25519 identity key, there is no
    * entry in this table.) */
@@ -89,10 +89,10 @@ ddmap_entry_set_digests(ddmap_entry_t *ent,
   memcpy(ent->d + DIGEST_LEN, ed25519, DIGEST256_LEN);
 }
 
-HT_PROTOTYPE(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash,
-             ddmap_entry_eq)
-HT_GENERATE2(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash,
-             ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_)
+HT_PROTOTYPE(double_digest_map, ddmap_entry_t, node, ddmap_entry_hash,
+             ddmap_entry_eq);
+HT_GENERATE2(double_digest_map, ddmap_entry_t, node, ddmap_entry_hash,
+             ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_);
 
 /** Helper: add a single vote_routerstatus_t <b>vrs</b> to the collator
  * <b>dc</b>, indexing it by its RSA key digest, and by the 2-tuple of its RSA
@@ -324,4 +324,3 @@ dircollator_get_votes_for_router(dircollator_t *dc, int idx)
   return digestmap_get(dc->by_collated_rsa_sha1,
                        smartlist_get(dc->all_rsa_sha1_lst, idx));
 }
-
diff --git a/src/feature/dirauth/dircollate.h b/src/feature/dirauth/dircollate.h
index 754a094817..90c6bddad5 100644
--- a/src/feature/dirauth/dircollate.h
+++ b/src/feature/dirauth/dircollate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "core/or/or.h"
 
-typedef struct dircollator_s dircollator_t;
+typedef struct dircollator_t dircollator_t;
 
 dircollator_t *dircollator_new(int n_votes, int n_authorities);
 void dircollator_free_(dircollator_t *obj);
@@ -30,11 +30,11 @@ vote_routerstatus_t **dircollator_get_votes_for_router(dircollator_t *dc,
                                                        int idx);
 
 #ifdef DIRCOLLATE_PRIVATE
-struct ddmap_entry_s;
-typedef HT_HEAD(double_digest_map, ddmap_entry_s) double_digest_map_t;
+struct ddmap_entry_t;
+typedef HT_HEAD(double_digest_map, ddmap_entry_t) double_digest_map_t;
 /** A dircollator keeps track of all the routerstatus entries in a
  * set of networkstatus votes, and matches them by an appropriate rule. */
-struct dircollator_s {
+struct dircollator_t {
   /** True iff we have run the collation algorithm. */
   int is_collated;
   /** The total number of votes that we received. */
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
index 9e01cee42a..d9fbd2a7ce 100644
--- a/src/feature/dirauth/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -1,11 +1,12 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRVOTE_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/or/policies.h"
 #include "core/or/protover.h"
 #include "core/or/tor_version_st.h"
@@ -28,6 +29,7 @@
 #include "feature/nodelist/fmt_routerstatus.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -35,15 +37,17 @@
 #include "feature/stats/rephist.h"
 #include "feature/client/entrynodes.h" /* needed for guardfraction methods */
 #include "feature/nodelist/torcert.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random_state.h"
+#include "feature/dirauth/dirauth_sys.h"
 
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/dircache/cached_dir_st.h"
 #include "feature/dirclient/dir_server_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/document_signature_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -60,6 +64,9 @@
 #include "lib/encoding/confline.h"
 #include "lib/crypt_ops/crypto_format.h"
 
+/* Algorithm to use for the bandwidth file digest. */
+#define DIGEST_ALG_BW_FILE DIGEST_SHA256
+
 /**
  * \file dirvote.c
  * \brief Functions to compute directory consensus, and schedule voting.
@@ -216,7 +223,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                           networkstatus_t *v3_ns)
 {
   smartlist_t *chunks = smartlist_new();
-  char *packages = NULL;
   char fingerprint[FINGERPRINT_LEN+1];
   char digest[DIGEST_LEN];
   uint32_t addr;
@@ -242,19 +248,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                                                 v3_ns->server_versions);
   protocols_lines = format_protocols_lines_for_vote(v3_ns);
 
-  if (v3_ns->package_lines) {
-    smartlist_t *tmp = smartlist_new();
-    SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p,
-                      if (validate_recommended_package_line(p))
-                        smartlist_add_asprintf(tmp, "package %s\n", p));
-    smartlist_sort_strings(tmp);
-    packages = smartlist_join_strings(tmp, "", 0, NULL);
-    SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp));
-    smartlist_free(tmp);
-  } else {
-    packages = tor_strdup("");
-  }
-
     /* Get shared random commitments/reveals line(s). */
   shared_random_vote_str = sr_get_string_for_vote();
 
@@ -268,6 +261,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     char *flag_thresholds = dirserv_get_flag_thresholds_line();
     char *params;
     char *bw_headers_line = NULL;
+    char *bw_file_digest = NULL;
     authority_cert_t *cert = v3_ns->cert;
     char *methods =
       make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
@@ -307,43 +301,68 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
       tor_free(bw_file_headers);
     }
 
-    smartlist_add_asprintf(chunks,
-                 "network-status-version 3\n"
-                 "vote-status %s\n"
-                 "consensus-methods %s\n"
-                 "published %s\n"
-                 "valid-after %s\n"
-                 "fresh-until %s\n"
-                 "valid-until %s\n"
-                 "voting-delay %d %d\n"
-                 "%s%s" /* versions */
-                 "%s" /* protocols */
-                 "%s" /* packages */
-                 "known-flags %s\n"
-                 "flag-thresholds %s\n"
-                 "params %s\n"
-                 "%s" /* bandwidth file headers */
-                 "dir-source %s %s %s %s %d %d\n"
-                 "contact %s\n"
-                 "%s" /* shared randomness information */
-                 ,
-                 v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion",
-                 methods,
-                 published, va, fu, vu,
-                 v3_ns->vote_seconds, v3_ns->dist_seconds,
-                 client_versions_line,
-                 server_versions_line,
-                 protocols_lines,
-                 packages,
-                 flags,
-                 flag_thresholds,
-                 params,
-                 bw_headers_line ? bw_headers_line : "",
-                 voter->nickname, fingerprint, voter->address,
-                 fmt_addr32(addr), voter->dir_port, voter->or_port,
-                 voter->contact,
-                 shared_random_vote_str ?
-                           shared_random_vote_str : "");
+    /* Create bandwidth-file-digest if applicable.
+     * v3_ns->b64_digest_bw_file will contain the digest when V3BandwidthsFile
+     * is configured and the bandwidth file could be read, even if it was not
+     * parseable.
+     */
+    if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) {
+      /* Encode the digest. */
+      char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0};
+      digest256_to_base64(b64_digest_bw_file,
+                          (const char *)v3_ns->bw_file_digest256);
+      /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */
+      char *digest_algo_b64_digest_bw_file = NULL;
+      tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s",
+                   crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE),
+                   b64_digest_bw_file);
+      /* No need for tor_strdup(""), format_line_if_present does it. */
+      bw_file_digest = format_line_if_present(
+          "bandwidth-file-digest", digest_algo_b64_digest_bw_file);
+      tor_free(digest_algo_b64_digest_bw_file);
+    }
+
+    const char *ip_str = fmt_addr32(addr);
+
+    if (ip_str[0]) {
+      smartlist_add_asprintf(chunks,
+                   "network-status-version 3\n"
+                   "vote-status %s\n"
+                   "consensus-methods %s\n"
+                   "published %s\n"
+                   "valid-after %s\n"
+                   "fresh-until %s\n"
+                   "valid-until %s\n"
+                   "voting-delay %d %d\n"
+                   "%s%s" /* versions */
+                   "%s" /* protocols */
+                   "known-flags %s\n"
+                   "flag-thresholds %s\n"
+                   "params %s\n"
+                   "%s" /* bandwidth file headers */
+                   "%s" /* bandwidth file digest */
+                   "dir-source %s %s %s %s %d %d\n"
+                   "contact %s\n"
+                   "%s" /* shared randomness information */
+                   ,
+                   v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion",
+                   methods,
+                   published, va, fu, vu,
+                   v3_ns->vote_seconds, v3_ns->dist_seconds,
+                   client_versions_line,
+                   server_versions_line,
+                   protocols_lines,
+                   flags,
+                   flag_thresholds,
+                   params,
+                   bw_headers_line ? bw_headers_line : "",
+                   bw_file_digest ? bw_file_digest: "",
+                   voter->nickname, fingerprint, voter->address,
+                   ip_str, voter->dir_port, voter->or_port,
+                   voter->contact,
+                   shared_random_vote_str ?
+                             shared_random_vote_str : "");
+    }
 
     tor_free(params);
     tor_free(flags);
@@ -351,6 +370,10 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     tor_free(methods);
     tor_free(shared_random_vote_str);
     tor_free(bw_headers_line);
+    tor_free(bw_file_digest);
+
+    if (ip_str[0] == '\0')
+      goto err;
 
     if (!tor_digest_is_zero(voter->legacy_id_digest)) {
       char fpbuf[HEX_DIGEST_LEN+1];
@@ -369,7 +392,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     rsf = routerstatus_format_entry(&vrs->status,
                                     vrs->version, vrs->protocols,
                                     NS_V3_VOTE,
-                                    ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
                                     vrs);
     if (rsf)
       smartlist_add(chunks, rsf);
@@ -412,7 +434,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
 
   {
     networkstatus_t *v;
-    if (!(v = networkstatus_parse_vote_from_string(status, NULL,
+    if (!(v = networkstatus_parse_vote_from_string(status, strlen(status),
+                                                   NULL,
                                                    v3_ns->type))) {
       log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: "
               "<<%s>>",
@@ -430,7 +453,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
   tor_free(client_versions_line);
   tor_free(server_versions_line);
   tor_free(protocols_lines);
-  tor_free(packages);
 
   SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
   smartlist_free(chunks);
@@ -872,7 +894,7 @@ dirvote_get_intermediate_param_value(const smartlist_t *param_list,
       int ok;
       value = (int32_t)
         tor_parse_long(integer_str, 10, INT32_MIN, INT32_MAX, &ok, NULL);
-      if (BUG(! ok))
+      if (BUG(!ok))
         return default_val;
       ++n_found;
     }
@@ -1525,14 +1547,11 @@ networkstatus_compute_consensus(smartlist_t *votes,
     consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD;
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE) {
+  {
     /* It's smarter to initialize these weights to 1, so that later on,
      * we can't accidentally divide by zero. */
     G = M = E = D = 1;
     T = 4;
-  } else {
-    /* ...but originally, they were set to zero. */
-    G = M = E = D = T = 0;
   }
 
   /* Compute medians of time-related things, and figure out how many
@@ -2233,7 +2252,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         /* Okay!! Now we can write the descriptor... */
         /*     First line goes into "buf". */
         buf = routerstatus_format_entry(&rs_out, NULL, NULL,
-                                        rs_format, consensus_method, NULL);
+                                        rs_format, NULL);
         if (buf)
           smartlist_add(chunks, buf);
       }
@@ -2253,8 +2272,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         smartlist_add_strdup(chunks, chosen_version);
       }
       smartlist_add_strdup(chunks, "\n");
-      if (chosen_protocol_list &&
-          consensus_method >= MIN_METHOD_FOR_RS_PROTOCOLS) {
+      if (chosen_protocol_list) {
         smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list);
       }
       /*     Now the weight line. */
@@ -2409,7 +2427,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
   {
     networkstatus_t *c;
-    if (!(c = networkstatus_parse_vote_from_string(result, NULL,
+    if (!(c = networkstatus_parse_vote_from_string(result, strlen(result),
+                                                   NULL,
                                                    NS_TYPE_CONSENSUS))) {
       log_err(LD_BUG, "Generated a networkstatus consensus we couldn't "
               "parse.");
@@ -2516,9 +2535,12 @@ compute_consensus_package_lines(smartlist_t *votes)
  * any new signatures in <b>src_voter_list</b> that should be added to
  * <b>target</b>. (A signature should be added if we have no signature for that
  * voter in <b>target</b> yet, or if we have no verifiable signature and the
- * new signature is verifiable.)  Return the number of signatures added or
- * changed, or -1 if the document signed by <b>sigs</b> isn't the same
- * document as <b>target</b>. */
+ * new signature is verifiable.)
+ *
+ * Return the number of signatures added or changed, or -1 if the document
+ * signatures are invalid. Sets *<b>msg_out</b> to a string constant
+ * describing the signature status.
+ */
 STATIC int
 networkstatus_add_detached_signatures(networkstatus_t *target,
                                       ns_detached_signatures_t *sigs,
@@ -2567,7 +2589,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target,
       return -1;
     }
     for (alg = DIGEST_SHA1; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) {
-      if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
+      if (!fast_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
         if (fast_memeq(target->digests.d[alg], digests->d[alg],
                        DIGEST256_LEN)) {
           ++n_matches;
@@ -2763,7 +2785,7 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses)
       char d[HEX_DIGEST256_LEN+1];
       const char *alg_name =
         crypto_digest_algorithm_get_name(alg);
-      if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
+      if (fast_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
         continue;
       base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN);
       smartlist_add_asprintf(elements, "additional-digest %s %s %s\n",
@@ -2839,7 +2861,7 @@ dirvote_act(const or_options_t *options, time_t now)
                "Mine is %s.",
                keys, hex_str(c->cache_info.identity_digest, DIGEST_LEN));
     tor_free(keys);
-    voting_schedule_recalculate_timing(options, now);
+    dirauth_sched_recalculate_timing(options, now);
   }
 
 #define IF_TIME_FOR_NEXT_ACTION(when_field, done_field) \
@@ -2885,7 +2907,7 @@ dirvote_act(const or_options_t *options, time_t now)
                 networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
     /* XXXX We will want to try again later if we haven't got enough
      * signatures yet.  Implement this if it turns out to ever happen. */
-    voting_schedule_recalculate_timing(options, now);
+    dirauth_sched_recalculate_timing(options, now);
     return voting_schedule.voting_starts;
   } ENDIF
 
@@ -2952,7 +2974,7 @@ dirvote_perform_vote(void)
   if (!contents)
     return -1;
 
-  pending_vote = dirvote_add_vote(contents, &msg, &status);
+  pending_vote = dirvote_add_vote(contents, 0, &msg, &status);
   tor_free(contents);
   if (!pending_vote) {
     log_warn(LD_DIR, "Couldn't store my own vote! (I told myself, '%s'.)",
@@ -3108,13 +3130,45 @@ list_v3_auth_ids(void)
   return keys;
 }
 
+/* Check the voter information <b>vi</b>, and  assert that at least one
+ * signature is good. Asserts on failure. */
+static void
+assert_any_sig_good(const networkstatus_voter_info_t *vi)
+{
+  int any_sig_good = 0;
+  SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig,
+                    if (sig->good_signature)
+                      any_sig_good = 1);
+  tor_assert(any_sig_good);
+}
+
+/* Add <b>cert</b> to our list of known authority certificates. */
+static void
+add_new_cert_if_needed(const struct authority_cert_t *cert)
+{
+  tor_assert(cert);
+  if (!authority_cert_get_by_digests(cert->cache_info.identity_digest,
+                                     cert->signing_key_digest)) {
+    /* Hey, it's a new cert! */
+    trusted_dirs_load_certs_from_string(
+                               cert->cache_info.signed_descriptor_body,
+                               TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/,
+                               NULL);
+    if (!authority_cert_get_by_digests(cert->cache_info.identity_digest,
+                                       cert->signing_key_digest)) {
+      log_warn(LD_BUG, "We added a cert, but still couldn't find it.");
+    }
+  }
+}
+
 /** Called when we have received a networkstatus vote in <b>vote_body</b>.
  * Parse and validate it, and on success store it as a pending vote (which we
  * then return).  Return NULL on failure.  Sets *<b>msg_out</b> and
  * *<b>status_out</b> to an HTTP response and status code.  (V3 authority
  * only) */
 pending_vote_t *
-dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+dirvote_add_vote(const char *vote_body, time_t time_posted,
+                 const char **msg_out, int *status_out)
 {
   networkstatus_t *vote;
   networkstatus_voter_info_t *vi;
@@ -3132,7 +3186,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
   *msg_out = NULL;
 
  again:
-  vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote,
+  vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body),
+                                              &end_of_vote,
                                               NS_TYPE_VOTE);
   if (!end_of_vote)
     end_of_vote = vote_body + strlen(vote_body);
@@ -3144,13 +3199,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
   }
   tor_assert(smartlist_len(vote->voters) == 1);
   vi = get_voter(vote);
-  {
-    int any_sig_good = 0;
-    SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig,
-                      if (sig->good_signature)
-                        any_sig_good = 1);
-    tor_assert(any_sig_good);
-  }
+  assert_any_sig_good(vi);
   ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest);
   if (!ds) {
     char *keys = list_v3_auth_ids();
@@ -3163,19 +3212,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
     *msg_out = "Vote not from a recognized v3 authority";
     goto err;
   }
-  tor_assert(vote->cert);
-  if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest,
-                                     vote->cert->signing_key_digest)) {
-    /* Hey, it's a new cert! */
-    trusted_dirs_load_certs_from_string(
-                               vote->cert->cache_info.signed_descriptor_body,
-                               TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/,
-                               NULL);
-    if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest,
-                                       vote->cert->signing_key_digest)) {
-      log_warn(LD_BUG, "We added a cert, but still couldn't find it.");
-    }
-  }
+  add_new_cert_if_needed(vote->cert);
 
   /* Is it for the right period? */
   if (vote->valid_after != voting_schedule.interval_starts) {
@@ -3188,6 +3225,23 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
     goto err;
   }
 
+  /* Check if we received it, as a post, after the cutoff when we
+   * start asking other dir auths for it. If we do, the best plan
+   * is to discard it, because using it greatly increases the chances
+   * of a split vote for this round (some dir auths got it in time,
+   * some didn't). */
+  if (time_posted && time_posted > voting_schedule.fetch_missing_votes) {
+    char tbuf1[ISO_TIME_LEN+1], tbuf2[ISO_TIME_LEN+1];
+    format_iso_time(tbuf1, time_posted);
+    format_iso_time(tbuf2, voting_schedule.fetch_missing_votes);
+    log_warn(LD_DIR, "Rejecting posted vote from %s received at %s; "
+             "our cutoff for received votes is %s. Check your clock, "
+             "CPU load, and network load. Also check the authority that "
+             "posted the vote.", vi->address, tbuf1, tbuf2);
+    *msg_out = "Posted vote received too late, would be dangerous to count it";
+    goto err;
+  }
+
   /* Fetch any new router descriptors we just learned about */
   update_consensus_router_descriptor_downloads(time(NULL), 1, vote);
 
@@ -3390,7 +3444,9 @@ dirvote_compute_consensuses(void)
                  flavor_name);
         continue;
       }
-      consensus = networkstatus_parse_vote_from_string(consensus_body, NULL,
+      consensus = networkstatus_parse_vote_from_string(consensus_body,
+                                                       strlen(consensus_body),
+                                                       NULL,
                                                        NS_TYPE_CONSENSUS);
       if (!consensus) {
         log_warn(LD_DIR, "Couldn't parse %s consensus we generated!",
@@ -3529,7 +3585,7 @@ dirvote_add_signatures_to_pending_consensus(
      * just in case we break detached signature processing at some point. */
     {
       networkstatus_t *v = networkstatus_parse_vote_from_string(
-                                             pc->body, NULL,
+                                             pc->body, strlen(pc->body), NULL,
                                              NS_TYPE_CONSENSUS);
       tor_assert(v);
       networkstatus_vote_free(v);
@@ -3550,6 +3606,14 @@ dirvote_add_signatures_to_pending_consensus(
   return r;
 }
 
+/** Helper: we just got the <b>detached_signatures_body</b> sent to us as
+ * signatures on the currently pending consensus.  Add them to the pending
+ * consensus (if we have one).
+ *
+ * Set *<b>msg</b> to a string constant describing the status, regardless of
+ * success or failure.
+ *
+ * Return negative on failure, nonnegative on success. */
 static int
 dirvote_add_signatures_to_all_pending_consensuses(
                        const char *detached_signatures_body,
@@ -3612,7 +3676,12 @@ dirvote_add_signatures_to_all_pending_consensuses(
 /** Helper: we just got the <b>detached_signatures_body</b> sent to us as
  * signatures on the currently pending consensus.  Add them to the pending
  * consensus (if we have one); otherwise queue them until we have a
- * consensus.  Return negative on failure, nonnegative on success. */
+ * consensus.
+ *
+ * Set *<b>msg</b> to a string constant describing the status, regardless of
+ * success or failure.
+ *
+ * Return negative on failure, nonnegative on success. */
 int
 dirvote_add_signatures(const char *detached_signatures_body,
                        const char *source,
@@ -3654,7 +3723,9 @@ dirvote_publish_consensus(void)
       continue;
     }
 
-    if (networkstatus_set_current_consensus(pending->body, name, 0, NULL))
+    if (networkstatus_set_current_consensus(pending->body,
+                                            strlen(pending->body),
+                                            name, 0, NULL))
       log_warn(LD_DIR, "Error publishing %s consensus", name);
     else
       log_notice(LD_DIR, "Published %s consensus", name);
@@ -3784,15 +3855,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf);
   }
 
-  /* We originally put a lines in the micrdescriptors, but then we worked out
-   * that we needed them in the microdesc consensus. See #20916. */
-  if (consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC &&
-      !tor_addr_is_null(&ri->ipv6_addr) && ri->ipv6_orport)
-    smartlist_add_asprintf(chunks, "a %s\n",
-                           fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
-
-  if (family)
-    smartlist_add_asprintf(chunks, "family %s\n", family);
+  if (family) {
+    if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) {
+      smartlist_add_asprintf(chunks, "family %s\n", family);
+    } else {
+      const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest;
+      char *canonical_family = nodefamily_canonicalize(family, id, 0);
+      smartlist_add_asprintf(chunks, "family %s\n", canonical_family);
+      tor_free(canonical_family);
+    }
+  }
 
   if (summary && strcmp(summary, "reject 1-65535"))
     smartlist_add_asprintf(chunks, "p %s\n", summary);
@@ -3869,8 +3941,7 @@ dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len,
                                ",");
   tor_assert(microdesc_consensus_methods);
 
-  if (digest256_to_base64(d64, md->digest)<0)
-    goto out;
+  digest256_to_base64(d64, md->digest);
 
   if (tor_snprintf(out_buf, out_buf_len, "m %s sha256=%s\n",
                    microdesc_consensus_methods, d64)<0)
@@ -3889,8 +3960,10 @@ static const struct consensus_method_range_t {
   int low;
   int high;
 } microdesc_consensus_methods[] = {
-  {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1},
-  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD},
+  {MIN_SUPPORTED_CONSENSUS_METHOD,
+   MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1},
+  {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS,
+   MAX_SUPPORTED_CONSENSUS_METHOD},
   {-1, -1}
 };
 
@@ -4199,7 +4272,7 @@ compare_routerinfo_by_ip_and_bw_(const void **a, const void **b)
 static digestmap_t *
 get_possible_sybil_list(const smartlist_t *routers)
 {
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
   digestmap_t *omit_as_sybil;
   smartlist_t *routers_by_ip = smartlist_new();
   uint32_t last_addr;
@@ -4364,6 +4437,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs)
    * forget to add it to this clause. */
 }
 
+/** Space-separated list of all the flags that we will always vote on. */
+const char DIRVOTE_UNIVERSAL_FLAGS[] =
+  "Authority "
+  "Exit "
+  "Fast "
+  "Guard "
+  "HSDir "
+  "Stable "
+  "StaleDesc "
+  "V2Dir "
+  "Valid";
+/** Space-separated list of all flags that we may or may not vote on,
+ * depending on our configuration. */
+const char DIRVOTE_OPTIONAL_FLAGS[] =
+  "BadExit "
+  "Running";
+
 /** Return a new networkstatus_t* containing our current opinion. (For v3
  * authorities) */
 networkstatus_t *
@@ -4371,6 +4461,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
                                         authority_cert_t *cert)
 {
   const or_options_t *options = get_options();
+  const dirauth_options_t *d_options = dirauth_get_options();
   networkstatus_t *v3_out = NULL;
   uint32_t addr;
   char *hostname = NULL, *client_versions = NULL, *server_versions = NULL;
@@ -4378,7 +4469,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   smartlist_t *routers, *routerstatuses;
   char identity_digest[DIGEST_LEN];
   char signing_key_digest[DIGEST_LEN];
-  int listbadexits = options->AuthDirListBadExits;
+  const int listbadexits = d_options->AuthDirListBadExits;
   routerlist_t *rl = router_get_routerlist();
   time_t now = time(NULL);
   time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
@@ -4388,6 +4479,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   const int vote_on_reachability = running_long_enough_to_decide_unreachable();
   smartlist_t *microdescriptors = NULL;
   smartlist_t *bw_file_headers = NULL;
+  uint8_t bw_file_digest256[DIGEST256_LEN] = {0};
 
   tor_assert(private_key);
   tor_assert(cert);
@@ -4409,11 +4501,16 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     hostname = tor_dup_ip(addr);
   }
 
-  if (options->VersioningAuthoritativeDir) {
+  if (!hostname) {
+    log_err(LD_BUG, "Failed to determine hostname AND duplicate address");
+    return NULL;
+  }
+
+  if (d_options->VersioningAuthoritativeDirectory) {
     client_versions =
-      format_recommended_version_list(options->RecommendedClientVersions, 0);
+      format_recommended_version_list(d_options->RecommendedClientVersions, 0);
     server_versions =
-      format_recommended_version_list(options->RecommendedServerVersions, 0);
+      format_recommended_version_list(d_options->RecommendedServerVersions, 0);
   }
 
   contact = get_options()->ContactInfo;
@@ -4425,7 +4522,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
    * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
    */
   if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                     NULL);
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4479,8 +4577,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
       vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       rs = &vrs->status;
-      set_routerstatus_from_routerinfo(rs, node, ri, now,
-                                       listbadexits);
+      dirauth_set_routerstatus_from_routerinfo(rs, node, ri, now,
+                                               listbadexits);
 
       if (ri->cache_info.signing_key_cert) {
         memcpy(vrs->ed25519_id,
@@ -4530,7 +4628,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     /* Only set bw_file_headers when V3BandwidthsFile is configured */
     bw_file_headers = smartlist_new();
     dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
-                                     routerstatuses, bw_file_headers);
+                                     routerstatuses, bw_file_headers,
+                                     bw_file_digest256);
+
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4557,7 +4657,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     else
       last_consensus_interval = options->TestingV3AuthInitialVotingInterval;
     v3_out->valid_after =
-      voting_schedule_get_start_of_next_interval(now,
+      voting_sched_get_start_of_interval_after(now,
                                    (int)last_consensus_interval,
                                    options->TestingV3AuthVotingStartOffset);
     format_iso_time(tbuf, v3_out->valid_after);
@@ -4579,17 +4679,14 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
   /* These are hardwired, to avoid disaster. */
   v3_out->recommended_relay_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 Microdesc=1-2 Relay=2");
+    tor_strdup(DIRVOTE_RECOMMEND_RELAY_PROTO);
   v3_out->recommended_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 Microdesc=1-2 Relay=2");
-  v3_out->required_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 Microdesc=1-2 Relay=2");
+    tor_strdup(DIRVOTE_RECOMMEND_CLIENT_PROTO);
+
   v3_out->required_relay_protocols =
-    tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=3-4 Microdesc=1 Relay=1-2");
+    tor_strdup(DIRVOTE_REQUIRE_RELAY_PROTO);
+  v3_out->required_client_protocols =
+    tor_strdup(DIRVOTE_REQUIRE_CLIENT_PROTO);
 
   /* We are not allowed to vote to require anything we don't have. */
   tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL));
@@ -4601,18 +4698,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   tor_assert_nonfatal(protover_all_supported(
                                v3_out->recommended_client_protocols, NULL));
 
-  v3_out->package_lines = smartlist_new();
-  {
-    config_line_t *cl;
-    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
-      if (validate_recommended_package_line(cl->value))
-        smartlist_add_strdup(v3_out->package_lines, cl->value);
-    }
-  }
-
   v3_out->known_flags = smartlist_new();
   smartlist_split_string(v3_out->known_flags,
-                         "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   if (vote_on_reachability)
     smartlist_add_strdup(v3_out->known_flags, "Running");
@@ -4620,13 +4708,17 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     smartlist_add_strdup(v3_out->known_flags, "BadExit");
   smartlist_sort_strings(v3_out->known_flags);
 
-  if (options->ConsensusParams) {
+  if (d_options->ConsensusParams) {
+    config_line_t *paramline = d_options->ConsensusParams;
     v3_out->net_params = smartlist_new();
-    smartlist_split_string(v3_out->net_params,
-                           options->ConsensusParams, NULL, 0, 0);
+    for ( ; paramline; paramline = paramline->next) {
+      smartlist_split_string(v3_out->net_params,
+                             paramline->value, NULL, 0, 0);
+    }
     smartlist_sort_strings(v3_out->net_params);
   }
   v3_out->bw_file_headers = bw_file_headers;
+  memcpy(v3_out->bw_file_digest256, bw_file_digest256, DIGEST256_LEN);
 
   voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
   voter->nickname = tor_strdup(options->Nickname);
diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h
index 02d88d19d1..a9b356b387 100644
--- a/src/feature/dirauth/dirvote.h
+++ b/src/feature/dirauth/dirvote.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,35 +49,17 @@
 #define MIN_VOTE_INTERVAL_TESTING_INITIAL \
                 ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)
 
-/* A placeholder for routerstatus_format_entry() when the consensus method
- * argument is not applicable. */
-#define ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD 0
-
 /** The lowest consensus method that we currently support. */
-#define MIN_SUPPORTED_CONSENSUS_METHOD 25
+#define MIN_SUPPORTED_CONSENSUS_METHOD 28
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 28
-
-/** Lowest consensus method where authorities vote on required/recommended
- * protocols. */
-#define MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS 25
-
-/** Lowest consensus method where authorities add protocols to routerstatus
- * entries. */
-#define MIN_METHOD_FOR_RS_PROTOCOLS 25
-
-/** Lowest consensus method where authorities initialize bandwidth weights to 1
- * instead of 0. See #14881 */
-#define MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE 26
+#define MAX_SUPPORTED_CONSENSUS_METHOD 29
 
-/** Lowest consensus method where the microdesc consensus contains relay IPv6
- * addresses. See #23826 and #20916. */
-#define MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS 27
-
-/** Lowest consensus method where microdescriptors do not contain relay IPv6
- * addresses. See #23828 and #20916. */
-#define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28
+/**
+ * Lowest consensus method where microdescriptor lines are put in canonical
+ * form for improved compressibility and ease of storage. See proposal 298.
+ **/
+#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29
 
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
@@ -92,6 +74,9 @@
 /** Maximum size of a line in a vote. */
 #define MAX_BW_FILE_HEADERS_LINE_LEN 1024
 
+extern const char DIRVOTE_UNIVERSAL_FLAGS[];
+extern const char DIRVOTE_OPTIONAL_FLAGS[];
+
 /*
  * Public API. Used outside of the dirauth subsystem.
  *
@@ -109,6 +94,7 @@ void dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
 
 /* Storing signatures and votes functions */
 struct pending_vote_t * dirvote_add_vote(const char *vote_body,
+                                         time_t time_posted,
                                          const char **msg_out,
                                          int *status_out);
 int dirvote_add_signatures(const char *detached_signatures_body,
@@ -119,7 +105,7 @@ struct config_line_t;
 char *format_recommended_version_list(const struct config_line_t *line,
                                       int warn);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline time_t
 dirvote_act(const or_options_t *options, time_t now)
@@ -157,9 +143,13 @@ dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
 }
 
 static inline struct pending_vote_t *
-dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+dirvote_add_vote(const char *vote_body,
+                 time_t time_posted,
+                 const char **msg_out,
+                 int *status_out)
 {
   (void) vote_body;
+  (void) time_posted;
   /* If the dirauth module is disabled, this should NEVER be called else we
    * failed to safeguard the dirauth module. */
   tor_assert_nonfatal_unreached();
@@ -177,14 +167,14 @@ dirvote_add_signatures(const char *detached_signatures_body,
 {
   (void) detached_signatures_body;
   (void) source;
-  (void) msg_out;
+  *msg_out = "No directory authority support";
   /* If the dirauth module is disabled, this should NEVER be called else we
    * failed to safeguard the dirauth module. */
   tor_assert_nonfatal_unreached();
   return 0;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Item access */
 MOCK_DECL(const char*, dirvote_get_pending_consensus,
@@ -245,6 +235,64 @@ char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
 STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri,
                                                    int consensus_method);
 
+/** The recommended relay protocols for this authority's votes.
+ * Recommending a new protocol causes old tor versions to log a warning.
+ */
+#define DIRVOTE_RECOMMEND_RELAY_PROTO           \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "DirCache=2 "                                 \
+  "HSDir=2 "                                    \
+  "HSIntro=4 "                                  \
+  "HSRend=2 "                                   \
+  "Link=4-5 "                                   \
+  "LinkAuth=3 "                                 \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
+/** The recommended client protocols for this authority's votes.
+ * Recommending a new protocol causes old tor versions to log a warning.
+ */
+#define DIRVOTE_RECOMMEND_CLIENT_PROTO          \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "DirCache=2 "                                 \
+  "HSDir=2 "                                    \
+  "HSIntro=4 "                                  \
+  "HSRend=2 "                                   \
+  "Link=4-5 "                                   \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
+/** The required relay protocols for this authority's votes.
+ * WARNING: Requiring a new protocol causes old tor versions to shut down.
+ *          Requiring the wrong protocols can break the tor network.
+ * See Proposal 303: When and how to remove support for protocol versions.
+ */
+#define DIRVOTE_REQUIRE_RELAY_PROTO             \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "DirCache=2 "                                 \
+  "HSDir=2 "                                    \
+  "HSIntro=4 "                                  \
+  "HSRend=2 "                                   \
+  "Link=4-5 "                                   \
+  "LinkAuth=3 "                                 \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
+/** The required relay protocols for this authority's votes.
+ * WARNING: Requiring a new protocol causes old tor versions to shut down.
+ *          Requiring the wrong protocols can break the tor network.
+ * See Proposal 303: When and how to remove support for protocol versions.
+ */
+#define DIRVOTE_REQUIRE_CLIENT_PROTO            \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "Link=4 "                                     \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
 #endif /* defined(DIRVOTE_PRIVATE) */
 
 #endif /* !defined(TOR_DIRVOTE_H) */
diff --git a/src/feature/dirauth/dsigs_parse.c b/src/feature/dirauth/dsigs_parse.c
index d88176fee9..d0bb931814 100644
--- a/src/feature/dirauth/dsigs_parse.c
+++ b/src/feature/dirauth/dsigs_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -127,7 +127,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos)
     }
     digests = detached_get_digests(sigs, flavor);
     tor_assert(digests);
-    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
+    if (!fast_mem_is_zero(digests->d[alg], digest_length)) {
       log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
                "signatures document", flavor, algname);
       continue;
diff --git a/src/feature/dirauth/dsigs_parse.h b/src/feature/dirauth/dsigs_parse.h
index fec51ba488..b25e3e0b28 100644
--- a/src/feature/dirauth/dsigs_parse.h
+++ b/src/feature/dirauth/dsigs_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ void ns_detached_signatures_free_(ns_detached_signatures_t *s);
 #define ns_detached_signatures_free(s) \
   FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
 
-#endif
+#endif /* !defined(TOR_DSIGS_PARSE_H) */
diff --git a/src/feature/dirauth/feature_dirauth.md b/src/feature/dirauth/feature_dirauth.md
new file mode 100644
index 0000000000..b152b94894
--- /dev/null
+++ b/src/feature/dirauth/feature_dirauth.md
@@ -0,0 +1,9 @@
+@dir /feature/dirauth
+@brief feature/dirauth: Directory authority implementation.
+
+This module handles running Tor as a directory authority.
+
+The directory protocol is specified in
+[dir-spec.txt](https://gitweb.torproject.org/torspec.git/tree/dir-spec.txt).
+
+
diff --git a/src/feature/dirauth/guardfraction.c b/src/feature/dirauth/guardfraction.c
index d1a7f194d4..b84f804f5f 100644
--- a/src/feature/dirauth/guardfraction.c
+++ b/src/feature/dirauth/guardfraction.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -188,7 +188,7 @@ guardfraction_file_parse_inputs_line(const char *inputs_line,
  *
  *      guardfraction-file-version 1
  *      written-at <date and time>
- *      n-inputs <number of consesuses parsed> <number of days considered>
+ *      n-inputs <number of consensuses parsed> <number of days considered>
  *
  *      guard-seen <fpr 1> <guardfraction percentage> <consensus appearances>
  *      guard-seen <fpr 2> <guardfraction percentage> <consensus appearances>
diff --git a/src/feature/dirauth/guardfraction.h b/src/feature/dirauth/guardfraction.h
index 72404907a4..c10fd9b7bb 100644
--- a/src/feature/dirauth/guardfraction.h
+++ b/src/feature/dirauth/guardfraction.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,9 +16,9 @@
 STATIC int
 dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
                                       smartlist_t *vote_routerstatuses);
-#endif /* defined(DIRSERV_PRIVATE) */
+#endif
 
 int dirserv_read_guardfraction_file(const char *fname,
                                  smartlist_t *vote_routerstatuses);
 
-#endif
+#endif /* !defined(TOR_GUARDFRACTION_H) */
diff --git a/src/feature/dirauth/include.am b/src/feature/dirauth/include.am
new file mode 100644
index 0000000000..e26f120d4e
--- /dev/null
+++ b/src/feature/dirauth/include.am
@@ -0,0 +1,54 @@
+
+# The Directory Authority module.
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+MODULE_DIRAUTH_SOURCES =					\
+	src/feature/dirauth/authmode.c				\
+	src/feature/dirauth/bridgeauth.c			\
+	src/feature/dirauth/bwauth.c				\
+	src/feature/dirauth/dirauth_config.c			\
+	src/feature/dirauth/dirauth_periodic.c			\
+	src/feature/dirauth/dirauth_sys.c			\
+	src/feature/dirauth/dircollate.c			\
+	src/feature/dirauth/dirvote.c				\
+	src/feature/dirauth/dsigs_parse.c			\
+	src/feature/dirauth/guardfraction.c			\
+	src/feature/dirauth/keypin.c				\
+	src/feature/dirauth/process_descs.c			\
+	src/feature/dirauth/reachability.c			\
+	src/feature/dirauth/recommend_pkg.c			\
+	src/feature/dirauth/shared_random.c			\
+	src/feature/dirauth/shared_random_state.c		\
+	src/feature/dirauth/voteflags.c				\
+	src/feature/dirauth/voting_schedule.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dirauth/authmode.h			\
+	src/feature/dirauth/bridgeauth.h		\
+	src/feature/dirauth/bwauth.h			\
+	src/feature/dirauth/dirauth_config.h		\
+	src/feature/dirauth/dirauth_options.inc		\
+	src/feature/dirauth/dirauth_options_st.h	\
+	src/feature/dirauth/dirauth_periodic.h		\
+	src/feature/dirauth/dirauth_sys.h		\
+	src/feature/dirauth/dircollate.h		\
+	src/feature/dirauth/dirvote.h			\
+	src/feature/dirauth/dsigs_parse.h		\
+	src/feature/dirauth/guardfraction.h		\
+	src/feature/dirauth/keypin.h			\
+	src/feature/dirauth/ns_detached_signatures_st.h	\
+	src/feature/dirauth/reachability.h		\
+	src/feature/dirauth/recommend_pkg.h		\
+	src/feature/dirauth/process_descs.h		\
+	src/feature/dirauth/shared_random.h		\
+	src/feature/dirauth/shared_random_state.h	\
+	src/feature/dirauth/vote_microdesc_hash_st.h	\
+	src/feature/dirauth/voteflags.h			\
+	src/feature/dirauth/voting_schedule.h
+
+if BUILD_MODULE_DIRAUTH
+LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
+else
+LIBTOR_APP_A_STUB_SOURCES += src/feature/dirauth/dirauth_stub.c
+endif
diff --git a/src/feature/dirauth/keypin.c b/src/feature/dirauth/keypin.c
index 06cb9ba1ff..5072a58573 100644
--- a/src/feature/dirauth/keypin.c
+++ b/src/feature/dirauth/keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,8 +15,6 @@
 #include "lib/cc/torint.h"
 #include "lib/crypt_ops/crypto_digest.h"
 #include "lib/crypt_ops/crypto_format.h"
-#include "lib/crypt_ops/crypto_format.h"
-#include "lib/ctime/di_ops.h"
 #include "lib/ctime/di_ops.h"
 #include "lib/encoding/binascii.h"
 #include "lib/encoding/time_fmt.h"
@@ -120,14 +118,14 @@ return (unsigned) siphash24g(a->ed25519_key, sizeof(a->ed25519_key));
 }
 
 HT_PROTOTYPE(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa,
-               keypin_ents_eq_rsa)
+             keypin_ents_eq_rsa);
 HT_GENERATE2(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa,
-               keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_)
+             keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_);
 
 HT_PROTOTYPE(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed,
-               keypin_ents_eq_ed)
+             keypin_ents_eq_ed);
 HT_GENERATE2(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed,
-               keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_)
+             keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_);
 
 /**
  * Check whether we already have an entry in the key pinning table for a
@@ -438,7 +436,7 @@ keypin_load_journal_impl(const char *data, size_t size)
   tor_log(severity, LD_DIRSERV,
           "Loaded %d entries from keypin journal. "
           "Found %d corrupt lines (ignored), %d duplicates (harmless), "
-          "and %d conflicts (resolved in favor or more recent entry).",
+          "and %d conflicts (resolved in favor of more recent entry).",
           n_entries, n_corrupt_lines, n_duplicates, n_conflicts);
 
   return 0;
diff --git a/src/feature/dirauth/keypin.h b/src/feature/dirauth/keypin.h
index 722b6ca5fc..881f010f0e 100644
--- a/src/feature/dirauth/keypin.h
+++ b/src/feature/dirauth/keypin.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file keypin.h
+ * @brief Header for keypin.c
+ **/
+
 #ifndef TOR_KEYPIN_H
 #define TOR_KEYPIN_H
 
@@ -11,10 +16,25 @@ int keypin_check_and_add(const uint8_t *rsa_id_digest,
                          const int replace_existing_entry);
 int keypin_check(const uint8_t *rsa_id_digest,
                  const uint8_t *ed25519_id_key);
+int keypin_close_journal(void);
 
+#ifdef HAVE_MODULE_DIRAUTH
 int keypin_open_journal(const char *fname);
-int keypin_close_journal(void);
 int keypin_load_journal(const char *fname);
+#else
+static inline int
+keypin_open_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+static inline int
+keypin_load_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 void keypin_clear(void);
 int keypin_check_lone_rsa(const uint8_t *rsa_id_digest);
 
@@ -25,6 +45,8 @@ int keypin_check_lone_rsa(const uint8_t *rsa_id_digest);
 
 #ifdef KEYPIN_PRIVATE
 
+#include "ext/ht.h"
+
 /**
  * In-memory representation of a key-pinning table entry.
  */
@@ -44,4 +66,3 @@ MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent));
 #endif /* defined(KEYPIN_PRIVATE) */
 
 #endif /* !defined(TOR_KEYPIN_H) */
-
diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h
index 0f92be2f0d..f409431ec1 100644
--- a/src/feature/dirauth/ns_detached_signatures_st.h
+++ b/src/feature/dirauth/ns_detached_signatures_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file ns_detached_signatures_st.h
+ * @brief Detached consensus signatures structure.
+ **/
+
 #ifndef NS_DETACHED_SIGNATURES_ST_H
 #define NS_DETACHED_SIGNATURES_ST_H
 
@@ -18,5 +23,4 @@ struct ns_detached_signatures_t {
                          * document_signature_t */
 };
 
-#endif
-
+#endif /* !defined(NS_DETACHED_SIGNATURES_ST_H) */
diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c
index 21b8e239ec..5025d0ae39 100644
--- a/src/feature/dirauth/process_descs.c
+++ b/src/feature/dirauth/process_descs.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,17 +12,21 @@
  * them make those decisions.
  **/
 
+#define PROCESS_DESCS_PRIVATE
+
 #include "core/or/or.h"
 #include "feature/dirauth/process_descs.h"
 
 #include "app/config/config.h"
 #include "core/or/policies.h"
 #include "core/or/versions.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
+#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo.h"
@@ -32,46 +36,28 @@
 #include "feature/relay/router.h"
 
 #include "core/or/tor_version_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/extrainfo_st.h"
 #include "feature/nodelist/node_st.h"
+#include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
 
 #include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
 
 /** How far in the future do we allow a router to get? (seconds) */
 #define ROUTER_ALLOW_SKEW (60*60*12)
 
 static void directory_remove_invalid(void);
-struct authdir_config_t;
 static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
                                                 const char **msg);
 static uint32_t
-dirserv_get_status_impl(const char *fp, const char *nickname,
-                        uint32_t addr, uint16_t or_port,
-                        const char *platform, const char **msg,
-                        int severity);
-
-/*                 1  Historically used to indicate Named */
-#define FP_INVALID 2  /**< Believed invalid. */
-#define FP_REJECT  4  /**< We will not publish this router. */
-/*                 8  Historically used to avoid using this as a dir. */
-#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
-/*                 32 Historically used to indicade Unnamed */
-
-/** Target of status_by_digest map. */
-typedef uint32_t router_status_t;
-
-static void add_fingerprint_to_dir(const char *fp,
-                                   struct authdir_config_t *list,
-                                   router_status_t add_status);
-
-/** List of nickname-\>identity fingerprint mappings for all the routers
- * that we name.  Used to prevent router impersonation. */
-typedef struct authdir_config_t {
-  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
-  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
-} authdir_config_t;
+dirserv_get_status_impl(const char *id_digest,
+                        const ed25519_public_key_t *ed25519_public_key,
+                        const char *nickname, uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg, int severity);
 
 /** Should be static; exposed for testing. */
 static authdir_config_t *fingerprint_list = NULL;
@@ -83,20 +69,39 @@ authdir_config_new(void)
   authdir_config_t *list = tor_malloc_zero(sizeof(authdir_config_t));
   list->fp_by_name = strmap_new();
   list->status_by_digest = digestmap_new();
+  list->status_by_digest256 = digest256map_new();
   return list;
 }
 
+#ifdef TOR_UNIT_TESTS
+
+/** Initialize fingerprint_list to a new authdir_config_t. Used for tests. */
+void
+authdir_init_fingerprint_list(void)
+{
+  fingerprint_list = authdir_config_new();
+}
+
+/* Return the current fingerprint_list. Used for tests. */
+authdir_config_t *
+authdir_return_fingerprint_list(void)
+{
+  return fingerprint_list;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's
  * <b>list</b>, or-ing the currently set status flags with
  * <b>add_status</b>.
  */
-/* static */ void
-add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
-                       router_status_t add_status)
+int
+add_rsa_fingerprint_to_dir(const char *fp, authdir_config_t *list,
+                           rtr_flags_t add_status)
 {
   char *fingerprint;
   char d[DIGEST_LEN];
-  router_status_t *status;
+  rtr_flags_t *status;
   tor_assert(fp);
   tor_assert(list);
 
@@ -107,24 +112,52 @@ add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
     log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"",
              escaped(fp));
     tor_free(fingerprint);
-    return;
+    return -1;
   }
 
   status = digestmap_get(list->status_by_digest, d);
   if (!status) {
-    status = tor_malloc_zero(sizeof(router_status_t));
+    status = tor_malloc_zero(sizeof(rtr_flags_t));
     digestmap_set(list->status_by_digest, d, status);
   }
 
   tor_free(fingerprint);
   *status |= add_status;
-  return;
+  return 0;
+}
+
+/** Add the ed25519 key <b>edkey</b> to the smartlist of fingerprint_entry_t's
+ * <b>list</b>, or-ing the currently set status flags with <b>add_status</b>.
+ * Return -1 if we were unable to decode the key, else return 0.
+ */
+int
+add_ed25519_to_dir(const ed25519_public_key_t *edkey, authdir_config_t *list,
+                   rtr_flags_t add_status)
+{
+  rtr_flags_t *status;
+
+  tor_assert(edkey);
+  tor_assert(list);
+
+  if (ed25519_validate_pubkey(edkey) < 0) {
+    log_warn(LD_DIRSERV, "Invalid ed25519 key \"%s\"", ed25519_fmt(edkey));
+    return -1;
+  }
+
+  status = digest256map_get(list->status_by_digest256, edkey->pubkey);
+  if (!status) {
+    status = tor_malloc_zero(sizeof(rtr_flags_t));
+    digest256map_set(list->status_by_digest256, edkey->pubkey, status);
+  }
+
+  *status |= add_status;
+  return 0;
 }
 
 /** Add the fingerprint for this OR to the global list of recognized
  * identity key fingerprints. */
 int
-dirserv_add_own_fingerprint(crypto_pk_t *pk)
+dirserv_add_own_fingerprint(crypto_pk_t *pk, const ed25519_public_key_t *edkey)
 {
   char fp[FINGERPRINT_LEN+1];
   if (crypto_pk_get_fingerprint(pk, fp, 0)<0) {
@@ -133,7 +166,14 @@ dirserv_add_own_fingerprint(crypto_pk_t *pk)
   }
   if (!fingerprint_list)
     fingerprint_list = authdir_config_new();
-  add_fingerprint_to_dir(fp, fingerprint_list, 0);
+  if (add_rsa_fingerprint_to_dir(fp, fingerprint_list, 0) < 0) {
+    log_err(LD_BUG, "Error adding RSA fingerprint");
+    return -1;
+  }
+  if (add_ed25519_to_dir(edkey, fingerprint_list, 0) < 0) {
+    log_err(LD_BUG, "Error adding ed25519 key");
+    return -1;
+  }
   return 0;
 }
 
@@ -174,27 +214,46 @@ dirserv_load_fingerprint_file(void)
   fingerprint_list_new = authdir_config_new();
 
   for (list=front; list; list=list->next) {
-    char digest_tmp[DIGEST_LEN];
-    router_status_t add_status = 0;
+    rtr_flags_t add_status = 0;
     nickname = list->key; fingerprint = list->value;
     tor_strstrip(fingerprint, " "); /* remove spaces */
-    if (strlen(fingerprint) != HEX_DIGEST_LEN ||
-        base16_decode(digest_tmp, sizeof(digest_tmp),
-                      fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) {
-      log_notice(LD_CONFIG,
-                 "Invalid fingerprint (nickname '%s', "
-                 "fingerprint %s). Skipping.",
-                 nickname, fingerprint);
-      continue;
-    }
+
+   /* Determine what we should do with the relay with the nickname field. */
     if (!strcasecmp(nickname, "!reject")) {
-        add_status = FP_REJECT;
+        add_status = RTR_REJECT;
     } else if (!strcasecmp(nickname, "!badexit")) {
-        add_status = FP_BADEXIT;
+        add_status = RTR_BADEXIT;
     } else if (!strcasecmp(nickname, "!invalid")) {
-        add_status = FP_INVALID;
+        add_status = RTR_INVALID;
+    }
+
+    /* Check if fingerprint is RSA or ed25519 by verifying it. */
+    int ed25519_not_ok = -1, rsa_not_ok = -1;
+
+    /* Attempt to add the RSA key. */
+    if (strlen(fingerprint) == HEX_DIGEST_LEN) {
+      rsa_not_ok = add_rsa_fingerprint_to_dir(fingerprint,
+                                              fingerprint_list_new,
+                                              add_status);
+    }
+
+    /* Check ed25519 key. We check the size to prevent buffer overflows.
+     * If valid, attempt to add it, */
+    ed25519_public_key_t ed25519_pubkey_tmp;
+    if (strlen(fingerprint) == BASE64_DIGEST256_LEN) {
+      if (!digest256_from_base64((char *) ed25519_pubkey_tmp.pubkey,
+                                 fingerprint)) {
+        ed25519_not_ok = add_ed25519_to_dir(&ed25519_pubkey_tmp,
+                                            fingerprint_list_new, add_status);
+      }
+    }
+
+    /* If both keys are invalid (or missing), log and skip. */
+    if (ed25519_not_ok && rsa_not_ok) {
+      log_warn(LD_CONFIG, "Invalid fingerprint (nickname '%s', "
+               "fingerprint %s). Skipping.", nickname, fingerprint);
+      continue;
     }
-    add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status);
   }
 
   config_free_lines(front);
@@ -216,30 +275,42 @@ dirserv_load_fingerprint_file(void)
 
 #define DISABLE_DISABLING_ED25519
 
-/** Check whether <b>router</b> has a nickname/identity key combination that
- * we recognize from the fingerprint list, or an IP we automatically act on
- * according to our configuration.  Return the appropriate router status.
+/** Check whether <b>router</b> has:
+ * - a nickname/identity key combination that we recognize from the fingerprint
+ *   list,
+ * - an IP we automatically act on according to our configuration,
+ * - an appropriate version, and
+ * - matching pinned keys.
+ *
+ * Return the appropriate router status.
  *
- * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
- * *<b>msg</b> to an explanation of why. */
+ * If the status is 'RTR_REJECT' and <b>msg</b> is provided, set
+ * *<b>msg</b> to a string constant explaining why. */
 uint32_t
 dirserv_router_get_status(const routerinfo_t *router, const char **msg,
                           int severity)
 {
   char d[DIGEST_LEN];
-  const int key_pinning = get_options()->AuthDirPinKeys;
+  const int key_pinning = dirauth_get_options()->AuthDirPinKeys;
+  uint32_t r;
+  ed25519_public_key_t *signing_key = NULL;
 
   if (crypto_pk_get_digest(router->identity_pkey, d)) {
     log_warn(LD_BUG,"Error computing fingerprint");
     if (msg)
       *msg = "Bug: Error computing fingerprint";
-    return FP_REJECT;
+    return RTR_REJECT;
   }
 
-  /* Check for the more usual versions to reject a router first. */
-  const uint32_t r = dirserv_get_status_impl(d, router->nickname,
-                                             router->addr, router->or_port,
-                                             router->platform, msg, severity);
+  /* First, check for the more common reasons to reject a router. */
+  if (router->cache_info.signing_key_cert) {
+    /* This has an ed25519 identity key. */
+    signing_key = &router->cache_info.signing_key_cert->signing_key;
+  }
+  r = dirserv_get_status_impl(d, signing_key, router->nickname, router->addr,
+                              router->or_port, router->platform, msg,
+                              severity);
+
   if (r)
     return r;
 
@@ -254,7 +325,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
            "key.", router_describe(router));
     if (msg)
       *msg = "Missing ntor curve25519 onion key. Please upgrade!";
-    return FP_REJECT;
+    return RTR_REJECT;
   }
 
   if (router->cache_info.signing_key_cert) {
@@ -270,7 +341,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
         if (msg) {
           *msg = "Ed25519 identity key or RSA identity key has changed.";
         }
-        return FP_REJECT;
+        return RTR_REJECT;
       }
     }
   } else {
@@ -287,7 +358,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
         if (msg) {
           *msg = "Ed25519 identity key has disappeared.";
         }
-        return FP_REJECT;
+        return RTR_REJECT;
       }
 #endif /* defined(DISABLE_DISABLING_ED25519) */
     }
@@ -299,31 +370,74 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
 /** Return true if there is no point in downloading the router described by
  * <b>rs</b> because this directory would reject it. */
 int
-dirserv_would_reject_router(const routerstatus_t *rs)
+dirserv_would_reject_router(const routerstatus_t *rs,
+                            const vote_routerstatus_t *vrs)
 {
   uint32_t res;
+  struct ed25519_public_key_t pk;
+  memcpy(&pk.pubkey, vrs->ed25519_id, ED25519_PUBKEY_LEN);
 
-  res = dirserv_get_status_impl(rs->identity_digest, rs->nickname,
-                                rs->addr, rs->or_port,
-                                NULL, NULL, LOG_DEBUG);
+  res = dirserv_get_status_impl(rs->identity_digest, &pk, rs->nickname,
+                                rs->addr, rs->or_port, NULL, NULL, LOG_DEBUG);
 
-  return (res & FP_REJECT) != 0;
+  return (res & RTR_REJECT) != 0;
+}
+
+/**
+ * Check whether the platform string in <b>platform</b> describes a platform
+ * that, as a directory authority, we want to reject.  If it does, return
+ * true, and set *<b>msg</b> (if present) to a rejection message.  Otherwise
+ * return false.
+ */
+STATIC bool
+dirserv_rejects_tor_version(const char *platform,
+                            const char **msg)
+{
+  if (!platform)
+    return false;
+
+  static const char please_upgrade_string[] =
+    "Tor version is insecure or unsupported. Please upgrade!";
+
+  /* Versions before Tor 0.3.5 are unsupported.
+   *
+   * Also, reject unstable versions of 0.3.5, since (as of this writing)
+   * they are almost none of the network. */
+  if (!tor_version_as_new_as(platform,"0.3.5.7")) {
+    if (msg)
+      *msg = please_upgrade_string;
+    return true;
+  }
+
+  /* Series between Tor 0.3.6 and 0.4.1.4-rc inclusive are unsupported.
+   * Reject them. 0.3.6.0-alpha-dev only existed for a short time, before
+   * it was renamed to 0.4.0.0-alpha-dev. */
+  if (tor_version_as_new_as(platform,"0.3.6.0-alpha-dev") &&
+      !tor_version_as_new_as(platform,"0.4.1.5")) {
+    if (msg) {
+      *msg = please_upgrade_string;
+    }
+    return true;
+  }
+
+  return false;
 }
 
 /** Helper: As dirserv_router_get_status, but takes the router fingerprint
- * (hex, no spaces), nickname, address (used for logging only), IP address, OR
- * port and platform (logging only) as arguments.
+ * (hex, no spaces), ed25519 key, nickname, address (used for logging only),
+ * IP address, OR port and platform (logging only) as arguments.
  *
  * Log messages at 'severity'. (There's not much point in
  * logging that we're rejecting servers we'll not download.)
  */
 static uint32_t
-dirserv_get_status_impl(const char *id_digest, const char *nickname,
-                        uint32_t addr, uint16_t or_port,
+dirserv_get_status_impl(const char *id_digest,
+                        const ed25519_public_key_t *ed25519_public_key,
+                        const char *nickname, uint32_t addr, uint16_t or_port,
                         const char *platform, const char **msg, int severity)
 {
   uint32_t result = 0;
-  router_status_t *status_by_digest;
+  rtr_flags_t *status_by_digest;
 
   if (!fingerprint_list)
     fingerprint_list = authdir_config_new();
@@ -338,27 +452,13 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
       if (msg) {
         *msg = "Malformed platform string.";
       }
-      return FP_REJECT;
+      return RTR_REJECT;
     }
   }
 
-  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
-   * missing some important security fixes too. Disable them. */
-  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
-    if (msg)
-      *msg = "Tor version is insecure or unsupported. Please upgrade!";
-    return FP_REJECT;
-  }
-
-  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
-   * keep their consensus up to date so they make bad guards.
-   * The simple fix is to just drop them from the network. */
-  if (platform &&
-      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
-      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
-    if (msg)
-      *msg = "Tor version contains bug 20499. Please upgrade!";
-    return FP_REJECT;
+  /* Check whether the version is obsolete, broken, insecure, etc... */
+  if (platform && dirserv_rejects_tor_version(platform, msg)) {
+    return RTR_REJECT;
   }
 
   status_by_digest = digestmap_get(fingerprint_list->status_by_digest,
@@ -366,23 +466,30 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
   if (status_by_digest)
     result |= *status_by_digest;
 
-  if (result & FP_REJECT) {
+  if (ed25519_public_key) {
+    status_by_digest = digest256map_get(fingerprint_list->status_by_digest256,
+                                        ed25519_public_key->pubkey);
+    if (status_by_digest)
+      result |= *status_by_digest;
+  }
+
+  if (result & RTR_REJECT) {
     if (msg)
-      *msg = "Fingerprint is marked rejected -- if you think this is a "
-             "mistake please set a valid email address in ContactInfo and "
-             "send an email to bad-relays@lists.torproject.org mentioning "
-             "your fingerprint(s)?";
-    return FP_REJECT;
-  } else if (result & FP_INVALID) {
+      *msg = "Fingerprint and/or ed25519 identity is marked rejected -- if "
+             "you think this is a mistake please set a valid email address "
+             "in ContactInfo and send an email to "
+             "bad-relays@lists.torproject.org mentioning your fingerprint(s)?";
+    return RTR_REJECT;
+  } else if (result & RTR_INVALID) {
     if (msg)
-      *msg = "Fingerprint is marked invalid";
+      *msg = "Fingerprint and/or ed25519 identity is marked invalid";
   }
 
   if (authdir_policy_badexit_address(addr, or_port)) {
     log_fn(severity, LD_DIRSERV,
            "Marking '%s' as bad exit because of address '%s'",
                nickname, fmt_addr32(addr));
-    result |= FP_BADEXIT;
+    result |= RTR_BADEXIT;
   }
 
   if (!authdir_policy_permits_address(addr, or_port)) {
@@ -393,13 +500,13 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
              "mistake please set a valid email address in ContactInfo and "
              "send an email to bad-relays@lists.torproject.org mentioning "
              "your address(es) and fingerprint(s)?";
-    return FP_REJECT;
+    return RTR_REJECT;
   }
   if (!authdir_policy_valid_address(addr, or_port)) {
     log_fn(severity, LD_DIRSERV,
            "Not marking '%s' valid because of address '%s'",
                nickname, fmt_addr32(addr));
-    result |= FP_INVALID;
+    result |= RTR_INVALID;
   }
 
   return result;
@@ -414,6 +521,7 @@ dirserv_free_fingerprint_list(void)
 
   strmap_free(fingerprint_list->fp_by_name, tor_free_);
   digestmap_free(fingerprint_list->status_by_digest, tor_free_);
+  digest256map_free(fingerprint_list->status_by_digest256, tor_free_);
   tor_free(fingerprint_list);
 }
 
@@ -423,27 +531,40 @@ dirserv_free_fingerprint_list(void)
 
 /** Return -1 if <b>ri</b> has a private or otherwise bad address,
  * unless we're configured to not care. Return 0 if all ok. */
-static int
+STATIC int
 dirserv_router_has_valid_address(routerinfo_t *ri)
 {
   tor_addr_t addr;
+
   if (get_options()->DirAllowPrivateAddresses)
     return 0; /* whatever it is, we're fine with it */
+
   tor_addr_from_ipv4h(&addr, ri->addr);
+  if (tor_addr_is_null(&addr) || tor_addr_is_internal(&addr, 0)) {
+    log_info(LD_DIRSERV,
+             "Router %s published internal IPv4 address. Refusing.",
+             router_describe(ri));
+    return -1; /* it's a private IP, we should reject it */
+  }
 
-  if (tor_addr_is_internal(&addr, 0)) {
+  /* We only check internal v6 on non-null addresses because we do not require
+   * IPv6 and null IPv6 is normal. */
+  if (!tor_addr_is_null(&ri->ipv6_addr) &&
+      tor_addr_is_internal(&ri->ipv6_addr, 0)) {
     log_info(LD_DIRSERV,
-             "Router %s published internal IP address. Refusing.",
+             "Router %s published internal IPv6 address. Refusing.",
              router_describe(ri));
     return -1; /* it's a private IP, we should reject it */
   }
+
   return 0;
 }
 
 /** Check whether we, as a directory server, want to accept <b>ri</b>.  If so,
  * set its is_valid,running fields and return 0.  Otherwise, return -1.
  *
- * If the router is rejected, set *<b>msg</b> to an explanation of why.
+ * If the router is rejected, set *<b>msg</b> to a string constant explining
+ * why.
  *
  * If <b>complain</b> then explain at log-level 'notice' why we refused
  * a descriptor; else explain at log-level 'info'.
@@ -457,7 +578,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
   int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO;
   uint32_t status = dirserv_router_get_status(ri, msg, severity);
   tor_assert(msg);
-  if (status & FP_REJECT)
+  if (status & RTR_REJECT)
     return -1; /* msg is already set. */
 
   /* Is there too much clock skew? */
@@ -493,7 +614,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
     return -1;
   }
 
-  *valid_out = ! (status & FP_INVALID);
+  *valid_out = ! (status & RTR_INVALID);
 
   return 0;
 }
@@ -505,8 +626,8 @@ void
 dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                  uint32_t authstatus)
 {
-  node->is_valid = (authstatus & FP_INVALID) ? 0 : 1;
-  node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0;
+  node->is_valid = (authstatus & RTR_INVALID) ? 0 : 1;
+  node->is_bad_exit = (authstatus & RTR_BADEXIT) ? 1 : 0;
 }
 
 /** True iff <b>a</b> is more severe than <b>b</b>. */
@@ -519,7 +640,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
 /** As for dirserv_add_descriptor(), but accepts multiple documents, and
  * returns the most severe error that occurred for any one of them. */
 was_router_added_t
-dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
                                  const char *source,
                                  const char **msg)
 {
@@ -534,7 +656,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   int general = purpose == ROUTER_PURPOSE_GENERAL;
   tor_assert(msg);
 
-  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
+  r=ROUTER_ADDED_SUCCESSFULLY; /* Least severe return value. */
+
+  if (!string_is_utf8_no_bom(desc, desclen)) {
+    *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM.";
+    return ROUTER_AUTHDIR_REJECTS;
+  }
 
   format_iso_time(time_buf, now);
   if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
@@ -545,14 +672,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
                    !general ? router_purpose_to_string(purpose) : "",
                    !general ? "\n" : "")<0) {
     *msg = "Couldn't format annotations";
-    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
-     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
-    return -1;
+    return ROUTER_AUTHDIR_BUG_ANNOTATIONS;
   }
 
   s = desc;
   list = smartlist_new();
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0,
                                      annotation_buf, NULL)) {
     SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
         msg_out = NULL;
@@ -568,7 +693,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   smartlist_clear(list);
 
   s = desc;
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0,
                                      NULL, NULL)) {
     SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
         msg_out = NULL;
@@ -605,7 +730,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
  * That means the caller must not access <b>ri</b> after this function
  * returns, since it might have been freed.
  *
- * Return the status of the operation.
+ * Return the status of the operation, and set *<b>msg</b> to a string
+ * constant describing the status.
  *
  * This function is only called when fresh descriptors are posted, not when
  * we re-load the cache.
@@ -618,7 +744,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
   char *desc, *nickname;
   const size_t desclen = ri->cache_info.signed_descriptor_len +
       ri->cache_info.annotations_len;
-  const int key_pinning = get_options()->AuthDirPinKeys;
+  const int key_pinning = dirauth_get_options()->AuthDirPinKeys;
   *msg = NULL;
 
   /* If it's too big, refuse it now. Otherwise we'll cache it all over the
@@ -816,21 +942,21 @@ directory_remove_invalid(void)
       continue;
     r = dirserv_router_get_status(ent, &msg, LOG_INFO);
     description = router_describe(ent);
-    if (r & FP_REJECT) {
+    if (r & RTR_REJECT) {
       log_info(LD_DIRSERV, "Router %s is now rejected: %s",
                description, msg?msg:"");
       routerlist_remove(rl, ent, 0, time(NULL));
       continue;
     }
-    if (bool_neq((r & FP_INVALID), !node->is_valid)) {
+    if (bool_neq((r & RTR_INVALID), !node->is_valid)) {
       log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description,
-               (r&FP_INVALID) ? "in" : "");
-      node->is_valid = (r&FP_INVALID)?0:1;
+               (r&RTR_INVALID) ? "in" : "");
+      node->is_valid = (r&RTR_INVALID)?0:1;
     }
-    if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) {
+    if (bool_neq((r & RTR_BADEXIT), node->is_bad_exit)) {
       log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description,
-               (r & FP_BADEXIT) ? "bad" : "good");
-      node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0;
+               (r & RTR_BADEXIT) ? "bad" : "good");
+      node->is_bad_exit = (r&RTR_BADEXIT) ? 1: 0;
     }
   } SMARTLIST_FOREACH_END(node);
 
diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h
index ae2d6ad25d..1461ab697d 100644
--- a/src/feature/dirauth/process_descs.h
+++ b/src/feature/dirauth/process_descs.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,27 +12,155 @@
 #ifndef TOR_RECV_UPLOADS_H
 #define TOR_RECV_UPLOADS_H
 
-int dirserv_load_fingerprint_file(void);
+// for was_router_added_t.
+#include "feature/nodelist/routerlist.h"
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+struct authdir_config_t;
+
+/** Target of status_by_digest map. */
+typedef uint32_t rtr_flags_t;
+
+int add_rsa_fingerprint_to_dir(const char *fp, struct authdir_config_t *list,
+                               rtr_flags_t add_status);
+
+int add_ed25519_to_dir(const ed25519_public_key_t *edkey,
+                       struct authdir_config_t *list,
+                       rtr_flags_t add_status);
+
+/** List of nickname-\>identity fingerprint mappings for all the routers
+ * that we name.  Used to prevent router impersonation. */
+typedef struct authdir_config_t {
+  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
+  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
+  digest256map_t *status_by_digest256; /**< Map from digest256 to
+                                        * router_status_t. */
+} authdir_config_t;
+
+#if defined(PROCESS_DESCS_PRIVATE) || defined(TOR_UNIT_TESTS)
+
+/*                 1  Historically used to indicate Named */
+#define RTR_INVALID 2  /**< Believed invalid. */
+#define RTR_REJECT  4  /**< We will not publish this router. */
+/*                 8  Historically used to avoid using this as a dir. */
+#define RTR_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
+/*                 32 Historically used to indicade Unnamed */
+
+#endif /* defined(PROCESS_DESCS_PRIVATE) || defined(TOR_UNIT_TESTS) */
+
+#ifdef TOR_UNIT_TESTS
+
+void authdir_init_fingerprint_list(void);
+
+authdir_config_t *authdir_return_fingerprint_list(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
 void dirserv_free_fingerprint_list(void);
-int dirserv_add_own_fingerprint(crypto_pk_t *pk);
 
+#ifdef HAVE_MODULE_DIRAUTH
+int dirserv_load_fingerprint_file(void);
 enum was_router_added_t dirserv_add_multiple_descriptors(
-                                     const char *desc, uint8_t purpose,
+                                     const char *desc, size_t desclen,
+                                     uint8_t purpose,
                                      const char *source,
                                      const char **msg);
 enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
                                                const char **msg,
                                                const char *source);
 
+int dirserv_would_reject_router(const routerstatus_t *rs,
+                                const vote_routerstatus_t *vrs);
 int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
                                    int complain,
                                    int *valid_out);
+int dirserv_add_own_fingerprint(crypto_pk_t *pk,
+                                const ed25519_public_key_t *edkey);
 uint32_t dirserv_router_get_status(const routerinfo_t *router,
                                    const char **msg,
                                    int severity);
 void dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                       uint32_t authstatus);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+static inline int
+dirserv_load_fingerprint_file(void)
+{
+  return 0;
+}
+static inline enum was_router_added_t
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
+                                 const char *source,
+                                 const char **msg)
+{
+  (void)desc;
+  (void)desclen;
+  (void)purpose;
+  (void)source;
+  *msg = "No directory authority support";
+  return (enum was_router_added_t)0;
+}
+static inline enum was_router_added_t
+dirserv_add_descriptor(routerinfo_t *ri,
+                       const char **msg,
+                       const char *source)
+{
+  (void)ri;
+  (void)source;
+  *msg = "No directory authority support";
+  return (enum was_router_added_t)0;
+}
+static inline int
+dirserv_would_reject_router(const routerstatus_t *rs,
+                            const vote_routerstatus_t *vrs)
+{
+  (void)rs;
+  (void)vrs;
+  return 0;
+}
+static inline int
+authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                               int complain,
+                               int *valid_out)
+{
+  (void)ri;
+  (void)complain;
+  *msg = "No directory authority support";
+  *valid_out = 0;
+  return 0;
+}
+static inline int
+dirserv_add_own_fingerprint(crypto_pk_t *pk, const ed25519_public_key_t *edkey)
+{
+  (void)pk;
+  (void)edkey;
+  return 0;
+}
+static inline uint32_t
+dirserv_router_get_status(const routerinfo_t *router,
+                          const char **msg,
+                          int severity)
+{
+  (void)router;
+  (void)severity;
+  if (msg)
+    *msg = "No directory authority support";
+  return 0;
+}
+static inline void
+dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                 uint32_t authstatus)
+{
+  (void)node;
+  (void)authstatus;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-int dirserv_would_reject_router(const routerstatus_t *rs);
+#ifdef TOR_UNIT_TESTS
+STATIC int dirserv_router_has_valid_address(routerinfo_t *ri);
+STATIC bool dirserv_rejects_tor_version(const char *platform,
+                                        const char **msg);
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_RECV_UPLOADS_H) */
diff --git a/src/feature/dirauth/reachability.c b/src/feature/dirauth/reachability.c
index 883b692cbb..65fa27ed80 100644
--- a/src/feature/dirauth/reachability.c
+++ b/src/feature/dirauth/reachability.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,7 @@
 #include "core/or/channeltls.h"
 #include "core/or/command.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo.h"
@@ -24,6 +25,7 @@
 #include "feature/nodelist/torcert.h"
 #include "feature/stats/rephist.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerlist_st.h"
@@ -53,7 +55,7 @@ dirserv_orconn_tls_done(const tor_addr_t *addr,
 
   ri = node->ri;
 
-  if (get_options()->AuthDirTestEd25519LinkKeys &&
+  if (dirauth_get_options()->AuthDirTestEd25519LinkKeys &&
       node_supports_ed25519_link_authentication(node, 1) &&
       ri->cache_info.signing_key_cert) {
     /* We allow the node to have an ed25519 key if we haven't been told one in
@@ -125,7 +127,7 @@ dirserv_should_launch_reachability_test(const routerinfo_t *ri,
 void
 dirserv_single_reachability_test(time_t now, routerinfo_t *router)
 {
-  const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
   channel_t *chan = NULL;
   const node_t *node = NULL;
   tor_addr_t router_addr;
@@ -136,7 +138,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
   node = node_get_by_id(router->cache_info.identity_digest);
   tor_assert(node);
 
-  if (options->AuthDirTestEd25519LinkKeys &&
+  if (dirauth_options->AuthDirTestEd25519LinkKeys &&
       node_supports_ed25519_link_authentication(node, 1) &&
       router->cache_info.signing_key_cert) {
     ed_id_key = &router->cache_info.signing_key_cert->signing_key;
@@ -154,7 +156,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
   if (chan) command_setup_channel(chan);
 
   /* Possible IPv6. */
-  if (get_options()->AuthDirHasIPv6Connectivity == 1 &&
+  if (dirauth_get_options()->AuthDirHasIPv6Connectivity == 1 &&
       !tor_addr_is_null(&router->ipv6_addr)) {
     char addrstr[TOR_ADDR_BUF_LEN];
     log_debug(LD_OR, "Testing reachability of %s at %s:%u.",
diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h
index 5a938673ff..19448a67f3 100644
--- a/src/feature/dirauth/reachability.h
+++ b/src/feature/dirauth/reachability.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,13 +24,27 @@
 #define REACHABILITY_TEST_CYCLE_PERIOD \
   (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
 
+#ifdef HAVE_MODULE_DIRAUTH
+void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
+void dirserv_test_reachability(time_t now);
+
+int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old);
 void dirserv_orconn_tls_done(const tor_addr_t *addr,
                              uint16_t or_port,
                              const char *digest_rcvd,
                              const struct ed25519_public_key_t *ed_id_rcvd);
-int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
-                                            const routerinfo_t *ri_old);
-void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
-void dirserv_test_reachability(time_t now);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+#define dirserv_single_reachability_test(now, router) \
+  (((void)(now)),((void)(router)))
+#define dirserv_test_reachability(now) \
+  (((void)(now)))
+
+#define dirserv_should_launch_reachability_test(ri, ri_old) \
+  (((void)(ri)),((void)(ri_old)),0)
+#define dirserv_orconn_tls_done(addr, or_port, digest_rcvd, ed_id_rcvd) \
+  (((void)(addr)),((void)(or_port)),((void)(digest_rcvd)), \
+   ((void)(ed_id_rcvd)))
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif
+#endif /* !defined(TOR_REACHABILITY_H) */
diff --git a/src/feature/dirauth/recommend_pkg.c b/src/feature/dirauth/recommend_pkg.c
index 0456ff8463..84254566c6 100644
--- a/src/feature/dirauth/recommend_pkg.c
+++ b/src/feature/dirauth/recommend_pkg.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h
index 8200d78f72..dcd9f8be8a 100644
--- a/src/feature/dirauth/recommend_pkg.h
+++ b/src/feature/dirauth/recommend_pkg.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,18 @@
 #ifndef TOR_RECOMMEND_PKG_H
 #define TOR_RECOMMEND_PKG_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 int validate_recommended_package_line(const char *line);
 
-#endif
+#else
+
+static inline int
+validate_recommended_package_line(const char *line)
+{
+  (void) line;
+  return 0;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(TOR_RECOMMEND_PKG_H) */
diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c
index 34b2283250..fd55008242 100644
--- a/src/feature/dirauth/shared_random.c
+++ b/src/feature/dirauth/shared_random.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -90,7 +90,7 @@
 #include "core/or/or.h"
 #include "feature/dirauth/shared_random.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
@@ -99,29 +99,31 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "feature/dirauth/shared_random_state.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_sys.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 
-/* String prefix of shared random values in votes/consensuses. */
+/** String prefix of shared random values in votes/consensuses. */
 static const char previous_srv_str[] = "shared-rand-previous-value";
 static const char current_srv_str[] = "shared-rand-current-value";
 static const char commit_ns_str[] = "shared-rand-commit";
 static const char sr_flag_ns_str[] = "shared-rand-participate";
 
-/* The value of the consensus param AuthDirNumSRVAgreements found in the
+/** The value of the consensus param AuthDirNumSRVAgreements found in the
  * vote. This is set once the consensus creation subsystem requests the
  * SRV(s) that should be put in the consensus. We use this value to decide
  * if we keep or not an SRV. */
 static int32_t num_srv_agreements_from_vote;
 
-/* Return a heap allocated copy of the SRV <b>orig</b>. */
-STATIC sr_srv_t *
-srv_dup(const sr_srv_t *orig)
+/** Return a heap allocated copy of the SRV <b>orig</b>. */
+sr_srv_t *
+sr_srv_dup(const sr_srv_t *orig)
 {
   sr_srv_t *duplicate = NULL;
 
@@ -135,7 +137,7 @@ srv_dup(const sr_srv_t *orig)
   return duplicate;
 }
 
-/* Allocate a new commit object and initializing it with <b>rsa_identity</b>
+/** Allocate a new commit object and initializing it with <b>rsa_identity</b>
  * that MUST be provided. The digest algorithm is set to the default one
  * that is supported. The rest is uninitialized. This never returns NULL. */
 static sr_commit_t *
@@ -153,7 +155,7 @@ commit_new(const char *rsa_identity)
   return commit;
 }
 
-/* Issue a log message describing <b>commit</b>. */
+/** Issue a log message describing <b>commit</b>. */
 static void
 commit_log(const sr_commit_t *commit)
 {
@@ -166,7 +168,7 @@ commit_log(const sr_commit_t *commit)
             commit->reveal_ts, safe_str(commit->encoded_reveal));
 }
 
-/* Make sure that the commitment and reveal information in <b>commit</b>
+/** Make sure that the commitment and reveal information in <b>commit</b>
  * match. If they match return 0, return -1 otherwise. This function MUST be
  * used everytime we receive a new reveal value. Furthermore, the commit
  * object MUST have a reveal value and the hash of the reveal value. */
@@ -220,15 +222,15 @@ verify_commit_and_reveal(const sr_commit_t *commit)
   return -1;
 }
 
-/* Return true iff the commit contains an encoded reveal value. */
+/** Return true iff the commit contains an encoded reveal value. */
 STATIC int
 commit_has_reveal_value(const sr_commit_t *commit)
 {
-  return !tor_mem_is_zero(commit->encoded_reveal,
+  return !fast_mem_is_zero(commit->encoded_reveal,
                           sizeof(commit->encoded_reveal));
 }
 
-/* Parse the encoded commit. The format is:
+/** Parse the encoded commit. The format is:
  *    base64-encode( TIMESTAMP || H(REVEAL) )
  *
  * If successfully decoded and parsed, commit is updated and 0 is returned.
@@ -283,7 +285,7 @@ commit_decode(const char *encoded, sr_commit_t *commit)
   return -1;
 }
 
-/* Parse the b64 blob at <b>encoded</b> containing reveal information and
+/** Parse the b64 blob at <b>encoded</b> containing reveal information and
  * store the information in-place in <b>commit</b>. Return 0 on success else
  * a negative value. */
 STATIC int
@@ -333,7 +335,7 @@ reveal_decode(const char *encoded, sr_commit_t *commit)
   return -1;
 }
 
-/* Encode a reveal element using a given commit object to dst which is a
+/** Encode a reveal element using a given commit object to dst which is a
  * buffer large enough to put the base64-encoded reveal construction. The
  * format is as follow:
  *     REVEAL = base64-encode( TIMESTAMP || H(RN) )
@@ -362,7 +364,7 @@ reveal_encode(const sr_commit_t *commit, char *dst, size_t len)
   return ret;
 }
 
-/* Encode the given commit object to dst which is a buffer large enough to
+/** Encode the given commit object to dst which is a buffer large enough to
  * put the base64-encoded commit. The format is as follow:
  *     COMMIT = base64-encode( TIMESTAMP || H(H(RN)) )
  * Return base64 encoded length on success else a negative value.
@@ -388,14 +390,14 @@ commit_encode(const sr_commit_t *commit, char *dst, size_t len)
   return base64_encode(dst, len, buf, sizeof(buf), 0);
 }
 
-/* Cleanup both our global state and disk state. */
+/** Cleanup both our global state and disk state. */
 static void
 sr_cleanup(void)
 {
   sr_state_free_all();
 }
 
-/* Using <b>commit</b>, return a newly allocated string containing the commit
+/** Using <b>commit</b>, return a newly allocated string containing the commit
  * information that should be used during SRV calculation. It's the caller
  * responsibility to free the memory. Return NULL if this is not a commit to be
  * used for SRV calculation. */
@@ -414,7 +416,7 @@ get_srv_element_from_commit(const sr_commit_t *commit)
   return element;
 }
 
-/* Return a srv object that is built with the construction:
+/** Return a srv object that is built with the construction:
  *    SRV = SHA3-256("shared-random" | INT_8(reveal_num) |
  *                   INT_4(version) | HASHED_REVEALS | previous_SRV)
  * This function cannot fail. */
@@ -456,7 +458,7 @@ generate_srv(const char *hashed_reveals, uint64_t reveal_num,
   return srv;
 }
 
-/* Compare reveal values and return the result. This should exclusively be
+/** Compare reveal values and return the result. This should exclusively be
  * used by smartlist_sort(). */
 static int
 compare_reveal_(const void **_a, const void **_b)
@@ -466,7 +468,7 @@ compare_reveal_(const void **_a, const void **_b)
                      sizeof(a->hashed_reveal));
 }
 
-/* Given <b>commit</b> give the line that we should place in our votes.
+/** Given <b>commit</b> give the line that we should place in our votes.
  * It's the responsibility of the caller to free the string. */
 static char *
 get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
@@ -486,7 +488,7 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
   {
     /* Send a reveal value for this commit if we have one. */
     const char *reveal_str = commit->encoded_reveal;
-    if (tor_mem_is_zero(commit->encoded_reveal,
+    if (fast_mem_is_zero(commit->encoded_reveal,
                         sizeof(commit->encoded_reveal))) {
       reveal_str = "";
     }
@@ -506,7 +508,7 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
   return vote_line;
 }
 
-/* Return a heap allocated string that contains the given <b>srv</b> string
+/** Return a heap allocated string that contains the given <b>srv</b> string
  * representation formatted for a networkstatus document using the
  * <b>key</b> as the start of the line. This doesn't return NULL. */
 static char *
@@ -524,7 +526,7 @@ srv_to_ns_string(const sr_srv_t *srv, const char *key)
   return srv_str;
 }
 
-/* Given the previous SRV and the current SRV, return a heap allocated
+/** Given the previous SRV and the current SRV, return a heap allocated
  * string with their data that could be put in a vote or a consensus. Caller
  * must free the returned string.  Return NULL if no SRVs were provided. */
 static char *
@@ -557,7 +559,7 @@ get_ns_str_from_sr_values(const sr_srv_t *prev_srv, const sr_srv_t *cur_srv)
   return srv_str;
 }
 
-/* Return 1 iff the two commits have the same commitment values. This
+/** Return 1 iff the two commits have the same commitment values. This
  * function does not care about reveal values. */
 STATIC int
 commitments_are_the_same(const sr_commit_t *commit_one,
@@ -572,7 +574,7 @@ commitments_are_the_same(const sr_commit_t *commit_one,
   return 1;
 }
 
-/* We just received a commit from the vote of authority with
+/** We just received a commit from the vote of authority with
  * <b>identity_digest</b>. Return 1 if this commit is authorititative that
  * is, it belongs to the authority that voted it. Else return 0 if not. */
 STATIC int
@@ -586,7 +588,7 @@ commit_is_authoritative(const sr_commit_t *commit,
                     sizeof(commit->rsa_identity));
 }
 
-/* Decide if the newly received <b>commit</b> should be kept depending on
+/** Decide if the newly received <b>commit</b> should be kept depending on
  * the current phase and state of the protocol. The <b>voter_key</b> is the
  * RSA identity key fingerprint of the authority's vote from which the
  * commit comes from. The <b>phase</b> is the phase we should be validating
@@ -705,7 +707,7 @@ should_keep_commit(const sr_commit_t *commit, const char *voter_key,
   return 0;
 }
 
-/* We are in reveal phase and we found a valid and verified <b>commit</b> in
+/** We are in reveal phase and we found a valid and verified <b>commit</b> in
  * a vote that contains reveal values that we could use. Update the commit
  * we have in our state. Never call this with an unverified commit. */
 STATIC void
@@ -726,7 +728,7 @@ save_commit_during_reveal_phase(const sr_commit_t *commit)
   sr_state_copy_reveal_info(saved_commit, commit);
 }
 
-/* Save <b>commit</b> to our persistent state. Depending on the current
+/** Save <b>commit</b> to our persistent state. Depending on the current
  * phase, different actions are taken. Steals reference of <b>commit</b>.
  * The commit object MUST be valid and verified before adding it to the
  * state. */
@@ -751,7 +753,7 @@ save_commit_to_state(sr_commit_t *commit)
   }
 }
 
-/* Return 1 if we should we keep an SRV voted by <b>n_agreements</b> auths.
+/** Return 1 if we should we keep an SRV voted by <b>n_agreements</b> auths.
  * Return 0 if we should ignore it. */
 static int
 should_keep_srv(int n_agreements)
@@ -781,7 +783,7 @@ should_keep_srv(int n_agreements)
   return 1;
 }
 
-/* Helper: compare two DIGEST256_LEN digests. */
+/** Helper: compare two DIGEST256_LEN digests. */
 static int
 compare_srvs_(const void **_a, const void **_b)
 {
@@ -789,7 +791,7 @@ compare_srvs_(const void **_a, const void **_b)
   return tor_memcmp(a->value, b->value, sizeof(a->value));
 }
 
-/* Return the most frequent member of the sorted list of DIGEST256_LEN
+/** Return the most frequent member of the sorted list of DIGEST256_LEN
  * digests in <b>sl</b> with the count of that most frequent element. */
 static sr_srv_t *
 smartlist_get_most_frequent_srv(const smartlist_t *sl, int *count_out)
@@ -806,7 +808,7 @@ compare_srv_(const void **_a, const void **_b)
                      sizeof(a->value));
 }
 
-/* Using a list of <b>votes</b>, return the SRV object from them that has
+/** Using a list of <b>votes</b>, return the SRV object from them that has
  * been voted by the majority of dirauths. If <b>current</b> is set, we look
  * for the current SRV value else the previous one. The returned pointer is
  * an object located inside a vote. NULL is returned if no appropriate value
@@ -868,7 +870,7 @@ get_majority_srv_from_votes(const smartlist_t *votes, int current)
   return the_srv;
 }
 
-/* Free a commit object. */
+/** Free a commit object. */
 void
 sr_commit_free_(sr_commit_t *commit)
 {
@@ -880,7 +882,7 @@ sr_commit_free_(sr_commit_t *commit)
   tor_free(commit);
 }
 
-/* Generate the commitment/reveal value for the protocol run starting at
+/** Generate the commitment/reveal value for the protocol run starting at
  * <b>timestamp</b>. <b>my_rsa_cert</b> is our authority RSA certificate. */
 sr_commit_t *
 sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
@@ -937,7 +939,8 @@ sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
   return NULL;
 }
 
-/* Compute the shared random value based on the active commits in our state. */
+/** Compute the shared random value based on the active commits in our
+ * state. */
 void
 sr_compute_srv(void)
 {
@@ -1010,7 +1013,7 @@ sr_compute_srv(void)
   tor_free(reveals);
 }
 
-/* Parse a commit from a vote or from our disk state and return a newly
+/** Parse a commit from a vote or from our disk state and return a newly
  * allocated commit object. NULL is returned on error.
  *
  * The commit's data is in <b>args</b> and the order matters very much:
@@ -1082,7 +1085,7 @@ sr_parse_commit(const smartlist_t *args)
   return NULL;
 }
 
-/* Called when we are done parsing a vote by <b>voter_key</b> that might
+/** Called when we are done parsing a vote by <b>voter_key</b> that might
  * contain some useful <b>commits</b>. Find if any of them should be kept
  * and update our state accordingly. Once done, the list of commitments will
  * be empty. */
@@ -1120,7 +1123,7 @@ sr_handle_received_commits(smartlist_t *commits, crypto_pk_t *voter_key)
   } SMARTLIST_FOREACH_END(commit);
 }
 
-/* Return a heap-allocated string containing commits that should be put in
+/** Return a heap-allocated string containing commits that should be put in
  * the votes. It's the responsibility of the caller to free the string.
  * This always return a valid string, either empty or with line(s). */
 char *
@@ -1129,7 +1132,7 @@ sr_get_string_for_vote(void)
   char *vote_str = NULL;
   digestmap_t *state_commits;
   smartlist_t *chunks = smartlist_new();
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
 
   /* Are we participating in the protocol? */
   if (!options->AuthDirSharedRandomness) {
@@ -1178,7 +1181,7 @@ sr_get_string_for_vote(void)
   return vote_str;
 }
 
-/* Return a heap-allocated string that should be put in the consensus and
+/** Return a heap-allocated string that should be put in the consensus and
  * contains the shared randomness values. It's the responsibility of the
  * caller to free the string. NULL is returned if no SRV(s) available.
  *
@@ -1194,7 +1197,7 @@ sr_get_string_for_consensus(const smartlist_t *votes,
                             int32_t num_srv_agreements)
 {
   char *srv_str;
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
 
   tor_assert(votes);
 
@@ -1222,7 +1225,7 @@ sr_get_string_for_consensus(const smartlist_t *votes,
   return NULL;
 }
 
-/* We just computed a new <b>consensus</b>. Update our state with the SRVs
+/** We just computed a new <b>consensus</b>. Update our state with the SRVs
  * from the consensus (might be NULL as well). Register the SRVs in our SR
  * state and prepare for the upcoming protocol round. */
 void
@@ -1253,15 +1256,15 @@ sr_act_post_consensus(const networkstatus_t *consensus)
      * decided by the majority. */
     sr_state_unset_fresh_srv();
     /* Set the SR values from the given consensus. */
-    sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv));
-    sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv));
+    sr_state_set_previous_srv(sr_srv_dup(consensus->sr_info.previous_srv));
+    sr_state_set_current_srv(sr_srv_dup(consensus->sr_info.current_srv));
   }
 
   /* Prepare our state so that it's ready for the next voting period. */
-  sr_state_update(voting_schedule_get_next_valid_after_time());
+  sr_state_update(dirauth_sched_get_next_valid_after_time());
 }
 
-/* Initialize shared random subsystem. This MUST be called early in the boot
+/** Initialize shared random subsystem. This MUST be called early in the boot
  * process of tor. Return 0 on success else -1 on error. */
 int
 sr_init(int save_to_disk)
@@ -1269,7 +1272,7 @@ sr_init(int save_to_disk)
   return sr_state_init(save_to_disk, 1);
 }
 
-/* Save our state to disk and cleanup everything. */
+/** Save our state to disk and cleanup everything. */
 void
 sr_save_and_cleanup(void)
 {
@@ -1279,7 +1282,7 @@ sr_save_and_cleanup(void)
 
 #ifdef TOR_UNIT_TESTS
 
-/* Set the global value of number of SRV agreements so the test can play
+/** Set the global value of number of SRV agreements so the test can play
  * along by calling specific functions that don't parse the votes prior for
  * the AuthDirNumSRVAgreements value. */
 void
diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h
index 25d95ebbc7..c4e259dcdb 100644
--- a/src/feature/dirauth/shared_random.h
+++ b/src/feature/dirauth/shared_random.h
@@ -1,86 +1,88 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_H
 #define TOR_SHARED_RANDOM_H
 
-/*
- * This file contains ABI/API of the shared random protocol defined in
+/**
+ * \file shared_random.h
+ *
+ * \brief This file contains ABI/API of the shared random protocol defined in
  * proposal #250. Every public functions and data structure are namespaced
  * with "sr_" which stands for shared random.
  */
 
 #include "core/or/or.h"
 
-/* Protocol version */
+/** Protocol version */
 #define SR_PROTO_VERSION  1
-/* Default digest algorithm. */
+/** Default digest algorithm. */
 #define SR_DIGEST_ALG DIGEST_SHA3_256
-/* Invariant token in the SRV calculation. */
+/** Invariant token in the SRV calculation. */
 #define SR_SRV_TOKEN "shared-random"
-/* Don't count the NUL terminated byte even though the TOKEN has it. */
+/** Don't count the NUL terminated byte even though the TOKEN has it. */
 #define SR_SRV_TOKEN_LEN (sizeof(SR_SRV_TOKEN) - 1)
 
-/* Length of the random number (in bytes). */
+/** Length of the random number (in bytes). */
 #define SR_RANDOM_NUMBER_LEN 32
-/* Size of a decoded commit value in a vote or state. It's a hash and a
+/** Size of a decoded commit value in a vote or state. It's a hash and a
  * timestamp. It adds up to 40 bytes. */
 #define SR_COMMIT_LEN (sizeof(uint64_t) + DIGEST256_LEN)
-/* Size of a decoded reveal value from a vote or state. It's a 64 bit
+/** Size of a decoded reveal value from a vote or state. It's a 64 bit
  * timestamp and the hashed random number. This adds up to 40 bytes. */
 #define SR_REVEAL_LEN (sizeof(uint64_t) + DIGEST256_LEN)
-/* Size of SRV message length. The construction is has follow:
+/** Size of SRV message length. The construction is has follow:
  *  "shared-random" | INT_8(reveal_num) | INT_4(version) | PREV_SRV */
 #define SR_SRV_MSG_LEN \
   (SR_SRV_TOKEN_LEN + sizeof(uint64_t) + sizeof(uint32_t) + DIGEST256_LEN)
 
-/* Length of base64 encoded commit NOT including the NUL terminated byte.
+/** Length of base64 encoded commit NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
 #define SR_COMMIT_BASE64_LEN (BASE64_LEN(SR_COMMIT_LEN))
-/* Length of base64 encoded reveal NOT including the NUL terminated byte.
+/** Length of base64 encoded reveal NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
 #define SR_REVEAL_BASE64_LEN (BASE64_LEN(SR_REVEAL_LEN))
-/* Length of base64 encoded shared random value. It's 32 bytes long so 44
+/** Length of base64 encoded shared random value. It's 32 bytes long so 44
  * bytes from the base64_encode_size formula. That includes the '='
  * character at the end. */
 #define SR_SRV_VALUE_BASE64_LEN (BASE64_LEN(DIGEST256_LEN))
 
-/* Assert if commit valid flag is not set. */
+/** Assert if commit valid flag is not set. */
 #define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid)
 
-/* Protocol phase. */
+/** Protocol phase. */
 typedef enum {
-  /* Commitment phase */
+  /** Commitment phase */
   SR_PHASE_COMMIT  = 1,
-  /* Reveal phase */
+  /** Reveal phase */
   SR_PHASE_REVEAL  = 2,
 } sr_phase_t;
 
-/* A shared random value (SRV). */
+/** A shared random value (SRV). */
 typedef struct sr_srv_t {
-  /* The number of reveal values used to derive this SRV. */
+  /** The number of reveal values used to derive this SRV. */
   uint64_t num_reveals;
-  /* The actual value. This is the stored result of SHA3-256. */
+  /** The actual value. This is the stored result of SHA3-256. */
   uint8_t value[DIGEST256_LEN];
 } sr_srv_t;
 
-/* A commit (either ours or from another authority). */
+/** A commit (either ours or from another authority). */
 typedef struct sr_commit_t {
-  /* Hashing algorithm used. */
+  /** Hashing algorithm used. */
   digest_algorithm_t alg;
-  /* Indicate if this commit has been verified thus valid. */
+  /** Indicate if this commit has been verified thus valid. */
   unsigned int valid:1;
 
   /* Commit owner info */
 
-  /* The RSA identity key of the authority and its base16 representation,
+  /** The RSA identity key of the authority and its base16 representation,
    * which includes the NUL terminated byte. */
   char rsa_identity[DIGEST_LEN];
   char rsa_identity_hex[HEX_DIGEST_LEN + 1];
 
   /* Commitment information */
 
-  /* Timestamp of reveal. Correspond to TIMESTAMP. */
+  /** Timestamp of reveal. Correspond to TIMESTAMP. */
   uint64_t reveal_ts;
   /* H(REVEAL) as found in COMMIT message. */
   char hashed_reveal[DIGEST256_LEN];
@@ -89,13 +91,13 @@ typedef struct sr_commit_t {
 
   /* Reveal information */
 
-  /* H(RN) which is what we used as the random value for this commit. We
+  /** H(RN) which is what we used as the random value for this commit. We
    * don't use the raw bytes since those are sent on the network thus
    * avoiding possible information leaks of our PRNG. */
   uint8_t random_number[SR_RANDOM_NUMBER_LEN];
-  /* Timestamp of commit. Correspond to TIMESTAMP. */
+  /** Timestamp of commit. Correspond to TIMESTAMP. */
   uint64_t commit_ts;
-  /* This is the whole reveal message. We use it during verification */
+  /** This is the whole reveal message. We use it during verification */
   char encoded_reveal[SR_REVEAL_BASE64_LEN + 1];
 } sr_commit_t;
 
@@ -110,7 +112,7 @@ int sr_init(int save_to_disk);
 void sr_save_and_cleanup(void);
 void sr_act_post_consensus(const networkstatus_t *consensus);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline int
 sr_init(int save_to_disk)
@@ -131,7 +133,7 @@ sr_act_post_consensus(const networkstatus_t *consensus)
   (void) consensus;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Public methods used only by dirauth code. */
 
@@ -154,6 +156,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
 void sr_compute_srv(void);
 sr_commit_t *sr_generate_our_commit(time_t timestamp,
                                     const authority_cert_t *my_rsa_cert);
+sr_srv_t *sr_srv_dup(const sr_srv_t *orig);
 
 #ifdef SHARED_RANDOM_PRIVATE
 
@@ -172,7 +175,6 @@ STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes,
                                              int current);
 
 STATIC void save_commit_to_state(sr_commit_t *commit);
-STATIC sr_srv_t *srv_dup(const sr_srv_t *orig);
 STATIC int commitments_are_the_same(const sr_commit_t *commit_one,
                                     const sr_commit_t *commit_two);
 STATIC int commit_is_authoritative(const sr_commit_t *commit,
@@ -191,4 +193,3 @@ void set_num_srv_agreements(int32_t value);
 #endif /* TOR_UNIT_TESTS */
 
 #endif /* !defined(TOR_SHARED_RANDOM_H) */
-
diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c
index b3e4a4ef92..07bc757506 100644
--- a/src/feature/dirauth/shared_random_state.c
+++ b/src/feature/dirauth/shared_random_state.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/nodelist/networkstatus.h"
@@ -20,23 +20,24 @@
 #include "feature/dirauth/shared_random.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "feature/dirauth/shared_random_state.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "lib/encoding/confline.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
-/* Default filename of the shared random state on disk. */
+/** Default filename of the shared random state on disk. */
 static const char default_fname[] = "sr-state";
 
-/* String representation of a protocol phase. */
+/** String representation of a protocol phase. */
 static const char *phase_str[] = { "unknown", "commit", "reveal" };
 
-/* Our shared random protocol state. There is only one possible state per
+/** Our shared random protocol state. There is only one possible state per
  * protocol run so this is the global state which is reset at every run once
  * the shared random value has been computed. */
 static sr_state_t *sr_state = NULL;
 
-/* Representation of our persistent state on disk. The sr_state above
+/** Representation of our persistent state on disk. The sr_state above
  * contains the data parsed from this state. When we save to disk, we
  * translate the sr_state to this sr_disk_state. */
 static sr_disk_state_t *sr_disk_state = NULL;
@@ -50,24 +51,18 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue";
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t);
 
-/* These next two are duplicates or near-duplicates from config.c */
-#define VAR(name, conftype, member, initvalue)                              \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member),      \
-      initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) }
-/* As VAR, but the option name and member name are the same. */
-#define V(member, conftype, initvalue) \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
-/* Our persistent state magic number. */
-#define SR_DISK_STATE_MAGIC 0x98AB1254
 
-static int
-disk_state_validate_cb(void *old_state, void *state, void *default_state,
-                       int from_setconf, char **msg);
-static void disk_state_free_cb(void *);
+/** Our persistent state magic number. */
+#define SR_DISK_STATE_MAGIC 0x98AB1254
 
-/* Array of variables that are saved to disk as a persistent state. */
-static config_var_t state_vars[] = {
-  V(Version,                    UINT, "0"),
+/** Array of variables that are saved to disk as a persistent state. */
+// clang-format off
+static const config_var_t state_vars[] = {
+  V(Version,                    POSINT, "0"),
   V(TorVersion,                 STRING, NULL),
   V(ValidAfter,                 ISOTIME, NULL),
   V(ValidUntil,                 ISOTIME, NULL),
@@ -79,29 +74,45 @@ static config_var_t state_vars[] = {
   VAR("SharedRandCurrentValue", LINELIST_S, SharedRandValues, NULL),
   END_OF_CONFIG_VARS
 };
+// clang-format on
 
-/* "Extra" variable in the state that receives lines we can't parse. This
+/** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST,
-  offsetof(sr_disk_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines)
+static const struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(sr_disk_state_t, ExtraLines),
 };
 
-/* Configuration format of sr_disk_state_t. */
+/** Configuration format of sr_disk_state_t. */
 static const config_format_t state_format = {
-  sizeof(sr_disk_state_t),
-  SR_DISK_STATE_MAGIC,
-  offsetof(sr_disk_state_t, magic_),
-  NULL,
-  NULL,
-  state_vars,
-  disk_state_validate_cb,
-  disk_state_free_cb,
-  &state_extra_var,
+  .size = sizeof(sr_disk_state_t),
+  .magic = {
+   "sr_disk_state_t",
+   SR_DISK_STATE_MAGIC,
+   offsetof(sr_disk_state_t, magic_),
+  },
+  .vars = state_vars,
+  .extra = &state_extra_var,
 };
 
-/* Return a string representation of a protocol phase. */
+/** Global configuration manager for the shared-random state file */
+static config_mgr_t *shared_random_state_mgr = NULL;
+
+/** Return the configuration manager for the shared-random state file. */
+static const config_mgr_t *
+get_srs_mgr(void)
+{
+  if (PREDICT_UNLIKELY(shared_random_state_mgr == NULL)) {
+    shared_random_state_mgr = config_mgr_new(&state_format);
+    config_mgr_freeze(shared_random_state_mgr);
+  }
+  return shared_random_state_mgr;
+}
+
+static void state_query_del_(sr_state_object_t obj_type, void *data);
+
+/** Return a string representation of a protocol phase. */
 STATIC const char *
 get_phase_str(sr_phase_t phase)
 {
@@ -119,7 +130,7 @@ get_phase_str(sr_phase_t phase)
 
   return the_string;
 }
-/* Return the time we should expire the state file created at <b>now</b>.
+/** Return the time we should expire the state file created at <b>now</b>.
  * We expire the state file in the beginning of the next protocol run. */
 STATIC time_t
 get_state_valid_until_time(time_t now)
@@ -130,7 +141,7 @@ get_state_valid_until_time(time_t now)
 
   voting_interval = get_voting_interval();
   /* Find the time the current round started. */
-  beginning_of_current_round = get_start_time_of_current_round();
+  beginning_of_current_round = dirauth_sched_get_cur_valid_after_time();
 
   /* Find how many rounds are left till the end of the protocol run */
   current_round = (now / voting_interval) % total_rounds;
@@ -150,7 +161,7 @@ get_state_valid_until_time(time_t now)
   return valid_until;
 }
 
-/* Given the consensus 'valid-after' time, return the protocol phase we should
+/** Given the consensus 'valid-after' time, return the protocol phase we should
  * be in. */
 STATIC sr_phase_t
 get_sr_protocol_phase(time_t valid_after)
@@ -170,7 +181,7 @@ get_sr_protocol_phase(time_t valid_after)
   }
 }
 
-/* Add the given <b>commit</b> to <b>state</b>. It MUST be a valid commit
+/** Add the given <b>commit</b> to <b>state</b>. It MUST be a valid commit
  * and there shouldn't be a commit from the same authority in the state
  * already else verification hasn't been done prior. This takes ownership of
  * the commit once in our state. */
@@ -195,7 +206,7 @@ commit_add_to_state(sr_commit_t *commit, sr_state_t *state)
   }
 }
 
-/* Helper: deallocate a commit object. (Used with digestmap_free(), which
+/** Helper: deallocate a commit object. (Used with digestmap_free(), which
  * requires a function pointer whose argument is void *). */
 static void
 commit_free_(void *p)
@@ -206,7 +217,7 @@ commit_free_(void *p)
 #define state_free(val) \
   FREE_AND_NULL(sr_state_t, state_free_, (val))
 
-/* Free a state that was allocated with state_new(). */
+/** Free a state that was allocated with state_new(). */
 static void
 state_free_(sr_state_t *state)
 {
@@ -220,7 +231,7 @@ state_free_(sr_state_t *state)
   tor_free(state);
 }
 
-/* Allocate an sr_state_t object and returns it. If no <b>fname</b>, the
+/** Allocate an sr_state_t object and returns it. If no <b>fname</b>, the
  * default file name is used. This function does NOT initialize the state
  * timestamp, phase or shared random value. NULL is never returned. */
 static sr_state_t *
@@ -239,7 +250,7 @@ state_new(const char *fname, time_t now)
   return new_state;
 }
 
-/* Set our global state pointer with the one given. */
+/** Set our global state pointer with the one given. */
 static void
 state_set(sr_state_t *state)
 {
@@ -253,34 +264,33 @@ state_set(sr_state_t *state)
 #define disk_state_free(val) \
   FREE_AND_NULL(sr_disk_state_t, disk_state_free_, (val))
 
-/* Free an allocated disk state. */
+/** Free an allocated disk state. */
 static void
 disk_state_free_(sr_disk_state_t *state)
 {
   if (state == NULL) {
     return;
   }
-  config_free(&state_format, state);
+  config_free(get_srs_mgr(), state);
 }
 
-/* Allocate a new disk state, initialize it and return it. */
+/** Allocate a new disk state, initialize it and return it. */
 static sr_disk_state_t *
 disk_state_new(time_t now)
 {
-  sr_disk_state_t *new_state = tor_malloc_zero(sizeof(*new_state));
+  sr_disk_state_t *new_state = config_new(get_srs_mgr());
 
-  new_state->magic_ = SR_DISK_STATE_MAGIC;
   new_state->Version = SR_PROTO_VERSION;
   new_state->TorVersion = tor_strdup(get_version());
   new_state->ValidUntil = get_state_valid_until_time(now);
   new_state->ValidAfter = now;
 
   /* Init config format. */
-  config_init(&state_format, new_state);
+  config_init(get_srs_mgr(), new_state);
   return new_state;
 }
 
-/* Set our global disk state with the given state. */
+/** Set our global disk state with the given state. */
 static void
 disk_state_set(sr_disk_state_t *state)
 {
@@ -291,7 +301,7 @@ disk_state_set(sr_disk_state_t *state)
   sr_disk_state = state;
 }
 
-/* Return -1 if the disk state is invalid (something in there that we can't or
+/** Return -1 if the disk state is invalid (something in there that we can't or
  * shouldn't use). Return 0 if everything checks out. */
 static int
 disk_state_validate(const sr_disk_state_t *state)
@@ -326,31 +336,7 @@ disk_state_validate(const sr_disk_state_t *state)
   return -1;
 }
 
-/* Validate the disk state (NOP for now). */
-static int
-disk_state_validate_cb(void *old_state, void *state, void *default_state,
-                       int from_setconf, char **msg)
-{
-  /* We don't use these; only options do. */
-  (void) from_setconf;
-  (void) default_state;
-  (void) old_state;
-
-  /* This is called by config_dump which is just before we are about to
-   * write it to disk. At that point, our global memory state has been
-   * copied to the disk state so it's fair to assume it's trustable. */
-  (void) state;
-  (void) msg;
-  return 0;
-}
-
-static void
-disk_state_free_cb(void *state)
-{
-  disk_state_free_(state);
-}
-
-/* Parse the Commit line(s) in the disk state and translate them to the
+/** Parse the Commit line(s) in the disk state and translate them to the
  * the memory state. Return 0 on success else -1 on error. */
 static int
 disk_state_parse_commits(sr_state_t *state,
@@ -405,7 +391,7 @@ disk_state_parse_commits(sr_state_t *state,
   return -1;
 }
 
-/* Parse a share random value line from the disk state and save it to dst
+/** Parse a share random value line from the disk state and save it to dst
  * which is an allocated srv object. Return 0 on success else -1. */
 static int
 disk_state_parse_srv(const char *value, sr_srv_t *dst)
@@ -440,7 +426,7 @@ disk_state_parse_srv(const char *value, sr_srv_t *dst)
   return ret;
 }
 
-/* Parse both SharedRandCurrentValue and SharedRandPreviousValue line from
+/** Parse both SharedRandCurrentValue and SharedRandPreviousValue line from
  * the state. Return 0 on success else -1. */
 static int
 disk_state_parse_sr_values(sr_state_t *state,
@@ -491,7 +477,7 @@ disk_state_parse_sr_values(sr_state_t *state,
   return -1;
 }
 
-/* Parse the given disk state and set a newly allocated state. On success,
+/** Parse the given disk state and set a newly allocated state. On success,
  * return that state else NULL. */
 static sr_state_t *
 disk_state_parse(const sr_disk_state_t *new_disk_state)
@@ -525,7 +511,7 @@ disk_state_parse(const sr_disk_state_t *new_disk_state)
   return NULL;
 }
 
-/* From a valid commit object and an allocated config line, set the line's
+/** From a valid commit object and an allocated config line, set the line's
  * value to the state string representation of a commit. */
 static void
 disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
@@ -535,7 +521,7 @@ disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
   tor_assert(commit);
   tor_assert(line);
 
-  if (!tor_mem_is_zero(commit->encoded_reveal,
+  if (!fast_mem_is_zero(commit->encoded_reveal,
                        sizeof(commit->encoded_reveal))) {
     /* Add extra whitespace so we can format the line correctly. */
     tor_asprintf(&reveal_str, " %s", commit->encoded_reveal);
@@ -552,7 +538,7 @@ disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
   }
 }
 
-/* From a valid srv object and an allocated config line, set the line's
+/** From a valid srv object and an allocated config line, set the line's
  * value to the state string representation of a shared random value. */
 static void
 disk_state_put_srv_line(const sr_srv_t *srv, config_line_t *line)
@@ -570,7 +556,7 @@ disk_state_put_srv_line(const sr_srv_t *srv, config_line_t *line)
   tor_asprintf(&line->value, "%" PRIu64 " %s", srv->num_reveals, encoded);
 }
 
-/* Reset disk state that is free allocated memory and zeroed the object. */
+/** Reset disk state that is free allocated memory and zeroed the object. */
 static void
 disk_state_reset(void)
 {
@@ -580,15 +566,16 @@ disk_state_reset(void)
   config_free_lines(sr_disk_state->ExtraLines);
   tor_free(sr_disk_state->TorVersion);
 
-  /* Clean up the struct */
-  memset(sr_disk_state, 0, sizeof(*sr_disk_state));
+  /* Clear other fields. */
+  sr_disk_state->ValidAfter = 0;
+  sr_disk_state->ValidUntil = 0;
+  sr_disk_state->Version = 0;
 
   /* Reset it with useful data */
-  sr_disk_state->magic_ = SR_DISK_STATE_MAGIC;
   sr_disk_state->TorVersion = tor_strdup(get_version());
 }
 
-/* Update our disk state based on our global SR state. */
+/** Update our disk state based on our global SR state. */
 static void
 disk_state_update(void)
 {
@@ -632,7 +619,7 @@ disk_state_update(void)
   } DIGESTMAP_FOREACH_END;
 }
 
-/* Load state from disk and put it into our disk state. If the state passes
+/** Load state from disk and put it into our disk state. If the state passes
  * validation, our global state will be updated with it. Return 0 on
  * success. On error, -EINVAL is returned if the state on disk did contained
  * something malformed or is unreadable. -ENOENT is returned indicating that
@@ -650,7 +637,7 @@ disk_state_load_from_disk(void)
   return ret;
 }
 
-/* Helper for disk_state_load_from_disk(). */
+/** Helper for disk_state_load_from_disk(). */
 STATIC int
 disk_state_load_from_disk_impl(const char *fname)
 {
@@ -679,7 +666,7 @@ disk_state_load_from_disk_impl(const char *fname)
     }
 
     disk_state = disk_state_new(time(NULL));
-    config_assign(&state_format, disk_state, lines, 0, &errmsg);
+    config_assign(get_srs_mgr(), disk_state, lines, 0, &errmsg);
     config_free_lines(lines);
     if (errmsg) {
       log_warn(LD_DIR, "SR: Reading state error: %s", errmsg);
@@ -712,7 +699,7 @@ disk_state_load_from_disk_impl(const char *fname)
   return ret;
 }
 
-/* Save the disk state to disk but before that update it from the current
+/** Save the disk state to disk but before that update it from the current
  * state so we always have the latest. Return 0 on success else -1. */
 static int
 disk_state_save_to_disk(void)
@@ -732,7 +719,7 @@ disk_state_save_to_disk(void)
   /* Make sure that our disk state is up to date with our memory state
    * before saving it to disk. */
   disk_state_update();
-  state = config_dump(&state_format, NULL, sr_disk_state, 0, 0);
+  state = config_dump(get_srs_mgr(), NULL, sr_disk_state, 0, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&content,
                "# Tor shared random state file last generated on %s "
@@ -756,7 +743,7 @@ disk_state_save_to_disk(void)
   return ret;
 }
 
-/* Reset our state to prepare for a new protocol run. Once this returns, all
+/** Reset our state to prepare for a new protocol run. Once this returns, all
  * commits in the state will be removed and freed. */
 STATIC void
 reset_state_for_new_protocol_run(time_t valid_after)
@@ -777,7 +764,7 @@ reset_state_for_new_protocol_run(time_t valid_after)
   sr_state_delete_commits();
 }
 
-/* This is the first round of the new protocol run starting at
+/** This is the first round of the new protocol run starting at
  * <b>valid_after</b>. Do the necessary housekeeping. */
 STATIC void
 new_protocol_run(time_t valid_after)
@@ -811,7 +798,7 @@ new_protocol_run(time_t valid_after)
   }
 }
 
-/* Return 1 iff the <b>next_phase</b> is a phase transition from the current
+/** Return 1 iff the <b>next_phase</b> is a phase transition from the current
  * phase that is it's different. */
 STATIC int
 is_phase_transition(sr_phase_t next_phase)
@@ -819,7 +806,7 @@ is_phase_transition(sr_phase_t next_phase)
   return sr_state->phase != next_phase;
 }
 
-/* Helper function: return a commit using the RSA fingerprint of the
+/** Helper function: return a commit using the RSA fingerprint of the
  * authority or NULL if no such commit is known. */
 static sr_commit_t *
 state_query_get_commit(const char *rsa_fpr)
@@ -828,11 +815,14 @@ state_query_get_commit(const char *rsa_fpr)
   return digestmap_get(sr_state->commits, rsa_fpr);
 }
 
-/* Helper function: This handles the GET state action using an
+/** Helper function: This handles the GET state action using an
  * <b>obj_type</b> and <b>data</b> needed for the action. */
 static void *
 state_query_get_(sr_state_object_t obj_type, const void *data)
 {
+  if (BUG(!sr_state))
+    return NULL;
+
   void *obj = NULL;
 
   switch (obj_type) {
@@ -860,24 +850,45 @@ state_query_get_(sr_state_object_t obj_type, const void *data)
   return obj;
 }
 
-/* Helper function: This handles the PUT state action using an
- * <b>obj_type</b> and <b>data</b> needed for the action. */
+/** Helper function: This handles the PUT state action using an
+ * <b>obj_type</b> and <b>data</b> needed for the action.
+ * PUT frees the previous data before replacing it, if needed. */
 static void
 state_query_put_(sr_state_object_t obj_type, void *data)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
     sr_commit_t *commit = data;
     tor_assert(commit);
+    /* commit_add_to_state() frees the old commit, if there is one */
     commit_add_to_state(commit, sr_state);
     break;
   }
   case SR_STATE_OBJ_CURSRV:
-    sr_state->current_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->current_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_CURSRV, NULL);
+      sr_state->current_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_PREVSRV:
-    sr_state->previous_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->previous_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_PREVSRV, NULL);
+      sr_state->previous_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_VALID_AFTER:
     sr_state->valid_after = *((time_t *) data);
@@ -892,11 +903,14 @@ state_query_put_(sr_state_object_t obj_type, void *data)
   }
 }
 
-/* Helper function: This handles the DEL_ALL state action using an
+/** Helper function: This handles the DEL_ALL state action using an
  * <b>obj_type</b> and <b>data</b> needed for the action. */
 static void
 state_query_del_all_(sr_state_object_t obj_type)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
@@ -907,7 +921,7 @@ state_query_del_all_(sr_state_object_t obj_type)
     } DIGESTMAP_FOREACH_END;
     break;
   }
-  /* The following object are _NOT_ suppose to be removed. */
+  /* The following objects are _NOT_ supposed to be removed. */
   case SR_STATE_OBJ_CURSRV:
   case SR_STATE_OBJ_PREVSRV:
   case SR_STATE_OBJ_PHASE:
@@ -918,13 +932,16 @@ state_query_del_all_(sr_state_object_t obj_type)
   }
 }
 
-/* Helper function: This handles the DEL state action using an
+/** Helper function: This handles the DEL state action using an
  * <b>obj_type</b> and <b>data</b> needed for the action. */
 static void
 state_query_del_(sr_state_object_t obj_type, void *data)
 {
   (void) data;
 
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_PREVSRV:
     tor_free(sr_state->previous_srv);
@@ -941,7 +958,7 @@ state_query_del_(sr_state_object_t obj_type, void *data)
   }
 }
 
-/* Query state using an <b>action</b> for an object type <b>obj_type</b>.
+/** Query state using an <b>action</b> for an object type <b>obj_type</b>.
  * The <b>data</b> pointer needs to point to an object that the action needs
  * to use and if anything is required to be returned, it is stored in
  * <b>out</b>.
@@ -983,7 +1000,7 @@ state_query(sr_state_action_t action, sr_state_object_t obj_type,
   }
 }
 
-/* Delete the current SRV value from the state freeing it and the value is set
+/** Delete the current SRV value from the state freeing it and the value is set
  * to NULL meaning empty. */
 STATIC void
 state_del_current_srv(void)
@@ -991,7 +1008,7 @@ state_del_current_srv(void)
   state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_CURSRV, NULL, NULL);
 }
 
-/* Delete the previous SRV value from the state freeing it and the value is
+/** Delete the previous SRV value from the state freeing it and the value is
  * set to NULL meaning empty. */
 STATIC void
 state_del_previous_srv(void)
@@ -999,20 +1016,20 @@ state_del_previous_srv(void)
   state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL);
 }
 
-/* Rotate SRV value by freeing the previous value, assigning the current
- * value to the previous one and nullifying the current one. */
+/** Rotate SRV value by setting the previous SRV to the current SRV, and
+ * clearing the current SRV. */
 STATIC void
 state_rotate_srv(void)
 {
   /* First delete previous SRV from the state. Object will be freed. */
   state_del_previous_srv();
-  /* Set previous SRV with the current one. */
-  sr_state_set_previous_srv(sr_state_get_current_srv());
-  /* Nullify the current srv. */
+  /* Set previous SRV to a copy of the current one. */
+  sr_state_set_previous_srv(sr_srv_dup(sr_state_get_current_srv()));
+  /* Free and NULL the current srv. */
   sr_state_set_current_srv(NULL);
 }
 
-/* Set valid after time in the our state. */
+/** Set valid after time in the our state. */
 void
 sr_state_set_valid_after(time_t valid_after)
 {
@@ -1020,16 +1037,19 @@ sr_state_set_valid_after(time_t valid_after)
               (void *) &valid_after, NULL);
 }
 
-/* Return the phase we are currently in according to our state. */
+/** Return the phase we are currently in according to our state. */
 sr_phase_t
 sr_state_get_phase(void)
 {
-  void *ptr;
+  void *ptr=NULL;
   state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PHASE, NULL, &ptr);
+  tor_assert(ptr);
   return *(sr_phase_t *) ptr;
 }
 
-/* Return the previous SRV value from our state. Value CAN be NULL. */
+/** Return the previous SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_previous_srv(void)
 {
@@ -1039,7 +1059,7 @@ sr_state_get_previous_srv(void)
   return srv;
 }
 
-/* Set the current SRV value from our state. Value CAN be NULL. The srv
+/** Set the current SRV value from our state. Value CAN be NULL. The srv
  * object ownership is transferred to the state object. */
 void
 sr_state_set_previous_srv(const sr_srv_t *srv)
@@ -1048,7 +1068,9 @@ sr_state_set_previous_srv(const sr_srv_t *srv)
               NULL);
 }
 
-/* Return the current SRV value from our state. Value CAN be NULL. */
+/** Return the current SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_current_srv(void)
 {
@@ -1058,7 +1080,7 @@ sr_state_get_current_srv(void)
   return srv;
 }
 
-/* Set the current SRV value from our state. Value CAN be NULL. The srv
+/** Set the current SRV value from our state. Value CAN be NULL. The srv
  * object ownership is transferred to the state object. */
 void
 sr_state_set_current_srv(const sr_srv_t *srv)
@@ -1067,7 +1089,7 @@ sr_state_set_current_srv(const sr_srv_t *srv)
               NULL);
 }
 
-/* Clean all the SRVs in our state. */
+/** Clean all the SRVs in our state. */
 void
 sr_state_clean_srvs(void)
 {
@@ -1076,7 +1098,7 @@ sr_state_clean_srvs(void)
   state_del_current_srv();
 }
 
-/* Return a pointer to the commits map from our state. CANNOT be NULL. */
+/** Return a pointer to the commits map from our state. CANNOT be NULL. */
 digestmap_t *
 sr_state_get_commits(void)
 {
@@ -1087,7 +1109,7 @@ sr_state_get_commits(void)
   return commits;
 }
 
-/* Update the current SR state as needed for the upcoming voting round at
+/** Update the current SR state as needed for the upcoming voting round at
  * <b>valid_after</b>. */
 void
 sr_state_update(time_t valid_after)
@@ -1151,7 +1173,7 @@ sr_state_update(time_t valid_after)
   }
 }
 
-/* Return commit object from the given authority digest <b>rsa_identity</b>.
+/** Return commit object from the given authority digest <b>rsa_identity</b>.
  * Return NULL if not found. */
 sr_commit_t *
 sr_state_get_commit(const char *rsa_identity)
@@ -1165,7 +1187,7 @@ sr_state_get_commit(const char *rsa_identity)
   return commit;
 }
 
-/* Add <b>commit</b> to the permanent state. The commit object ownership is
+/** Add <b>commit</b> to the permanent state. The commit object ownership is
  * transferred to the state so the caller MUST not free it. */
 void
 sr_state_add_commit(sr_commit_t *commit)
@@ -1180,14 +1202,14 @@ sr_state_add_commit(sr_commit_t *commit)
             sr_commit_get_rsa_fpr(commit));
 }
 
-/* Remove all commits from our state. */
+/** Remove all commits from our state. */
 void
 sr_state_delete_commits(void)
 {
   state_query(SR_STATE_ACTION_DEL_ALL, SR_STATE_OBJ_COMMIT, NULL, NULL);
 }
 
-/* Copy the reveal information from <b>commit</b> into <b>saved_commit</b>.
+/** Copy the reveal information from <b>commit</b> into <b>saved_commit</b>.
  * This <b>saved_commit</b> MUST come from our current SR state. Once modified,
  * the disk state is updated. */
 void
@@ -1208,7 +1230,7 @@ sr_state_copy_reveal_info(sr_commit_t *saved_commit, const sr_commit_t *commit)
             sr_commit_get_rsa_fpr(saved_commit));
 }
 
-/* Set the fresh SRV flag from our state. This doesn't need to trigger a
+/** Set the fresh SRV flag from our state. This doesn't need to trigger a
  * disk state synchronization so we directly change the state. */
 void
 sr_state_set_fresh_srv(void)
@@ -1216,7 +1238,7 @@ sr_state_set_fresh_srv(void)
   sr_state->is_srv_fresh = 1;
 }
 
-/* Unset the fresh SRV flag from our state. This doesn't need to trigger a
+/** Unset the fresh SRV flag from our state. This doesn't need to trigger a
  * disk state synchronization so we directly change the state. */
 void
 sr_state_unset_fresh_srv(void)
@@ -1224,14 +1246,14 @@ sr_state_unset_fresh_srv(void)
   sr_state->is_srv_fresh = 0;
 }
 
-/* Return the value of the fresh SRV flag. */
+/** Return the value of the fresh SRV flag. */
 unsigned int
 sr_state_srv_is_fresh(void)
 {
   return sr_state->is_srv_fresh;
 }
 
-/* Cleanup and free our disk and memory state. */
+/** Cleanup and free our disk and memory state. */
 void
 sr_state_free_all(void)
 {
@@ -1240,9 +1262,10 @@ sr_state_free_all(void)
   /* Nullify our global state. */
   sr_state = NULL;
   sr_disk_state = NULL;
+  config_mgr_free(shared_random_state_mgr);
 }
 
-/* Save our current state in memory to disk. */
+/** Save our current state in memory to disk. */
 void
 sr_state_save(void)
 {
@@ -1250,7 +1273,7 @@ sr_state_save(void)
   state_query(SR_STATE_ACTION_SAVE, 0, NULL, NULL);
 }
 
-/* Return 1 iff the state has been initialized that is it exists in memory.
+/** Return 1 iff the state has been initialized that is it exists in memory.
  * Return 0 otherwise. */
 int
 sr_state_is_initialized(void)
@@ -1258,7 +1281,7 @@ sr_state_is_initialized(void)
   return sr_state == NULL ? 0 : 1;
 }
 
-/* Initialize the disk and memory state.
+/** Initialize the disk and memory state.
  *
  * If save_to_disk is set to 1, the state is immediately saved to disk after
  * creation else it's not thus only kept in memory.
@@ -1310,7 +1333,7 @@ sr_state_init(int save_to_disk, int read_from_disk)
   /* We have a state in memory, let's make sure it's updated for the current
    * and next voting round. */
   {
-    time_t valid_after = voting_schedule_get_next_valid_after_time();
+    time_t valid_after = dirauth_sched_get_next_valid_after_time();
     sr_state_update(valid_after);
   }
   return 0;
@@ -1321,7 +1344,7 @@ sr_state_init(int save_to_disk, int read_from_disk)
 
 #ifdef TOR_UNIT_TESTS
 
-/* Set the current phase of the protocol. Used only by unit tests. */
+/** Set the current phase of the protocol. Used only by unit tests. */
 void
 set_sr_phase(sr_phase_t phase)
 {
@@ -1330,7 +1353,7 @@ set_sr_phase(sr_phase_t phase)
   sr_state->phase = phase;
 }
 
-/* Get the SR state. Used only by unit tests */
+/** Get the SR state. Used only by unit tests */
 sr_state_t *
 get_sr_state(void)
 {
diff --git a/src/feature/dirauth/shared_random_state.h b/src/feature/dirauth/shared_random_state.h
index 08f999f9d4..3a34bcc3e7 100644
--- a/src/feature/dirauth/shared_random_state.h
+++ b/src/feature/dirauth/shared_random_state.h
@@ -1,12 +1,17 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file shared_random_state.h
+ * @brief Header for shared_random_state.c
+ **/
+
 #ifndef TOR_SHARED_RANDOM_STATE_H
 #define TOR_SHARED_RANDOM_STATE_H
 
 #include "feature/dirauth/shared_random.h"
 
-/* Action that can be performed on the state for any objects. */
+/** Action that can be performed on the state for any objects. */
 typedef enum {
   SR_STATE_ACTION_GET     = 1,
   SR_STATE_ACTION_PUT     = 2,
@@ -15,52 +20,53 @@ typedef enum {
   SR_STATE_ACTION_SAVE    = 5,
 } sr_state_action_t;
 
-/* Object in the state that can be queried through the state API. */
+/** Object in the state that can be queried through the state API. */
 typedef enum {
-  /* Will return a single commit using an authority identity key. */
+  /** Will return a single commit using an authority identity key. */
   SR_STATE_OBJ_COMMIT,
-  /* Returns the entire list of commits from the state. */
+  /** Returns the entire list of commits from the state. */
   SR_STATE_OBJ_COMMITS,
-  /* Return the current SRV object pointer. */
+  /** Return the current SRV object pointer. */
   SR_STATE_OBJ_CURSRV,
-  /* Return the previous SRV object pointer. */
+  /** Return the previous SRV object pointer. */
   SR_STATE_OBJ_PREVSRV,
-  /* Return the phase. */
+  /** Return the phase. */
   SR_STATE_OBJ_PHASE,
-  /* Get or Put the valid after time. */
+  /** Get or Put the valid after time. */
   SR_STATE_OBJ_VALID_AFTER,
 } sr_state_object_t;
 
-/* State of the protocol. It's also saved on disk in fname. This data
+/** State of the protocol. It's also saved on disk in fname. This data
  * structure MUST be synchronized at all time with the one on disk. */
 typedef struct sr_state_t {
-  /* Filename of the state file on disk. */
+  /** Filename of the state file on disk. */
   char *fname;
-  /* Version of the protocol. */
+  /** Version of the protocol. */
   uint32_t version;
-  /* The valid-after of the voting period we have prepared the state for. */
+  /** The valid-after of the voting period we have prepared the state for. */
   time_t valid_after;
-  /* Until when is this state valid? */
+  /** Until when is this state valid? */
   time_t valid_until;
-  /* Protocol phase. */
+  /** Protocol phase. */
   sr_phase_t phase;
 
-  /* Number of runs completed. */
+  /** Number of runs completed. */
   uint64_t n_protocol_runs;
-  /* The number of commitment rounds we've performed in this protocol run. */
+  /** The number of commitment rounds we've performed in this protocol run. */
   unsigned int n_commit_rounds;
-  /* The number of reveal rounds we've performed in this protocol run. */
+  /** The number of reveal rounds we've performed in this protocol run. */
   unsigned int n_reveal_rounds;
 
-  /* A map of all the received commitments for this protocol run. This is
+  /** A map of all the received commitments for this protocol run. This is
    * indexed by authority RSA identity digest. */
   digestmap_t *commits;
 
-  /* Current and previous shared random value. */
+  /** Current shared random value. */
   sr_srv_t *previous_srv;
+  /** Previous shared random value. */
   sr_srv_t *current_srv;
 
-  /* Indicate if the state contains an SRV that was _just_ generated. This is
+  /** Indicate if the state contains an SRV that was _just_ generated. This is
    * used during voting so that we know whether to use the super majority rule
    * or not when deciding on keeping it for the consensus. It is _always_ set
    * to 0 post consensus.
@@ -73,22 +79,22 @@ typedef struct sr_state_t {
   unsigned int is_srv_fresh:1;
 } sr_state_t;
 
-/* Persistent state of the protocol, as saved to disk. */
+/** Persistent state of the protocol, as saved to disk. */
 typedef struct sr_disk_state_t {
   uint32_t magic_;
-  /* Version of the protocol. */
+  /** Version of the protocol. */
   int Version;
-  /* Version of our running tor. */
+  /** Version of our running tor. */
   char *TorVersion;
-  /* Creation time of this state */
+  /** Creation time of this state */
   time_t ValidAfter;
-  /* State valid until? */
+  /** State valid until? */
   time_t ValidUntil;
-  /* All commits seen that are valid. */
+  /** All commits seen that are valid. */
   struct config_line_t *Commit;
-  /* Previous and current shared random value. */
+  /** Previous and current shared random value. */
   struct config_line_t *SharedRandValues;
-  /* Extra Lines for configuration we might not know. */
+  /** Extra Lines for configuration we might not know. */
   struct config_line_t *ExtraLines;
 } sr_disk_state_t;
 
diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h
index 92acdf1157..7f8ebf7fd7 100644
--- a/src/feature/dirauth/vote_microdesc_hash_st.h
+++ b/src/feature/dirauth/vote_microdesc_hash_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file vote_microdesc_hash_st.h
+ * @brief Microdescriptor-hash voting strcture.
+ **/
+
 #ifndef VOTE_MICRODESC_HASH_ST_H
 #define VOTE_MICRODESC_HASH_ST_H
 
@@ -18,5 +23,4 @@ struct vote_microdesc_hash_t {
   char *microdesc_hash_line;
 };
 
-#endif
-
+#endif /* !defined(VOTE_MICRODESC_HASH_ST_H) */
diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c
index 54c70b989a..477eb6f0b7 100644
--- a/src/feature/dirauth/voteflags.c
+++ b/src/feature/dirauth/voteflags.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,7 @@
 #include "core/or/policies.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/reachability.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/networkstatus.h"
@@ -27,8 +28,10 @@
 #include "feature/relay/router.h"
 #include "feature/stats/rephist.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
 #include "lib/container/order.h"
@@ -95,7 +98,7 @@ real_uptime(const routerinfo_t *router, time_t now)
  */
 static int
 dirserv_thinks_router_is_unreliable(time_t now,
-                                    routerinfo_t *router,
+                                    const routerinfo_t *router,
                                     int need_uptime, int need_capacity)
 {
   if (need_uptime) {
@@ -144,7 +147,7 @@ router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
    * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
   if (!ri->bandwidthcapacity) {
     if (get_options()->TestingTorNetwork) {
-      if (get_options()->TestingMinExitFlagThreshold > 0) {
+      if (dirauth_get_options()->TestingMinExitFlagThreshold > 0) {
         /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is,
          * then require bandwidthcapacity */
         return 0;
@@ -174,14 +177,14 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
   long uptime;
 
   /* If we haven't been running for at least
-   * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't
+   * MinUptimeHidServDirectoryV2 seconds, we can't
    * have accurate data telling us a relay has been up for at least
    * that long. We also want to allow a bit of slack: Reachability
    * tests aren't instant. If we haven't been running long enough,
    * trust the relay. */
 
   if (get_uptime() >
-      get_options()->MinUptimeHidServDirectoryV2 * 1.1)
+      dirauth_get_options()->MinUptimeHidServDirectoryV2 * 1.1)
     uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
                  real_uptime(router, now));
   else
@@ -190,7 +193,7 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
   return (router->wants_to_be_hs_dir &&
           router->supports_tunnelled_dir_requests &&
           node->is_stable && node->is_fast &&
-          uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
+          uptime >= dirauth_get_options()->MinUptimeHidServDirectoryV2 &&
           router_is_active(router, node, now));
 }
 
@@ -213,9 +216,10 @@ router_counts_toward_thresholds(const node_t *node, time_t now,
     dirserv_has_measured_bw(node->identity);
   uint64_t min_bw_kb = ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB;
   const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
 
   if (options->TestingTorNetwork) {
-    min_bw_kb = (int64_t)options->TestingMinExitFlagThreshold / 1000;
+    min_bw_kb = (int64_t)dirauth_options->TestingMinExitFlagThreshold / 1000;
   }
 
   return node->ri && router_is_active(node->ri, node, now) &&
@@ -238,14 +242,15 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
   uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
   long *tks;
   double *mtbfs, *wfus;
-  smartlist_t *nodelist;
+  const smartlist_t *nodelist;
   time_t now = time(NULL);
   const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
 
   /* Require mbw? */
   int require_mbw =
     (dirserv_get_last_n_measured_bws() >
-     options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
+     dirauth_options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
 
   /* initialize these all here, in case there are no routers */
   stable_uptime = 0;
@@ -337,7 +342,7 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
       ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
       INT32_MAX);
     if (options->TestingTorNetwork) {
-      min_fast = (int32_t)options->TestingMinFastFlagThreshold;
+      min_fast = (int32_t)dirauth_options->TestingMinFastFlagThreshold;
     }
     max_fast = networkstatus_get_param(NULL, "FastFlagMaxThreshold",
                                        INT32_MAX, min_fast, INT32_MAX);
@@ -351,9 +356,11 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
   }
   /* Protect sufficiently fast nodes from being pushed out of the set
    * of Fast nodes. */
-  if (options->AuthDirFastGuarantee &&
-      fast_bandwidth_kb > options->AuthDirFastGuarantee/1000)
-    fast_bandwidth_kb = (uint32_t)options->AuthDirFastGuarantee/1000;
+  {
+    const uint64_t fast_opt = dirauth_get_options()->AuthDirFastGuarantee;
+    if (fast_opt && fast_bandwidth_kb > fast_opt / 1000)
+      fast_bandwidth_kb = (uint32_t)(fast_opt / 1000);
+  }
 
   /* Now that we have a time-known that 7/8 routers are known longer than,
    * fill wfus with the wfu of every such "familiar" router. */
@@ -427,7 +434,7 @@ dirserv_get_flag_thresholds_line(void)
 {
   char *result=NULL;
   const int measured_threshold =
-    get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+    dirauth_get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
   const int enough_measured_bw =
     dirserv_get_last_n_measured_bws() > measured_threshold;
 
@@ -454,8 +461,9 @@ dirserv_get_flag_thresholds_line(void)
 int
 running_long_enough_to_decide_unreachable(void)
 {
-  return time_of_process_start
-    + get_options()->TestingAuthDirTimeToLearnReachability < approx_time();
+  const dirauth_options_t *opts = dirauth_get_options();
+  return time_of_process_start +
+    opts->TestingAuthDirTimeToLearnReachability < approx_time();
 }
 
 /** Each server needs to have passed a reachability test no more
@@ -480,6 +488,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
    */
   int answer;
   const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
   node_t *node = node_get_mutable_by_id(router->cache_info.identity_digest);
   tor_assert(node);
 
@@ -506,7 +515,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
        IPv6 OR port since that'd kill all dual stack relays until a
        majority of the dir auths have IPv6 connectivity. */
     answer = (now < node->last_reachable + REACHABLE_TIMEOUT &&
-              (options->AuthDirHasIPv6Connectivity != 1 ||
+              (dirauth_options->AuthDirHasIPv6Connectivity != 1 ||
                tor_addr_is_null(&router->ipv6_addr) ||
                now < node->last_reachable6 + REACHABLE_TIMEOUT));
   }
@@ -531,42 +540,49 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
   node->is_running = answer;
 }
 
-/** Extract status information from <b>ri</b> and from other authority
- * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
- * set.
- *
- * We assume that ri-\>is_running has already been set, e.g. by
- *   dirserv_set_router_is_running(ri, now);
+/* Check <b>node</b> and <b>ri</b> on whether or not we should publish a
+ * relay's IPv6 addresses. */
+static int
+should_publish_node_ipv6(const node_t *node, const routerinfo_t *ri,
+                         time_t now)
+{
+  const dirauth_options_t *options = dirauth_get_options();
+
+  return options->AuthDirHasIPv6Connectivity == 1 &&
+    !tor_addr_is_null(&ri->ipv6_addr) &&
+    ((node->last_reachable6 >= now - REACHABLE_TIMEOUT) ||
+     router_is_me(ri));
+}
+
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>, as per
+ * <b>set_routerstatus_from_routerinfo</b>.  Additionally, sets information
+ * in from the authority subsystem.
  */
 void
-set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                 node_t *node,
-                                 routerinfo_t *ri,
-                                 time_t now,
-                                 int listbadexits)
+dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                         node_t *node,
+                                         const routerinfo_t *ri,
+                                         time_t now,
+                                         int listbadexits)
 {
   const or_options_t *options = get_options();
   uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
 
-  memset(rs, 0, sizeof(routerstatus_t));
-
-  rs->is_authority =
-    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
-
-  /* Already set by compute_performance_thresholds. */
-  rs->is_exit = node->is_exit;
-  rs->is_stable = node->is_stable =
-    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
-  rs->is_fast = node->is_fast =
-    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
-  rs->is_flagged_running = node->is_running; /* computed above */
+  /* Set these flags so that set_routerstatus_from_routerinfo can copy them.
+   */
+  node->is_stable = !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
+  node->is_fast = !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
+  node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now);
 
-  rs->is_valid = node->is_valid;
+  set_routerstatus_from_routerinfo(rs, node, ri);
 
+  /* Override rs->is_possible_guard. */
+  const uint64_t bw_opt = dirauth_get_options()->AuthDirGuardBWGuarantee;
   if (node->is_fast && node->is_stable &&
       ri->supports_tunnelled_dir_requests &&
-      ((options->AuthDirGuardBWGuarantee &&
-        routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
+      ((bw_opt && routerbw_kb >= bw_opt / 1000) ||
        routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
                           guard_bandwidth_excluding_exits_kb))) {
     long tk = rep_hist_get_weighted_time_known(
@@ -578,29 +594,16 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
     rs->is_possible_guard = 0;
   }
 
+  /* Override rs->is_bad_exit */
   rs->is_bad_exit = listbadexits && node->is_bad_exit;
-  rs->is_hs_dir = node->is_hs_dir =
-    dirserv_thinks_router_is_hs_dir(ri, node, now);
-
-  rs->is_named = rs->is_unnamed = 0;
-
-  rs->published_on = ri->cache_info.published_on;
-  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
-  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
-         DIGEST_LEN);
-  rs->addr = ri->addr;
-  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
-  rs->or_port = ri->or_port;
-  rs->dir_port = ri->dir_port;
-  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
-  if (options->AuthDirHasIPv6Connectivity == 1 &&
-      !tor_addr_is_null(&ri->ipv6_addr) &&
-      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
-    /* We're configured as having IPv6 connectivity. There's an IPv6
-       OR port and it's reachable so copy it to the routerstatus.  */
-    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
-    rs->ipv6_orport = ri->ipv6_orport;
-  } else {
+
+  /* Set rs->is_staledesc. */
+  rs->is_staledesc =
+    (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now;
+
+  if (! should_publish_node_ipv6(node, ri, now)) {
+    /* We're not configured as having IPv6 connectivity or the node isn't:
+     * zero its IPv6 information. */
     tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
     rs->ipv6_orport = 0;
   }
@@ -617,9 +620,9 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
 STATIC void
 dirserv_set_routerstatus_testing(routerstatus_t *rs)
 {
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
 
-  tor_assert(options->TestingTorNetwork);
+  tor_assert(get_options()->TestingTorNetwork);
 
   if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit,
                                       rs, 0)) {
@@ -642,3 +645,20 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs)
     rs->is_hs_dir = 0;
   }
 }
+
+/** Use dirserv_set_router_is_running() to set bridges as running if they're
+ * reachable.
+ *
+ * This function is called from set_bridge_running_callback() when running as
+ * a bridge authority.
+ */
+void
+dirserv_set_bridges_running(time_t now)
+{
+  routerlist_t *rl = router_get_routerlist();
+
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+    if (ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      dirserv_set_router_is_running(ri, now);
+  } SMARTLIST_FOREACH_END(ri);
+}
diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h
index aa7b6ed082..91f3854573 100644
--- a/src/feature/dirauth/voteflags.h
+++ b/src/feature/dirauth/voteflags.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,20 +12,28 @@
 #ifndef TOR_VOTEFLAGS_H
 #define TOR_VOTEFLAGS_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
 char *dirserv_get_flag_thresholds_line(void);
 void dirserv_compute_bridge_flag_thresholds(void);
 int running_long_enough_to_decide_unreachable(void);
 
-void set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                      node_t *node,
-                                      routerinfo_t *ri, time_t now,
-                                      int listbadexits);
+void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                              node_t *node,
+                                              const routerinfo_t *ri,
+                                              time_t now,
+                                              int listbadexits);
 
 void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+void dirserv_set_bridges_running(time_t now);
 
 #ifdef VOTEFLAGS_PRIVATE
+/** Any descriptor older than this age causes the authorities to set the
+ * StaleDesc flag. */
+#define DESC_IS_STALE_INTERVAL (18*60*60)
 STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
-#endif
+#endif /* defined(VOTEFLAGS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_VOTEFLAGS_H) */
diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dirauth/voting_schedule.c
index 0a7476eda7..efc4a0b316 100644
--- a/src/feature/dircommon/voting_schedule.c
+++ b/src/feature/dirauth/voting_schedule.c
@@ -1,15 +1,13 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file voting_schedule.c
- * \brief This file contains functions that are from the directory authority
- *        subsystem related to voting specifically but used by many part of
- *        tor. The full feature is built as part of the dirauth module.
+ * \brief Compute information about our voting schedule as a directory
+ *    authority.
  **/
 
-#define VOTING_SCHEDULE_PRIVATE
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "core/or/or.h"
 #include "app/config/config.h"
@@ -21,55 +19,11 @@
  * Vote scheduling
  * ===== */
 
-/** Return the start of the next interval of size <b>interval</b> (in
- * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
- * starts a fresh interval, and if the last interval of a day would be
- * truncated to less than half its size, it is rolled into the
- * previous interval. */
-time_t
-voting_schedule_get_start_of_next_interval(time_t now, int interval,
-                                           int offset)
-{
-  struct tm tm;
-  time_t midnight_today=0;
-  time_t midnight_tomorrow;
-  time_t next;
-
-  tor_gmtime_r(&now, &tm);
-  tm.tm_hour = 0;
-  tm.tm_min = 0;
-  tm.tm_sec = 0;
-
-  if (tor_timegm(&tm, &midnight_today) < 0) {
-    // LCOV_EXCL_START
-    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
-    // LCOV_EXCL_STOP
-  }
-  midnight_tomorrow = midnight_today + (24*60*60);
-
-  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
-
-  /* Intervals never cross midnight. */
-  if (next > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  /* If the interval would only last half as long as it's supposed to, then
-   * skip over to the next day. */
-  if (next + interval/2 > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  next += offset;
-  if (next - interval > now)
-    next -= interval;
-
-  return next;
-}
-
 /* Populate and return a new voting_schedule_t that can be used to schedule
  * voting. The object is allocated on the heap and it's the responsibility of
  * the caller to free it. Can't fail. */
 static voting_schedule_t *
-get_voting_schedule(const or_options_t *options, time_t now, int severity)
+create_voting_schedule(const or_options_t *options, time_t now, int severity)
 {
   int interval, vote_delay, dist_delay;
   time_t start;
@@ -96,14 +50,15 @@ get_voting_schedule(const or_options_t *options, time_t now, int severity)
   }
 
   tor_assert(interval > 0);
+  new_voting_schedule->interval = interval;
 
   if (vote_delay + dist_delay > interval/2)
     vote_delay = dist_delay = interval / 4;
 
   start = new_voting_schedule->interval_starts =
-    voting_schedule_get_start_of_next_interval(now,interval,
+    voting_sched_get_start_of_interval_after(now,interval,
                                       options->TestingV3AuthVotingStartOffset);
-  end = voting_schedule_get_start_of_next_interval(start+1, interval,
+  end = voting_sched_get_start_of_interval_after(start+1, interval,
                                       options->TestingV3AuthVotingStartOffset);
 
   tor_assert(end > start);
@@ -140,9 +95,13 @@ voting_schedule_free_(voting_schedule_t *voting_schedule_to_free)
 
 voting_schedule_t voting_schedule;
 
-/* Using the time <b>now</b>, return the next voting valid-after time. */
-time_t
-voting_schedule_get_next_valid_after_time(void)
+/**
+ * Return the current voting schedule, recreating it if necessary.
+ *
+ * Dirauth only.
+ **/
+static const voting_schedule_t *
+dirauth_get_voting_schedule(void)
 {
   time_t now = approx_time();
   bool need_to_recalculate_voting_schedule = false;
@@ -150,7 +109,7 @@ voting_schedule_get_next_valid_after_time(void)
   /* This is a safe guard in order to make sure that the voting schedule
    * static object is at least initialized. Using this function with a zeroed
    * voting schedule can lead to bugs. */
-  if (tor_mem_is_zero((const char *) &voting_schedule,
+  if (fast_mem_is_zero((const char *) &voting_schedule,
                       sizeof(voting_schedule))) {
     need_to_recalculate_voting_schedule = true;
     goto done; /* no need for next check if we have to recalculate anyway */
@@ -168,27 +127,62 @@ voting_schedule_get_next_valid_after_time(void)
 
  done:
   if (need_to_recalculate_voting_schedule) {
-    voting_schedule_recalculate_timing(get_options(), approx_time());
+    dirauth_sched_recalculate_timing(get_options(), approx_time());
     voting_schedule.created_on_demand = 1;
   }
 
-  return voting_schedule.interval_starts;
+  return &voting_schedule;
+}
+
+/** Return the next voting valid-after time.
+ *
+ * Dirauth only. */
+time_t
+dirauth_sched_get_next_valid_after_time(void)
+{
+  return dirauth_get_voting_schedule()->interval_starts;
+}
+
+/**
+ * Return our best idea of what the valid-after time for the _current_
+ * consensus, whether we have one or not.
+ *
+ * Dirauth only.
+ **/
+time_t
+dirauth_sched_get_cur_valid_after_time(void)
+{
+  const voting_schedule_t *sched = dirauth_get_voting_schedule();
+  time_t next_start = sched->interval_starts;
+  int interval = sched->interval;
+  int offset = get_options()->TestingV3AuthVotingStartOffset;
+  return voting_sched_get_start_of_interval_after(next_start - interval - 1,
+                                                  interval,
+                                                  offset);
+}
+
+/** Return the voting interval that we are configured to use.
+ *
+ * Dirauth only. */
+int
+dirauth_sched_get_configured_interval(void)
+{
+  return get_options()->V3AuthVotingInterval;
 }
 
 /** Set voting_schedule to hold the timing for the next vote we should be
  * doing. All type of tor do that because HS subsystem needs the timing as
  * well to function properly. */
 void
-voting_schedule_recalculate_timing(const or_options_t *options, time_t now)
+dirauth_sched_recalculate_timing(const or_options_t *options, time_t now)
 {
   voting_schedule_t *new_voting_schedule;
 
   /* get the new voting schedule */
-  new_voting_schedule = get_voting_schedule(options, now, LOG_INFO);
+  new_voting_schedule = create_voting_schedule(options, now, LOG_INFO);
   tor_assert(new_voting_schedule);
 
   /* Fill in the global static struct now */
   memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule));
   voting_schedule_free(new_voting_schedule);
 }
-
diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dirauth/voting_schedule.h
index bafd81184e..9e2ac29c75 100644
--- a/src/feature/dircommon/voting_schedule.h
+++ b/src/feature/dirauth/voting_schedule.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,8 @@
 
 #include "core/or/or.h"
 
+#ifdef HAVE_MODULE_DIRAUTH
+
 /** Scheduling information for a voting interval. */
 typedef struct {
   /** When do we generate and distribute our vote for this interval? */
@@ -26,15 +28,18 @@ typedef struct {
   /** When do we publish the consensus? */
   time_t interval_starts;
 
-  /* True iff we have generated and distributed our vote. */
+  /** Our computed dirauth interval */
+  int interval;
+
+  /** True iff we have generated and distributed our vote. */
   int have_voted;
-  /* True iff we've requested missing votes. */
+  /** True iff we've requested missing votes. */
   int have_fetched_missing_votes;
-  /* True iff we have built a consensus and sent the signatures around. */
+  /** True iff we have built a consensus and sent the signatures around. */
   int have_built_consensus;
-  /* True iff we've fetched missing signatures. */
+  /** True iff we've fetched missing signatures. */
   int have_fetched_missing_signatures;
-  /* True iff we have published our consensus. */
+  /** True iff we have published our consensus. */
   int have_published_consensus;
 
   /* True iff this voting schedule was set on demand meaning not through the
@@ -53,13 +58,36 @@ typedef struct {
 
 extern voting_schedule_t voting_schedule;
 
-void voting_schedule_recalculate_timing(const or_options_t *options,
+void dirauth_sched_recalculate_timing(const or_options_t *options,
                                         time_t now);
 
-time_t voting_schedule_get_start_of_next_interval(time_t now,
-                                                  int interval,
-                                                  int offset);
-time_t voting_schedule_get_next_valid_after_time(void);
+time_t dirauth_sched_get_next_valid_after_time(void);
+time_t dirauth_sched_get_cur_valid_after_time(void);
+int dirauth_sched_get_configured_interval(void);
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+#define dirauth_sched_recalculate_timing(opt,now) \
+  ((void)(opt), (void)(now))
 
-#endif /* TOR_VOTING_SCHEDULE_H */
+static inline time_t
+dirauth_sched_get_next_valid_after_time(void)
+{
+  tor_assert_unreached();
+  return 0;
+}
+static inline time_t
+dirauth_sched_get_cur_valid_after_time(void)
+{
+  tor_assert_unreached();
+  return 0;
+}
+static inline int
+dirauth_sched_get_configured_interval(void)
+{
+  tor_assert_unreached();
+  return 1;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
+#endif /* !defined(TOR_VOTING_SCHEDULE_H) */
diff --git a/src/feature/dircache/.may_include b/src/feature/dircache/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dircache/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h
index 71dca8c3a2..ede1d028da 100644
--- a/src/feature/dircache/cached_dir_st.h
+++ b/src/feature/dircache/cached_dir_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cached_dir_st.h
+ * @brief Cached large directory object structure.
+ **/
+
 #ifndef CACHED_DIR_ST_H
 #define CACHED_DIR_ST_H
 
@@ -21,5 +26,4 @@ struct cached_dir_t {
   int refcnt; /**< Reference count for this cached_dir_t. */
 };
 
-#endif
-
+#endif /* !defined(CACHED_DIR_ST_H) */
diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c
index cf4fe8701d..2a831aa447 100644
--- a/src/feature/dircache/conscache.c
+++ b/src/feature/dircache/conscache.c
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file conscache.c
+ * @brief Consensus and diff on-disk cache.
+ **/
+
 #include "core/or/or.h"
 
 #include "app/config/config.h"
@@ -92,7 +97,7 @@ consensus_cache_open(const char *subdir, int max_entries)
    */
 #define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
   storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
-#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
+#else /* !defined(MUST_UNMAP_TO_UNLINK) */
   /* Otherwise, we can just tell the storagedir to use the same limits
    * as this cache. */
   storagedir_max_entries = max_entries;
@@ -127,13 +132,22 @@ consensus_cache_may_overallocate(consensus_cache_t *cache)
 #endif
 }
 
+// HACK: GCC on Appveyor hates that we may assert before returning. Work around
+// the error.
+#ifdef _WIN32
+#ifndef COCCI
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsuggest-attribute=noreturn"
+#endif
+#endif /* defined(_WIN32) */
+
 /**
  * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
  * operations that <b>cache</b> will need.
  */
 int
 consensus_cache_register_with_sandbox(consensus_cache_t *cache,
-                                      struct sandbox_cfg_elem **cfg)
+                                      struct sandbox_cfg_elem_t **cfg)
 {
 #ifdef MUST_UNMAP_TO_UNLINK
   /* Our Linux sandbox doesn't support huge file lists like the one that would
@@ -151,6 +165,12 @@ consensus_cache_register_with_sandbox(consensus_cache_t *cache,
   return storage_dir_register_with_sandbox(cache->dir, cfg);
 }
 
+#ifdef _WIN32
+#ifndef COCCI
+#pragma GCC diagnostic pop
+#endif
+#endif
+
 /**
  * Helper: clear all entries from <b>cache</b> (but do not delete
  * any that aren't marked for removal
@@ -246,7 +266,7 @@ consensus_cache_find_first(consensus_cache_t *cache,
 }
 
 /**
- * Given a <b>cache</b>, add every entry to <b>out<b> for which
+ * Given a <b>cache</b>, add every entry to <b>out</b> for which
  * <b>key</b>=<b>value</b>.  If <b>key</b> is NULL, add every entry.
  *
  * Do not add any entry that has been marked for removal.
diff --git a/src/feature/dircache/conscache.h b/src/feature/dircache/conscache.h
index d848e57617..ace5908e40 100644
--- a/src/feature/dircache/conscache.h
+++ b/src/feature/dircache/conscache.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file conscache.h
+ * @brief Header for conscache.c
+ **/
+
 #ifndef TOR_CONSCACHE_H
 #define TOR_CONSCACHE_H
 
@@ -9,6 +14,8 @@
 typedef struct consensus_cache_entry_t consensus_cache_entry_t;
 typedef struct consensus_cache_t consensus_cache_t;
 
+struct config_line_t;
+
 HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, )
 #define consensus_cache_entry_handle_free(h)    \
   FREE_AND_NULL(consensus_cache_entry_handle_t, \
@@ -18,10 +25,10 @@ consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
 void consensus_cache_free_(consensus_cache_t *cache);
 #define consensus_cache_free(cache) \
   FREE_AND_NULL(consensus_cache_t, consensus_cache_free_, (cache))
-struct sandbox_cfg_elem;
+struct sandbox_cfg_elem_t;
 int consensus_cache_may_overallocate(consensus_cache_t *cache);
 int consensus_cache_register_with_sandbox(consensus_cache_t *cache,
-                                          struct sandbox_cfg_elem **cfg);
+                                          struct sandbox_cfg_elem_t **cfg);
 void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
 void consensus_cache_delete_pending(consensus_cache_t *cache,
                                     int force);
diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c
index 025361fa60..10590cd6d2 100644
--- a/src/feature/dircache/consdiffmgr.c
+++ b/src/feature/dircache/consdiffmgr.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file consdiffmsr.c
+ * \file consdiffmgr.c
  *
  * \brief consensus diff manager functions
  *
@@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = {
 
 static int consdiffmgr_ensure_space_for_files(int n);
 static int consensus_queue_compression_work(const char *consensus,
+                                            size_t consensus_len,
                                             const networkstatus_t *as_parsed);
 static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
                                           consensus_cache_entry_t *diff_to);
@@ -217,9 +218,9 @@ cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
     diff1->compress_method == diff2->compress_method;
 }
 
-HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
+HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq);
 HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
-             0.6, tor_reallocarray, tor_free_)
+             0.6, tor_reallocarray, tor_free_);
 
 #define cdm_diff_free(diff) \
   FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
@@ -509,8 +510,25 @@ get_max_age_to_cache(void)
                                         MAX_MAX_AGE_TO_CACHE);
 }
 
+#ifdef TOR_UNIT_TESTS
+/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For
+ * testing. */
+int
+consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                  const networkstatus_t *as_parsed)
+{
+  size_t len = strlen(consensus);
+  /* make a non-nul-terminated copy so that we can have a better chance
+   * of catching errors. */
+  char *ctmp = tor_memdup(consensus, len);
+  int r = consdiffmgr_add_consensus(ctmp, len, as_parsed);
+  tor_free(ctmp);
+  return r;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /**
- * Given a string containing a networkstatus consensus, and the results of
+ * Given a buffer containing a networkstatus consensus, and the results of
  * having parsed that consensus, add that consensus to the cache if it is not
  * already present and not too old.  Create new consensus diffs from or to
  * that consensus as appropriate.
@@ -519,6 +537,7 @@ get_max_age_to_cache(void)
  */
 int
 consdiffmgr_add_consensus(const char *consensus,
+                          size_t consensus_len,
                           const networkstatus_t *as_parsed)
 {
   if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
@@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus,
   }
 
   /* We don't have it. Add it to the cache. */
-  return consensus_queue_compression_work(consensus, as_parsed);
+  return consensus_queue_compression_work(consensus, consensus_len, as_parsed);
 }
 
 /**
@@ -825,7 +844,7 @@ consdiffmgr_configure(const consdiff_cfg_t *cfg)
  * operations that the consensus diff manager will need.
  */
 int
-consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg)
 {
   return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
 }
@@ -1274,7 +1293,7 @@ typedef struct compressed_result_t {
 
 /**
  * Compress the bytestring <b>input</b> of length <b>len</b> using the
- * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
+ * <b>n_methods</b> compression methods listed in the array <b>methods</b>.
  *
  * For each successful compression, set the fields in the <b>results_out</b>
  * array in the position corresponding to the compression method. Use
@@ -1387,19 +1406,21 @@ typedef struct consensus_diff_worker_job_t {
 } consensus_diff_worker_job_t;
 
 /** Given a consensus_cache_entry_t, check whether it has a label claiming
- * that it was compressed.  If so, uncompress its contents into <b>out</b> and
- * set <b>outlen</b> to hold their size.  If not, just copy the body into
- * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
- * -1 on failure.
- *
- * In all cases, the output is nul-terminated. */
+ * that it was compressed.  If so, uncompress its contents into *<b>out</b> and
+ * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer
+ * that the caller will need to free.  If not, just set *<b>out</b> and
+ * <b>outlen</b> to its extent in memory.  Return 0 on success, -1 on failure.
+ **/
 STATIC int
-uncompress_or_copy(char **out, size_t *outlen,
-                   consensus_cache_entry_t *ent)
+uncompress_or_set_ptr(const char **out, size_t *outlen,
+                      char **owned_out,
+                      consensus_cache_entry_t *ent)
 {
   const uint8_t *body;
   size_t bodylen;
 
+  *owned_out = NULL;
+
   if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
     return -1;
 
@@ -1410,8 +1431,17 @@ uncompress_or_copy(char **out, size_t *outlen,
   if (lv_compression)
     method = compression_method_get_by_name(lv_compression);
 
-  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+  int rv;
+  if (method == NO_METHOD) {
+    *out = (const char *)body;
+    *outlen = bodylen;
+    rv = 0;
+  } else {
+    rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen,
                         method, 1, LOG_WARN);
+    *out = *owned_out;
+  }
+  return rv;
 }
 
 /**
@@ -1478,16 +1508,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
 
   char *consensus_diff;
   {
-    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    const char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    char *owned1 = NULL, *owned2 = NULL;
     size_t diff_from_nt_len, diff_to_nt_len;
 
-    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
-                           job->diff_from) < 0) {
+    if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1,
+                              job->diff_from) < 0) {
       return WQ_RPL_REPLY;
     }
-    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
-                           job->diff_to) < 0) {
-      tor_free(diff_from_nt);
+    if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2,
+                              job->diff_to) < 0) {
+      tor_free(owned1);
       return WQ_RPL_REPLY;
     }
     tor_assert(diff_from_nt);
@@ -1496,9 +1527,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
     // XXXX ugh; this is going to calculate the SHA3 of both its
     // XXXX inputs again, even though we already have that. Maybe it's time
     // XXXX to change the API here?
-    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
-    tor_free(diff_from_nt);
-    tor_free(diff_to_nt);
+    consensus_diff = consensus_diff_generate(diff_from_nt,
+                                             diff_from_nt_len,
+                                             diff_to_nt,
+                                             diff_to_nt_len);
+    tor_free(owned1);
+    tor_free(owned2);
   }
   if (!consensus_diff) {
     /* Couldn't generate consensus; we'll leave the reply blank. */
@@ -1746,8 +1780,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_)
                           (const uint8_t *)consensus, bodylen);
   {
     const char *start, *end;
-    if (router_get_networkstatus_v3_signed_boundaries(consensus,
-                                                        &start, &end) < 0) {
+    if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen,
+                                                      &start, &end) < 0) {
       start = consensus;
       end = consensus+bodylen;
     }
@@ -1811,14 +1845,15 @@ static int background_compression = 0;
  */
 static int
 consensus_queue_compression_work(const char *consensus,
+                                 size_t consensus_len,
                                  const networkstatus_t *as_parsed)
 {
   tor_assert(consensus);
   tor_assert(as_parsed);
 
   consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
-  job->consensus = tor_strdup(consensus);
-  job->consensus_len = strlen(consensus);
+  job->consensus = tor_memdup_nulterm(consensus, consensus_len);
+  job->consensus_len = strlen(job->consensus);
   job->flavor = as_parsed->flavor;
 
   char va_str[ISO_TIME_LEN+1];
diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h
index 39e8fa31cb..27b8165e94 100644
--- a/src/feature/dircache/consdiffmgr.h
+++ b/src/feature/dircache/consdiffmgr.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file consdiffmgr.h
+ * @brief Header for consdiffmgr.c
+ **/
+
 #ifndef TOR_CONSDIFFMGR_H
 #define TOR_CONSDIFFMGR_H
 
@@ -22,6 +27,7 @@ typedef struct consdiff_cfg_t {
 struct consensus_cache_entry_t; // from conscache.h
 
 int consdiffmgr_add_consensus(const char *consensus,
+                              size_t consensus_len,
                               const networkstatus_t *as_parsed);
 
 consdiff_status_t consdiffmgr_find_consensus(
@@ -54,22 +60,30 @@ void consdiffmgr_rescan(void);
 int consdiffmgr_cleanup(void);
 void consdiffmgr_enable_background_compression(void);
 void consdiffmgr_configure(const consdiff_cfg_t *cfg);
-struct sandbox_cfg_elem;
-int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg);
+struct sandbox_cfg_elem_t;
+int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg);
 void consdiffmgr_free_all(void);
 int consdiffmgr_validate(void);
 
 #ifdef CONSDIFFMGR_PRIVATE
+struct consensus_cache_t;
+struct consensus_cache_entry_t;
 STATIC unsigned n_diff_compression_methods(void);
 STATIC unsigned n_consensus_compression_methods(void);
-STATIC consensus_cache_t *cdm_cache_get(void);
-STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
+STATIC struct consensus_cache_t *cdm_cache_get(void);
+STATIC struct consensus_cache_entry_t *cdm_cache_lookup_consensus(
                           consensus_flavor_t flavor, time_t valid_after);
 STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
-                                    consensus_cache_entry_t *ent,
+                                    struct consensus_cache_entry_t *ent,
                                     const char *label);
-STATIC int uncompress_or_copy(char **out, size_t *outlen,
-                              consensus_cache_entry_t *ent);
+STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen,
+                                 char **owned_out,
+                                 struct consensus_cache_entry_t *ent);
 #endif /* defined(CONSDIFFMGR_PRIVATE) */
 
+#ifdef TOR_UNIT_TESTS
+int consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                      const networkstatus_t *as_parsed);
+#endif
+
 #endif /* !defined(TOR_CONSDIFFMGR_H) */
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
index e8cb284165..ca127720f2 100644
--- a/src/feature/dircache/dircache.c
+++ b/src/feature/dircache/dircache.c
@@ -1,13 +1,19 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dircache.c
+ * @brief Cache directories and serve them to clients.
+ **/
+
 #define DIRCACHE_PRIVATE
 
 #include "core/or/or.h"
 
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/mainloop/connection.h"
 #include "core/or/relay.h"
 #include "feature/dirauth/dirvote.h"
@@ -23,6 +29,7 @@
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/relay/relay_config.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
 #include "feature/stats/geoip_stats.h"
@@ -49,7 +56,8 @@
 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
 #define ROBOTS_CACHE_LIFETIME (24*60*60)
 #define MICRODESC_CACHE_LIFETIME (48*60*60)
-
+/* Bandwidth files change every hour. */
+#define BANDWIDTH_CACHE_LIFETIME (30*60)
 /** Parse an HTTP request string <b>headers</b> of the form
  * \verbatim
  * "\%s [http[s]://]\%s HTTP/1..."
@@ -123,7 +131,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
                            long cache_lifetime)
 {
   char date[RFC1123_TIME_LEN+1];
-  time_t now = time(NULL);
+  time_t now = approx_time();
   buf_t *buf = buf_new_with_capacity(1024);
 
   tor_assert(conn);
@@ -166,22 +174,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
   buf_free(buf);
 }
 
-/** As write_http_response_header_impl, but sets encoding and content-typed
- * based on whether the response will be <b>compressed</b> or not. */
+/** As write_http_response_header_impl, but translates method into
+ * encoding */
 static void
 write_http_response_headers(dir_connection_t *conn, ssize_t length,
                             compress_method_t method,
                             const char *extra_headers, long cache_lifetime)
 {
-  const char *methodname = compression_method_get_name(method);
-  const char *doctype;
-  if (method == NO_METHOD)
-    doctype = "text/plain";
-  else
-    doctype = "application/octet-stream";
   write_http_response_header_impl(conn, length,
-                                  doctype,
-                                  methodname,
+                                  "text/plain",
+                                  compression_method_get_name(method),
                                   extra_headers,
                                   cache_lifetime);
 }
@@ -333,7 +335,7 @@ typedef struct get_handler_args_t {
  * an arguments structure, and must return 0 on success or -1 if we should
  * close the connection.
  **/
-typedef struct url_table_ent_s {
+typedef struct url_table_ent_t {
   const char *string;
   int is_prefix;
   int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
@@ -357,12 +359,15 @@ static int handle_get_robots(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
                                 const get_handler_args_t *args);
+static int handle_get_next_bandwidth(dir_connection_t *conn,
+                                     const get_handler_args_t *args);
 
 /** Table for handling GET requests. */
 static const url_table_ent_t url_table[] = {
   { "/tor/", 0, handle_get_frontpage },
   { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
   { "/tor/status-vote/current/", 1, handle_get_status_vote },
+  { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth },
   { "/tor/status-vote/next/", 1, handle_get_status_vote },
   { "/tor/micro/d/", 1, handle_get_microdesc },
   { "/tor/server/", 1, handle_get_descriptor },
@@ -475,7 +480,7 @@ static int
 handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
 {
   (void) args; /* unused */
-  const char *frontpage = get_dirportfrontpage();
+  const char *frontpage = relay_get_dirportfrontpage();
 
   if (frontpage) {
     size_t dlen;
@@ -495,28 +500,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
 }
 
 /** Warn that the cached consensus <b>consensus</b> of type
- * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
- * warning to avoid logging an entry on every request.
+ * <b>flavor</b> too new or too old, based on <b>is_too_new</b>,
+ * and will not be served to clients. Rate-limit the warning to avoid logging
+ * an entry on every request.
  */
 static void
-warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
-                          const char *flavor, time_t now)
+warn_consensus_is_not_reasonably_live(
+                          const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now, bool is_too_new)
 {
-#define TOO_OLD_WARNING_INTERVAL (60*60)
-  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60)
+  static ratelim_t warned[2] = { RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL),
+                                RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL) };
   char timestamp[ISO_TIME_LEN+1];
-  time_t valid_until;
-  char *dupes;
+  /* valid_after if is_too_new, valid_until if !is_too_new */
+  time_t valid_time = 0;
+  char *dupes = NULL;
 
-  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
-    return;
-
-  if ((dupes = rate_limit_log(&warned, now))) {
-    format_local_iso_time(timestamp, valid_until);
-    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
-             "serve it to clients. It was valid until %s local time and we "
-             "continued to serve it for up to 24 hours after it expired.%s",
-             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+  if (is_too_new) {
+    if (consensus_cache_entry_get_valid_after(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[1], now);
+  } else {
+    if (consensus_cache_entry_get_valid_until(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[0], now);
+  }
+
+  if (dupes) {
+    format_local_iso_time(timestamp, valid_time);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not "
+             "serve it to clients. It was valid %s %s local time and we "
+             "continued to serve it for up to 24 hours %s.%s",
+             flavor ? flavor : "",
+             flavor ? " " : "",
+             is_too_new ? "new" : "old",
+             is_too_new ? "after" : "until",
+             timestamp,
+             is_too_new ? "before it was valid" : "after it expired",
+             dupes);
     tor_free(dupes);
   }
 }
@@ -543,7 +567,7 @@ parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
 }
 
 /** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
- * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * set <b>digest_out</b> to a new smartlist containing every 256-bit
  * hex-encoded digest listed in that header and return 0.  Otherwise return
  * -1.  */
 static int
@@ -859,7 +883,6 @@ handle_get_current_consensus(dir_connection_t *conn,
 
   if (req.diff_only && !cached_consensus) {
     write_short_http_response(conn, 404, "No such diff available");
-    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -870,19 +893,30 @@ handle_get_current_consensus(dir_connection_t *conn,
                                            &compression_used);
   }
 
-  time_t fresh_until, valid_until;
-  int have_fresh_until = 0, have_valid_until = 0;
+  time_t valid_after, fresh_until, valid_until;
+  int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0;
   if (cached_consensus) {
+    have_valid_after =
+      !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after);
     have_fresh_until =
       !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
     have_valid_until =
       !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
   }
 
-  if (cached_consensus && have_valid_until &&
+  if (cached_consensus && have_valid_after &&
+      !networkstatus_valid_after_is_reasonably_live(valid_after, now)) {
+    write_short_http_response(conn, 404, "Consensus is too new");
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          1);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (
+      cached_consensus && have_valid_until &&
       !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
     write_short_http_response(conn, 404, "Consensus is too old");
-    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          0);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -924,7 +958,7 @@ handle_get_current_consensus(dir_connection_t *conn,
     goto done;
   }
 
-  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+  if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
     log_debug(LD_DIRSERV,
               "Client asked for network status lists, but we've been "
               "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1033,7 +1067,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
         }
       });
 
-    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), estimated_len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto vote_done;
     }
@@ -1045,13 +1079,11 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
       if (compress_method != NO_METHOD) {
         conn->compress_state = tor_compress_new(1, compress_method,
                            choose_compression_level(estimated_len));
-        SMARTLIST_FOREACH(items, const char *, c,
-                 connection_buf_add_compress(c, strlen(c), conn, 0));
-        connection_buf_add_compress("", 0, conn, 1);
-      } else {
-        SMARTLIST_FOREACH(items, const char *, c,
-                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
       }
+
+      SMARTLIST_FOREACH(items, const char *, c,
+                        connection_dir_buf_add(c, strlen(c), conn,
+                                               c_sl_idx == c_sl_len - 1));
     } else {
       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
           connection_buf_add(compress_method != NO_METHOD ?
@@ -1094,7 +1126,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
       write_short_http_response(conn, 404, "Not found");
       goto done;
     }
-    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
       log_info(LD_DIRSERV,
                "Client asked for server descriptors, but we've been "
                "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1192,7 +1224,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
         msg = "Not found";
       write_short_http_response(conn, 404, msg);
     } else {
-      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
         log_info(LD_DIRSERV,
                  "Client asked for server descriptors, but we've been "
                  "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1288,9 +1320,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
                       len += c->cache_info.signed_descriptor_len);
 
-    if (global_write_bucket_low(TO_CONN(conn),
-                                compress_method != NO_METHOD ? len/2 : len,
-                                2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto keys_done;
     }
@@ -1302,19 +1333,13 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     if (compress_method != NO_METHOD) {
       conn->compress_state = tor_compress_new(1, compress_method,
                                               choose_compression_level(len));
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add_compress(
-                c->cache_info.signed_descriptor_body,
-                c->cache_info.signed_descriptor_len,
-                conn, 0));
-      connection_buf_add_compress("", 0, conn, 1);
-    } else {
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add(c->cache_info.signed_descriptor_body,
-                                    c->cache_info.signed_descriptor_len,
-                                    TO_CONN(conn)));
     }
-  keys_done:
+
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+          connection_dir_buf_add(c->cache_info.signed_descriptor_body,
+                                 c->cache_info.signed_descriptor_len,
+                                 conn, c_sl_idx == c_sl_len - 1));
+ keys_done:
     smartlist_free(certs);
     goto done;
   }
@@ -1360,7 +1385,7 @@ handle_get_hs_descriptor_v2(dir_connection_t *conn,
   return 0;
 }
 
-/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+/** Helper function for GET `/tor/hs/3/...`. Only for version 3.
  */
 STATIC int
 handle_get_hs_descriptor_v3(dir_connection_t *conn,
@@ -1371,9 +1396,11 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn,
   const char *pubkey_str = NULL;
   const char *url = args->url;
 
-  /* Reject unencrypted dir connections */
-  if (!connection_dir_is_encrypted(conn)) {
-    write_short_http_response(conn, 404, "Not found");
+  /* Reject non anonymous dir connections (which also tests if encrypted). We
+   * do not allow single hop clients to query an HSDir. */
+  if (!connection_dir_is_anonymous(conn)) {
+    write_short_http_response(conn, 503,
+                              "Rejecting single hop HS v3 descriptor request");
     goto done;
   }
 
@@ -1438,6 +1465,39 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
   return 0;
 }
 
+/** Helper function for GET the bandwidth file used for the next vote */
+static int
+handle_get_next_bandwidth(dir_connection_t *conn,
+                          const get_handler_args_t *args)
+{
+  log_debug(LD_DIR, "Getting next bandwidth.");
+  const or_options_t *options = get_options();
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+
+  if (options->V3BandwidthsFile) {
+    char *bandwidth = read_file_to_str(options->V3BandwidthsFile,
+                                       RFTS_IGNORE_MISSING, NULL);
+    if (bandwidth != NULL) {
+      ssize_t len = strlen(bandwidth);
+      write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len,
+                                 compress_method, BANDWIDTH_CACHE_LIFETIME);
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(len/2));
+        log_debug(LD_DIR, "Compressing bandwidth file.");
+      } else {
+        log_debug(LD_DIR, "Not compressing bandwidth file.");
+      }
+      connection_dir_buf_add((const char*)bandwidth, len, conn, 1);
+      tor_free(bandwidth);
+      return 0;
+    }
+  }
+  write_short_http_response(conn, 404, "Not found");
+  return 0;
+}
+
 /** Helper function for GET robots.txt or /tor/robots.txt */
 static int
 handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
@@ -1580,10 +1640,15 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     goto done;
   }
 
-  /* Handle HS descriptor publish request. */
-  /* XXX: This should be disabled with a consensus param until we want to
-   * the prop224 be deployed and thus use. */
-  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+  /* Handle HS descriptor publish request. We force an anonymous connection
+   * (which also tests for encrypted). We do not allow single-hop client to
+   * post a descriptor onto an HSDir. */
+  if (!strcmpstart(url, "/tor/hs/")) {
+    if (!connection_dir_is_anonymous(conn)) {
+      write_short_http_response(conn, 503,
+                                "Rejecting single hop HS descriptor post");
+      goto done;
+    }
     const char *msg = "HS descriptor stored successfully.";
 
     /* We most probably have a publish request for an HS descriptor. */
@@ -1608,8 +1673,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     const char *msg = "[None]";
     uint8_t purpose = authdir_mode_bridge(options) ?
                       ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
-                                             conn->base_.address, &msg);
+    was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len,
+                                           purpose, conn->base_.address, &msg);
     tor_assert(msg);
 
     if (r == ROUTER_ADDED_SUCCESSFULLY) {
@@ -1631,7 +1696,7 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
       !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
     const char *msg = "OK";
     int status;
-    if (dirvote_add_vote(body, &msg, &status)) {
+    if (dirvote_add_vote(body, approx_time(), &msg, &status)) {
       write_short_http_response(conn, status, "Vote stored");
     } else {
       tor_assert(msg);
diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h
index 236ea649ef..d6392e2d42 100644
--- a/src/feature/dircache/dircache.h
+++ b/src/feature/dircache/dircache.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,6 +38,6 @@ STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
                                       const char **end_pos);
 
 STATIC unsigned parse_accept_encoding_header(const char *h);
-#endif
+#endif /* defined(DIRCACHE_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCACHE_H) */
diff --git a/src/feature/dircache/dircache_stub.c b/src/feature/dircache/dircache_stub.c
new file mode 100644
index 0000000000..725c44bd4d
--- /dev/null
+++ b/src/feature/dircache/dircache_stub.c
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dircache_stub.c
+ * @brief Stub declarations for use when dircache module is disabled.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/dir_connection_st.h"
+
+DISABLE_GCC_WARNING("-Wmissing-noreturn")
+int
+directory_handle_command(dir_connection_t *conn)
+{
+  (void) conn;
+  tor_assert_nonfatal_unreached_once();
+  return -1;
+}
+
+int
+connection_dirserv_flushed_some(dir_connection_t *conn)
+{
+  (void) conn;
+  tor_assert_nonfatal_unreached_once();
+  return -1;
+}
+ENABLE_GCC_WARNING("-Wmissing-noreturn")
+
+void
+dir_conn_clear_spool(dir_connection_t *conn)
+{
+  if (!conn)
+    return;
+  tor_assert_nonfatal_once(conn->spool == NULL);
+}
+
+void
+consdiffmgr_enable_background_compression(void)
+{
+}
+
+int
+consdiffmgr_add_consensus(const char *consensus,
+                          size_t consensus_len,
+                          const networkstatus_t *as_parsed)
+{
+  (void)consensus;
+  (void)consensus_len;
+  (void)as_parsed;
+  return 0;
+}
+
+int
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg)
+{
+  (void)cfg;
+  return 0;
+}
+
+int
+consdiffmgr_cleanup(void)
+{
+  return 0;
+}
+
+void
+consdiffmgr_free_all(void)
+{
+}
+
+void
+dirserv_free_all(void)
+{
+}
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
index 213c490314..fb8db879a4 100644
--- a/src/feature/dircache/dirserv.c
+++ b/src/feature/dircache/dirserv.c
@@ -1,9 +1,8 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define DIRSERV_PRIVATE
 #include "core/or/or.h"
 
 #include "app/config/config.h"
@@ -69,55 +68,7 @@ static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp);
 /********************************************************************/
 
 /* A set of functions to answer questions about how we'd like to behave
- * as a directory mirror/client. */
-
-/** Return 1 if we fetch our directory material directly from the
- * authorities, rather than from a mirror. */
-int
-directory_fetches_from_authorities(const or_options_t *options)
-{
-  const routerinfo_t *me;
-  uint32_t addr;
-  int refuseunknown;
-  if (options->FetchDirInfoEarly)
-    return 1;
-  if (options->BridgeRelay == 1)
-    return 0;
-  if (server_mode(options) &&
-      router_pick_published_address(options, &addr, 1) < 0)
-    return 1; /* we don't know our IP address; ask an authority. */
-  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
-    should_refuse_unknown_exits(options);
-  if (!dir_server_mode(options) && !refuseunknown)
-    return 0;
-  if (!server_mode(options) || !advertised_server_mode())
-    return 0;
-  me = router_get_my_routerinfo();
-  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
-    return 0; /* if we don't service directory requests, return 0 too */
-  return 1;
-}
-
-/** Return 1 if we should fetch new networkstatuses, descriptors, etc
- * on the "mirror" schedule rather than the "client" schedule.
- */
-int
-directory_fetches_dir_info_early(const or_options_t *options)
-{
-  return directory_fetches_from_authorities(options);
-}
-
-/** Return 1 if we should fetch new networkstatuses, descriptors, etc
- * on a very passive schedule -- waiting long enough for ordinary clients
- * to probably have the info we want. These would include bridge users,
- * and maybe others in the future e.g. if a Tor client uses another Tor
- * client as a directory guard.
- */
-int
-directory_fetches_dir_info_later(const or_options_t *options)
-{
-  return options->UseBridges != 0;
-}
+ * as a directory mirror */
 
 /** Return true iff we want to serve certificates for authorities
  * that we don't acknowledge as authorities ourself.
@@ -161,19 +112,6 @@ directory_permits_begindir_requests(const or_options_t *options)
   return options->BridgeRelay != 0 || dir_server_mode(options);
 }
 
-/** Return 1 if we have no need to fetch new descriptors. This generally
- * happens when we're not a dir cache and we haven't built any circuits
- * lately.
- */
-int
-directory_too_idle_to_fetch_descriptors(const or_options_t *options,
-                                        time_t now)
-{
-  return !directory_caches_dir_info(options) &&
-         !options->FetchUselessDescriptors &&
-         rep_hist_circbuilding_dormant(now);
-}
-
 /********************************************************************/
 
 /** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
@@ -234,6 +172,7 @@ free_cached_dir_(void *_d)
  * validation is performed. */
 void
 dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
+                                           size_t networkstatus_len,
                                            const char *flavor_name,
                                            const common_digests_t *digests,
                                            const uint8_t *sha3_as_signed,
@@ -244,7 +183,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
   if (!cached_consensuses)
     cached_consensuses = strmap_new();
 
-  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
+  new_networkstatus =
+    new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len),
+                   published);
   memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
   memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
          DIGEST256_LEN);
@@ -256,14 +197,45 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
 
 /** Return the latest downloaded consensus networkstatus in encoded, signed,
  * optionally compressed format, suitable for sending to clients. */
-cached_dir_t *
-dirserv_get_consensus(const char *flavor_name)
+MOCK_IMPL(cached_dir_t *,
+dirserv_get_consensus,(const char *flavor_name))
 {
   if (!cached_consensuses)
     return NULL;
   return strmap_get(cached_consensuses, flavor_name);
 }
 
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+                                  dir_spool_source_t source,
+                                  smartlist_t *spool_out,
+                                  int *compressed_out,
+                                  int flags)
+{
+  smartlist_t *fingerprints = smartlist_new();
+
+  tor_assert(flags & (DSR_HEX|DSR_BASE64));
+  const size_t digest_len =
+    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+                                               compressed_out, flags);
+  /* This is not a very efficient implementation XXXX */
+  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+    spooled_resource_t *spooled =
+      spooled_resource_new(source, digest, digest_len);
+    if (spooled)
+      smartlist_add(spool_out, spooled);
+    tor_free(digest);
+  } SMARTLIST_FOREACH_END(digest);
+
+  smartlist_free(fingerprints);
+  return r;
+}
+
 /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
  * pointers, adds copies of digests to fps_out, and doesn't use the
  * /tor/server/ prefix.  For a /d/ request, adds descriptor digests; for other
@@ -330,87 +302,6 @@ dirserv_get_routerdesc_spool(smartlist_t *spool_out,
   return 0;
 }
 
-/** Add a signed_descriptor_t to <b>descs_out</b> for each router matching
- * <b>key</b>.  The key should be either
- *   - "/tor/server/authority" for our own routerinfo;
- *   - "/tor/server/all" for all the routerinfos we have, concatenated;
- *   - "/tor/server/fp/FP" where FP is a plus-separated sequence of
- *     hex identity digests; or
- *   - "/tor/server/d/D" where D is a plus-separated sequence
- *     of server descriptor digests, in hex.
- *
- * Return 0 if we found some matching descriptors, or -1 if we do not
- * have any descriptors, no matching descriptors, or if we did not
- * recognize the key (URL).
- * If -1 is returned *<b>msg</b> will be set to an appropriate error
- * message.
- *
- * XXXX rename this function.  It's only called from the controller.
- * XXXX in fact, refactor this function, merging as much as possible.
- */
-int
-dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
-                        const char **msg)
-{
-  *msg = NULL;
-
-  if (!strcmp(key, "/tor/server/all")) {
-    routerlist_t *rl = router_get_routerlist();
-    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
-                      smartlist_add(descs_out, &(r->cache_info)));
-  } else if (!strcmp(key, "/tor/server/authority")) {
-    const routerinfo_t *ri = router_get_my_routerinfo();
-    if (ri)
-      smartlist_add(descs_out, (void*) &(ri->cache_info));
-  } else if (!strcmpstart(key, "/tor/server/d/")) {
-    smartlist_t *digests = smartlist_new();
-    key += strlen("/tor/server/d/");
-    dir_split_resource_into_fingerprints(key, digests, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-    SMARTLIST_FOREACH(digests, const char *, d,
-       {
-         signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
-         if (sd)
-           smartlist_add(descs_out,sd);
-       });
-    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
-    smartlist_free(digests);
-  } else if (!strcmpstart(key, "/tor/server/fp/")) {
-    smartlist_t *digests = smartlist_new();
-    time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
-    key += strlen("/tor/server/fp/");
-    dir_split_resource_into_fingerprints(key, digests, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-    SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
-         if (router_digest_is_me(d)) {
-           /* calling router_get_my_routerinfo() to make sure it exists */
-           const routerinfo_t *ri = router_get_my_routerinfo();
-           if (ri)
-             smartlist_add(descs_out, (void*) &(ri->cache_info));
-         } else {
-           const routerinfo_t *ri = router_get_by_id_digest(d);
-           /* Don't actually serve a descriptor that everyone will think is
-            * expired.  This is an (ugly) workaround to keep buggy 0.1.1.10
-            * Tors from downloading descriptors that they will throw away.
-            */
-           if (ri && ri->cache_info.published_on > cutoff)
-             smartlist_add(descs_out, (void*) &(ri->cache_info));
-         }
-    } SMARTLIST_FOREACH_END(d);
-    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
-    smartlist_free(digests);
-  } else {
-    *msg = "Key not recognized";
-    return -1;
-  }
-
-  if (!smartlist_len(descs_out)) {
-    *msg = "Servers unavailable";
-    return -1;
-  }
-  return 0;
-}
-
 /* ==========
  * Spooling code.
  * ========== */
@@ -580,11 +471,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
       /* Absent objects count as "done". */
       return SRFS_DONE;
     }
-    if (conn->compress_state) {
-      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
-    } else {
-      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add((const char*)body, bodylen, conn, 0);
+
     return SRFS_DONE;
   } else {
     cached_dir_t *cached = spooled->cached_dir_ref;
@@ -619,14 +508,10 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
     if (BUG(remaining < 0))
       return SRFS_ERR;
     ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
-    if (conn->compress_state) {
-      connection_buf_add_compress(
-              ptr + spooled->cached_dir_offset,
-              bytes, conn, 0);
-    } else {
-      connection_buf_add(ptr + spooled->cached_dir_offset,
-                              bytes, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add(ptr + spooled->cached_dir_offset,
+                           bytes, conn, 0);
+
     spooled->cached_dir_offset += bytes;
     if (spooled->cached_dir_offset >= (off_t)total_len) {
       return SRFS_DONE;
diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h
index 890b10fd80..73a64b1b7e 100644
--- a/src/feature/dircache/dirserv.h
+++ b/src/feature/dircache/dirserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -73,28 +73,52 @@ typedef struct spooled_resource_t {
 
 int connection_dirserv_flushed_some(dir_connection_t *conn);
 
-int directory_fetches_from_authorities(const or_options_t *options);
-int directory_fetches_dir_info_early(const or_options_t *options);
-int directory_fetches_dir_info_later(const or_options_t *options);
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+                                      enum dir_spool_source_t source,
+                                      smartlist_t *spool_out,
+                                      int *compressed_out,
+                                      int flags);
+
+#ifdef HAVE_MODULE_DIRCACHE
+/** Is the dircache module enabled? */
+#define have_module_dircache() (1)
 int directory_caches_unknown_auth_certs(const or_options_t *options);
 int directory_caches_dir_info(const or_options_t *options);
 int directory_permits_begindir_requests(const or_options_t *options);
-int directory_too_idle_to_fetch_descriptors(const or_options_t *options,
-                                            time_t now);
-
-cached_dir_t *dirserv_get_consensus(const char *flavor_name);
+MOCK_DECL(cached_dir_t *, dirserv_get_consensus, (const char *flavor_name));
 void dirserv_set_cached_consensus_networkstatus(const char *consensus,
+                                              size_t consensus_len,
                                               const char *flavor_name,
                                               const common_digests_t *digests,
                                               const uint8_t *sha3_as_signed,
                                               time_t published);
+#else /* !defined(HAVE_MODULE_DIRCACHE) */
+#define have_module_dircache() (0)
+#define directory_caches_unknown_auth_certs(opt) \
+  ((void)(opt), 0)
+#define directory_caches_dir_info(opt) \
+  ((void)(opt), 0)
+#define directory_permits_begindir_requests(opt) \
+  ((void)(opt), 0)
+#define dirserv_get_consensus(flav) \
+  ((void)(flav), NULL)
+#define dirserv_set_cached_consensus_networkstatus(a,b,c,d,e,f) \
+  STMT_BEGIN {                                                  \
+    (void)(a);                                                  \
+    (void)(b);                                                  \
+    (void)(c);                                                  \
+    (void)(d);                                                  \
+    (void)(e);                                                  \
+    (void)(f);                                                  \
+  } STMT_END
+#endif /* defined(HAVE_MODULE_DIRCACHE) */
+
 void dirserv_clear_old_networkstatuses(time_t cutoff);
 int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key,
                                  dir_spool_source_t source,
                                  int conn_is_encrypted,
                                  const char **msg_out);
-int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
-                            const char **msg);
 
 void dirserv_free_all(void);
 void cached_dir_decref(cached_dir_t *d);
diff --git a/src/feature/dircache/feature_dircache.md b/src/feature/dircache/feature_dircache.md
new file mode 100644
index 0000000000..97734f2a34
--- /dev/null
+++ b/src/feature/dircache/feature_dircache.md
@@ -0,0 +1,6 @@
+@dir /feature/dircache
+@brief feature/dircache: Run as a directory cache server
+
+This module handles the directory caching functionality that all relays may
+provide, for serving cached directory objects to objects.
+
diff --git a/src/feature/dircache/include.am b/src/feature/dircache/include.am
new file mode 100644
index 0000000000..ab162565f7
--- /dev/null
+++ b/src/feature/dircache/include.am
@@ -0,0 +1,21 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+MODULE_DIRCACHE_SOURCES = 			\
+	src/feature/dircache/conscache.c	\
+	src/feature/dircache/consdiffmgr.c	\
+	src/feature/dircache/dircache.c		\
+	src/feature/dircache/dirserv.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dircache/cached_dir_st.h		\
+	src/feature/dircache/conscache.h		\
+	src/feature/dircache/consdiffmgr.h		\
+	src/feature/dircache/dircache.h			\
+	src/feature/dircache/dirserv.h
+
+if BUILD_MODULE_DIRCACHE
+LIBTOR_APP_A_SOURCES += $(MODULE_DIRCACHE_SOURCES)
+else
+LIBTOR_APP_A_STUB_SOURCES += src/feature/dircache/dircache_stub.c
+endif
diff --git a/src/feature/dirclient/.may_include b/src/feature/dirclient/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dirclient/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h
index 2f5706cdd9..37fa3148a7 100644
--- a/src/feature/dirclient/dir_server_st.h
+++ b/src/feature/dirclient/dir_server_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dir_server_st.h
+ * @brief Trusted/fallback directory server structure.
+ **/
+
 #ifndef DIR_SERVER_ST_H
 #define DIR_SERVER_ST_H
 
@@ -51,4 +56,4 @@ struct dir_server_t {
                                **/
 };
 
-#endif
+#endif /* !defined(DIR_SERVER_ST_H) */
diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c
index 6725fc3369..ae1e018df2 100644
--- a/src/feature/dirclient/dirclient.c
+++ b/src/feature/dirclient/dirclient.c
@@ -1,8 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dirclient.c
+ * @brief Download directory information
+ **/
+
 #define DIRCLIENT_PRIVATE
 
 #include "core/or/or.h"
@@ -14,12 +19,14 @@
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/shared_random.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/consdiff.h"
 #include "feature/dircommon/directory.h"
@@ -37,6 +44,7 @@
 #include "feature/nodelist/routerinfo.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
+#include "feature/relay/relay_find_addr.h"
 #include "feature/relay/routermode.h"
 #include "feature/relay/selftest.h"
 #include "feature/rend/rendcache.h"
@@ -45,6 +53,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/predict_ports.h"
 
+#include "lib/cc/ctassert.h"
 #include "lib/compress/compress.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -448,7 +457,7 @@ directory_get_from_dirserver,(
 {
   const routerstatus_t *rs = NULL;
   const or_options_t *options = get_options();
-  int prefer_authority = (directory_fetches_from_authorities(options)
+  int prefer_authority = (dirclient_fetches_from_authorities(options)
                           || want_authority == DL_WANT_AUTHORITY);
   int require_authority = 0;
   int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
@@ -667,7 +676,7 @@ directory_choose_address_routerstatus(const routerstatus_t *status,
   if (indirection == DIRIND_DIRECT_CONN ||
       indirection == DIRIND_ANON_DIRPORT ||
       (indirection == DIRIND_ONEHOP
-       && !directory_must_use_begindir(options))) {
+       && !dirclient_must_use_begindir(options))) {
     fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0,
                                        use_dir_ap);
     have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0);
@@ -866,16 +875,6 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status)
   update_certificate_downloads(time(NULL));
 }
 
-/* Should this tor instance only use begindir for all its directory requests?
- */
-int
-directory_must_use_begindir(const or_options_t *options)
-{
-  /* Clients, onion services, and bridges must use begindir,
-   * relays and authorities do not have to */
-  return !public_server_mode(options);
-}
-
 /** Evaluate the situation and decide if we should use an encrypted
  * "begindir-style" connection for this directory request.
  * 0) If there is no DirPort, yes.
@@ -927,7 +926,7 @@ directory_command_should_use_begindir(const or_options_t *options,
   }
   /* Reasons why we want to avoid using begindir */
   if (indirection == DIRIND_ONEHOP) {
-    if (!directory_must_use_begindir(options)) {
+    if (!dirclient_must_use_begindir(options)) {
       *reason = "in relay mode";
       return 0;
     }
@@ -1289,7 +1288,7 @@ directory_initiate_request,(directory_request_t *request))
 
   /* use encrypted begindir connections for everything except relays
    * this provides better protection for directory fetches */
-  if (!use_begindir && directory_must_use_begindir(options)) {
+  if (!use_begindir && dirclient_must_use_begindir(options)) {
     log_warn(LD_BUG, "Client could not use begindir connection: %s",
              begindir_reason ? begindir_reason : "(NULL)");
     return;
@@ -1447,9 +1446,7 @@ compare_strs_(const void **a, const void **b)
 }
 
 #define CONDITIONAL_CONSENSUS_FPR_LEN 3
-#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
-#error "conditional consensus fingerprint length is larger than digest length"
-#endif
+CTASSERT(CONDITIONAL_CONSENSUS_FPR_LEN <= DIGEST_LEN);
 
 /** Return the URL we should use for a consensus download.
  *
@@ -1968,6 +1965,44 @@ dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
   return rv;
 }
 
+/**
+ * Total number of bytes downloaded of each directory purpose, when
+ * bootstrapped, and when not bootstrapped.
+ *
+ * (For example, the number of bytes downloaded of purpose p while
+ * not fully bootstrapped is total_dl[p][false].)
+ **/
+static uint64_t total_dl[DIR_PURPOSE_MAX_][2];
+
+/**
+ * Heartbeat: dump a summary of how many bytes of which purpose we've
+ * downloaded, when bootstrapping and when not bootstrapping.
+ **/
+void
+dirclient_dump_total_dls(void)
+{
+  const or_options_t *options = get_options();
+  for (int bootstrapped = 0; bootstrapped < 2; ++bootstrapped) {
+    bool first_time = true;
+    for (int i=0; i < DIR_PURPOSE_MAX_; ++i) {
+      uint64_t n = total_dl[i][bootstrapped];
+      if (n == 0)
+        continue;
+      if (options->SafeLogging_ != SAFELOG_SCRUB_NONE &&
+          purpose_needs_anonymity(i, ROUTER_PURPOSE_GENERAL, NULL))
+        continue;
+      if (first_time) {
+        log_notice(LD_NET,
+                   "While %sbootstrapping, fetched this many bytes: ",
+                   bootstrapped?"not ":"");
+        first_time = false;
+      }
+      log_notice(LD_NET, "    %"PRIu64" (%s)",
+                 n, dir_conn_purpose_to_string(i));
+    }
+  }
+}
+
 /** We are a client, and we've finished reading the server's
  * response. Parse it and act appropriately.
  *
@@ -2001,6 +2036,16 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
 
   received_bytes = connection_get_inbuf_len(TO_CONN(conn));
 
+  log_debug(LD_DIR, "Downloaded %"TOR_PRIuSZ" bytes on connection of purpose "
+             "%s; bootstrap %d%%",
+             received_bytes,
+             dir_conn_purpose_to_string(conn->base_.purpose),
+             control_get_bootstrap_percent());
+  {
+    bool bootstrapped = control_get_bootstrap_percent() == 100;
+    total_dl[conn->base_.purpose][bootstrapped] += received_bytes;
+  }
+
   switch (connection_fetch_from_buf_http(TO_CONN(conn),
                               &headers, MAX_HEADERS_SIZE,
                               &body, &body_len, MAX_DIR_DL_SIZE,
@@ -2205,24 +2250,31 @@ handle_response_fetch_consensus(dir_connection_t *conn,
   if (looks_like_a_consensus_diff(body, body_len)) {
     /* First find our previous consensus. Maybe it's in ram, maybe not. */
     cached_dir_t *cd = dirserv_get_consensus(flavname);
-    const char *consensus_body;
-    char *owned_consensus = NULL;
+    const char *consensus_body = NULL;
+    size_t consensus_body_len;
+    tor_mmap_t *mapped_consensus = NULL;
     if (cd) {
       consensus_body = cd->dir;
+      consensus_body_len = cd->dir_len;
     } else {
-      owned_consensus = networkstatus_read_cached_consensus(flavname);
-      consensus_body = owned_consensus;
+      mapped_consensus = networkstatus_map_cached_consensus(flavname);
+      if (mapped_consensus) {
+        consensus_body = mapped_consensus->data;
+        consensus_body_len = mapped_consensus->size;
+      }
     }
     if (!consensus_body) {
       log_warn(LD_DIR, "Received a consensus diff, but we can't find "
                "any %s-flavored consensus in our current cache.",flavname);
+      tor_munmap_file(mapped_consensus);
       networkstatus_consensus_download_failed(0, flavname);
       // XXXX if this happens too much, see below
       return -1;
     }
 
-    new_consensus = consensus_diff_apply(consensus_body, body);
-    tor_free(owned_consensus);
+    new_consensus = consensus_diff_apply(consensus_body, consensus_body_len,
+                                         body, body_len);
+    tor_munmap_file(mapped_consensus);
     if (new_consensus == NULL) {
       log_warn(LD_DIR, "Could not apply consensus diff received from server "
                "'%s:%d'", conn->base_.address, conn->base_.port);
@@ -2244,7 +2296,9 @@ handle_response_fetch_consensus(dir_connection_t *conn,
     sourcename = "downloaded";
   }
 
-  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+  if ((r=networkstatus_set_current_consensus(consensus,
+                                             strlen(consensus),
+                                             flavname, 0,
                                              conn->identity_digest))<0) {
     log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
            "Unable to load %s consensus directory %s from "
@@ -2359,7 +2413,7 @@ handle_response_fetch_status_vote(dir_connection_t *conn,
              conn->base_.port, conn->requested_resource);
     return -1;
   }
-  dirvote_add_vote(body, &msg, &st);
+  dirvote_add_vote(body, 0, &msg, &st);
   if (st > 299) {
     log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
   } else {
@@ -2522,7 +2576,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn,
            conn->base_.port);
   tor_assert(conn->requested_resource &&
              !strcmpstart(conn->requested_resource, "d/"));
-  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  tor_assert_nonfatal(!fast_mem_is_zero(conn->identity_digest, DIGEST_LEN));
   which = smartlist_new();
   dir_split_resource_into_fingerprints(conn->requested_resource+2,
                                        which, NULL,
@@ -2719,62 +2773,7 @@ handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
   log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))",
            (int)body_len, status_code, escaped(reason));
 
-  switch (status_code) {
-  case 200:
-    /* We got something: Try storing it in the cache. */
-    if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) {
-      log_info(LD_REND, "Failed to store hidden service descriptor");
-      /* Fire control port FAILED event. */
-      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                   "BAD_DESC");
-      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                    NULL);
-    } else {
-      log_info(LD_REND, "Stored hidden service descriptor successfully.");
-      TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
-      hs_client_desc_has_arrived(conn->hs_ident);
-      /* Fire control port RECEIVED event. */
-      hs_control_desc_event_received(conn->hs_ident, conn->identity_digest);
-      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                    body);
-    }
-    break;
-  case 404:
-    /* Not there. We'll retry when connection_about_to_close_connection()
-     * tries to clean this conn up. */
-    log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
-                      "Retrying at another directory.");
-    /* Fire control port FAILED event. */
-    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                 "NOT_FOUND");
-    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                  NULL);
-    break;
-  case 400:
-    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
-                      "http status 400 (%s). Dirserver didn't like our "
-                      "query? Retrying at another directory.",
-             escaped(reason));
-    /* Fire control port FAILED event. */
-    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                 "QUERY_REJECTED");
-    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                  NULL);
-    break;
-  default:
-    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
-             "http status %d (%s) response unexpected from HSDir server "
-             "'%s:%d'. Retrying at another directory.",
-             status_code, escaped(reason), TO_CONN(conn)->address,
-             TO_CONN(conn)->port);
-    /* Fire control port FAILED event. */
-    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                 "UNEXPECTED");
-    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                  NULL);
-    break;
-  }
-
+  hs_client_dir_fetch_done(conn, reason, body, status_code);
   return 0;
 }
 
@@ -3134,7 +3133,7 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
 {
   char digest[DIGEST_LEN];
   time_t now = time(NULL);
-  int server = directory_fetches_from_authorities(get_options());
+  int server = dirclient_fetches_from_authorities(get_options());
   if (!was_descriptor_digests) {
     if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
       tor_assert(!was_extrainfo);
@@ -3179,7 +3178,7 @@ dir_microdesc_download_failed(smartlist_t *failed,
   routerstatus_t *rs;
   download_status_t *dls;
   time_t now = time(NULL);
-  int server = directory_fetches_from_authorities(get_options());
+  int server = dirclient_fetches_from_authorities(get_options());
 
   if (! consensus)
     return;
diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h
index 1a93265dc3..096b197526 100644
--- a/src/feature/dirclient/dirclient.h
+++ b/src/feature/dirclient/dirclient.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,8 @@
 
 #include "feature/hs/hs_ident.h"
 
+void dirclient_dump_total_dls(void);
+
 int directories_have_accepted_server_descriptor(void);
 void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
                                   dirinfo_type_t type, const char *payload,
@@ -41,8 +43,6 @@ typedef enum {
   DIRIND_ANON_DIRPORT,
 } dir_indirection_t;
 
-int directory_must_use_begindir(const or_options_t *options);
-
 /**
  * A directory_request_t describes the information about a directory request
  * at the client side.  It describes what we're going to ask for, which
@@ -167,6 +167,6 @@ STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
 
 STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
                                      const char *resource);
-#endif
+#endif /* defined(DIRCLIENT_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCLIENT_H) */
diff --git a/src/feature/dirclient/dirclient_modes.c b/src/feature/dirclient/dirclient_modes.c
new file mode 100644
index 0000000000..31a3f8af58
--- /dev/null
+++ b/src/feature/dirclient/dirclient_modes.c
@@ -0,0 +1,96 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirclient_modes.c
+ * @brief Functions to answer questions about how we'd like to behave
+ *     as a directory client
+ **/
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+
+#include "feature/dirclient/dirclient_modes.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/relay/relay_find_addr.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/stats/predict_ports.h"
+
+#include "app/config/or_options_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/* Should this tor instance only use begindir for all its directory requests?
+ */
+int
+dirclient_must_use_begindir(const or_options_t *options)
+{
+  /* Clients, onion services, and bridges must use begindir,
+   * relays and authorities do not have to */
+  return !public_server_mode(options);
+}
+
+/** Return 1 if we fetch our directory material directly from the
+ * authorities, rather than from a mirror. */
+int
+dirclient_fetches_from_authorities(const or_options_t *options)
+{
+  const routerinfo_t *me;
+  uint32_t addr;
+  int refuseunknown;
+  if (options->FetchDirInfoEarly)
+    return 1;
+  if (options->BridgeRelay == 1)
+    return 0;
+  if (server_mode(options) &&
+      router_pick_published_address(options, &addr, 1) < 0)
+    return 1; /* we don't know our IP address; ask an authority. */
+  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
+    should_refuse_unknown_exits(options);
+  if (!dir_server_mode(options) && !refuseunknown)
+    return 0;
+  if (!server_mode(options) || !advertised_server_mode())
+    return 0;
+  me = router_get_my_routerinfo();
+  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
+    return 0; /* if we don't service directory requests, return 0 too */
+  return 1;
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on the "mirror" schedule rather than the "client" schedule.
+ */
+int
+dirclient_fetches_dir_info_early(const or_options_t *options)
+{
+  return dirclient_fetches_from_authorities(options);
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on a very passive schedule -- waiting long enough for ordinary clients
+ * to probably have the info we want. These would include bridge users,
+ * and maybe others in the future e.g. if a Tor client uses another Tor
+ * client as a directory guard.
+ */
+int
+dirclient_fetches_dir_info_later(const or_options_t *options)
+{
+  return options->UseBridges != 0;
+}
+
+/** Return 1 if we have no need to fetch new descriptors. This generally
+ * happens when we're not a dir cache and we haven't built any circuits
+ * lately.
+ */
+int
+dirclient_too_idle_to_fetch_descriptors(const or_options_t *options,
+                                        time_t now)
+{
+  return !directory_caches_dir_info(options) &&
+         !options->FetchUselessDescriptors &&
+         rep_hist_circbuilding_dormant(now);
+}
diff --git a/src/feature/dirclient/dirclient_modes.h b/src/feature/dirclient/dirclient_modes.h
new file mode 100644
index 0000000000..c402207724
--- /dev/null
+++ b/src/feature/dirclient/dirclient_modes.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirclient_modes.h
+ * @brief Header for feature/dirclient/dirclient_modes.c
+ **/
+
+#ifndef TOR_FEATURE_DIRCLIENT_DIRCLIENT_MODES_H
+#define TOR_FEATURE_DIRCLIENT_DIRCLIENT_MODES_H
+
+struct or_options_t;
+
+int dirclient_must_use_begindir(const or_options_t *options);
+int dirclient_fetches_from_authorities(const struct or_options_t *options);
+int dirclient_fetches_dir_info_early(const struct or_options_t *options);
+int dirclient_fetches_dir_info_later(const struct or_options_t *options);
+int dirclient_too_idle_to_fetch_descriptors(const struct or_options_t *options,
+                                            time_t now);
+
+#endif /* !defined(TOR_FEATURE_DIRCLIENT_DIRCLIENT_MODES_H) */
diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c
index 0842a2c676..ab3fbb8577 100644
--- a/src/feature/dirclient/dlstatus.c
+++ b/src/feature/dirclient/dlstatus.c
@@ -1,8 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dlstatus.c
+ * @brief Track status and retry schedule of a downloadable object.
+ **/
+
 #define DLSTATUS_PRIVATE
 
 #include "core/or/or.h"
diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h
index 99e0d0225b..e5c8b756c4 100644
--- a/src/feature/dirclient/dlstatus.h
+++ b/src/feature/dirclient/dlstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -53,6 +53,6 @@ STATIC void next_random_exponential_delay_range(int *low_bound_out,
 /* no more than triple the previous delay */
 #define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
 
-#endif
+#endif /* defined(DLSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_DLSTATUS_H) */
diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h
index 11555a1dcc..92efcb44d0 100644
--- a/src/feature/dirclient/download_status_st.h
+++ b/src/feature/dirclient/download_status_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file download_status_st.h
+ * @brief Directory download status/schedule structure.
+ **/
+
 #ifndef DOWNLOAD_STATUS_ST_H
 #define DOWNLOAD_STATUS_ST_H
 
@@ -61,5 +66,4 @@ struct download_status_t {
                         * only updated if backoff == 1 */
 };
 
-#endif
-
+#endif /* !defined(DOWNLOAD_STATUS_ST_H) */
diff --git a/src/feature/dirclient/feature_dirclient.md b/src/feature/dirclient/feature_dirclient.md
new file mode 100644
index 0000000000..5c7ee964d3
--- /dev/null
+++ b/src/feature/dirclient/feature_dirclient.md
@@ -0,0 +1,7 @@
+@dir /feature/dirclient
+@brief feature/dirclient: Directory client implementation.
+
+The code here is used by all Tor instances that need to download directory
+information.  Currently, that is all of them, since even authorities need to
+launch downloads to learn about relays that other authorities have listed.
+
diff --git a/src/feature/dirclient/include.am b/src/feature/dirclient/include.am
new file mode 100644
index 0000000000..24cae9eedd
--- /dev/null
+++ b/src/feature/dirclient/include.am
@@ -0,0 +1,14 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/dirclient/dirclient.c	\
+	src/feature/dirclient/dirclient_modes.c	\
+	src/feature/dirclient/dlstatus.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dirclient/dir_server_st.h		\
+	src/feature/dirclient/dirclient.h		\
+	src/feature/dirclient/dirclient_modes.h		\
+	src/feature/dirclient/dlstatus.h		\
+	src/feature/dirclient/download_status_st.h
diff --git a/src/feature/dircommon/.may_include b/src/feature/dircommon/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dircommon/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c
index d0f7594ce3..e42378c44c 100644
--- a/src/feature/dircommon/consdiff.c
+++ b/src/feature/dircommon/consdiff.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2014, Daniel Martí
- * Copyright (c) 2014-2019, The Tor Project, Inc. */
+ * Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest,(const char *cons,
+consensus_compute_digest,(const char *cons, size_t len,
                           consensus_digest_t *digest_out))
 {
   int r = crypto_digest256((char*)digest_out->sha3_256,
-                           cons, strlen(cons), DIGEST_SHA3_256);
+                           cons, len, DIGEST_SHA3_256);
   return r;
 }
 
@@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons,
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest_as_signed,(const char *cons,
+consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out))
 {
   return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
-                                                    cons);
+                                                    cons, len);
 }
 
 /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
@@ -530,10 +530,12 @@ typedef struct router_id_iterator_t {
   cdline_t hash;
 } router_id_iterator_t;
 
+#ifndef COCCI
 /**
  * Initializer for a router_id_iterator_t.
  */
 #define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } }
+#endif /* !defined(COCCI) */
 
 /** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance
  * the index to the next router line ("r ...") in the consensus, or to
@@ -570,7 +572,7 @@ find_next_router_line(const smartlist_t *cons,
 /** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t)
  * to remove the signatures from it.  If the footer is removed, return a
  * cdline_t containing a delete command to delete the footer, allocated in
- * <b>area</>.  If no footer is removed, return NULL.
+ * <b>area</b>.  If no footer is removed, return NULL.
  *
  * We remove the signatures here because they are not themselves signed, and
  * as such there might be different encodings for them.
@@ -1048,7 +1050,7 @@ consdiff_gen_diff(const smartlist_t *cons1,
   if (smartlist_len(cons2) == smartlist_len(ed_cons2)) {
     SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) {
       const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx);
-      if (! lines_eq(line1, line2) ) {
+      if (!lines_eq(line1, line2)) {
         cons2_eq = 0;
         break;
       }
@@ -1229,7 +1231,8 @@ consdiff_apply_diff(const smartlist_t *cons1,
   cons2_str = consensus_join_lines(cons2);
 
   consensus_digest_t cons2_digests;
-  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+  if (consensus_compute_digest(cons2_str, strlen(cons2_str),
+                               &cons2_digests) < 0) {
     /* LCOV_EXCL_START -- digest can't fail */
     log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
         "resulting from applying a consensus diff.");
@@ -1283,12 +1286,13 @@ consdiff_apply_diff(const smartlist_t *cons1,
  * generated cdlines will become invalid.
  */
 STATIC int
-consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+consensus_split_lines(smartlist_t *out,
+                      const char *s, size_t len,
+                      memarea_t *area)
 {
-  const char *end_of_str = s + strlen(s);
-  tor_assert(*end_of_str == '\0');
+  const char *end_of_str = s + len;
 
-  while (*s) {
+  while (s < end_of_str) {
     const char *eol = memchr(s, '\n', end_of_str - s);
     if (!eol) {
       /* File doesn't end with newline. */
@@ -1334,25 +1338,25 @@ consensus_join_lines(const smartlist_t *inp)
  * success, retun a newly allocated string containing that diff.  On failure,
  * return NULL. */
 char *
-consensus_diff_generate(const char *cons1,
-                        const char *cons2)
+consensus_diff_generate(const char *cons1, size_t cons1len,
+                        const char *cons2, size_t cons2len)
 {
   consensus_digest_t d1, d2;
   smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
   int r1, r2;
   char *result = NULL;
 
-  r1 = consensus_compute_digest_as_signed(cons1, &d1);
-  r2 = consensus_compute_digest(cons2, &d2);
+  r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1);
+  r2 = consensus_compute_digest(cons2, cons2len, &d2);
   if (BUG(r1 < 0 || r2 < 0))
     return NULL; // LCOV_EXCL_LINE
 
   memarea_t *area = memarea_new();
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, cons1, area) < 0)
+  if (consensus_split_lines(lines1, cons1, cons1len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, cons2, area) < 0)
+  if (consensus_split_lines(lines2, cons2, cons2len, area) < 0)
     goto done;
 
   result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
@@ -1375,7 +1379,9 @@ consensus_diff_generate(const char *cons1,
  * consensus.  On failure, return NULL. */
 char *
 consensus_diff_apply(const char *consensus,
-                     const char *diff)
+                     size_t consensus_len,
+                     const char *diff,
+                     size_t diff_len)
 {
   consensus_digest_t d1;
   smartlist_t *lines1 = NULL, *lines2 = NULL;
@@ -1383,15 +1389,15 @@ consensus_diff_apply(const char *consensus,
   char *result = NULL;
   memarea_t *area = memarea_new();
 
-  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1);
   if (BUG(r1 < 0))
     goto done;
 
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, consensus, area) < 0)
+  if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, diff, area) < 0)
+  if (consensus_split_lines(lines2, diff, diff_len, area) < 0)
     goto done;
 
   result = consdiff_apply_diff(lines1, lines2, &d1);
diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h
index 98217e6d46..c2dcb6da24 100644
--- a/src/feature/dircommon/consdiff.h
+++ b/src/feature/dircommon/consdiff.h
@@ -1,16 +1,21 @@
 /* Copyright (c) 2014, Daniel Martí
- * Copyright (c) 2014-2019, The Tor Project, Inc. */
+ * Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file consdiff.h
+ * @brief Header for consdiff.c
+ **/
+
 #ifndef TOR_CONSDIFF_H
 #define TOR_CONSDIFF_H
 
 #include "core/or/or.h"
 
-char *consensus_diff_generate(const char *cons1,
-                              const char *cons2);
-char *consensus_diff_apply(const char *consensus,
-                           const char *diff);
+char *consensus_diff_generate(const char *cons1, size_t cons1len,
+                              const char *cons2, size_t cons2len);
+char *consensus_diff_apply(const char *consensus, size_t consensus_len,
+                           const char *diff, size_t diff_len);
 
 int looks_like_a_consensus_diff(const char *document, size_t len);
 
@@ -78,7 +83,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
 STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
                         const smartlist_slice_t *slice1,
                         const smartlist_slice_t *slice2);
-STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+STATIC int consensus_split_lines(smartlist_t *out,
+                                 const char *s, size_t len,
                                  struct memarea_t *area);
 STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
                                   const char *s);
@@ -86,10 +92,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
 STATIC int line_str_eq(const cdline_t *a, const char *b);
 
 MOCK_DECL(STATIC int,
-          consensus_compute_digest,(const char *cons,
+          consensus_compute_digest,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
-          consensus_compute_digest_as_signed,(const char *cons,
+          consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                               consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
           consensus_digest_eq,(const uint8_t *d1,
diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h
index 8c59cc7a46..12230e6741 100644
--- a/src/feature/dircommon/dir_connection_st.h
+++ b/src/feature/dircommon/dir_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dir_connection_st.h
+ * @brief Client/server directory connection structure.
+ **/
+
 #ifndef DIR_CONNECTION_ST_H
 #define DIR_CONNECTION_ST_H
 
@@ -23,7 +28,9 @@ struct dir_connection_t {
   * fingerprints.
   **/
   char *requested_resource;
-  unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
+  /** Is this dirconn direct, or via a multi-hop Tor circuit?
+   * Direct connections can use the DirPort, or BEGINDIR over the ORPort. */
+  unsigned int dirconn_direct:1;
 
   /** If we're fetching descriptors, what router purpose shall we assign
    * to them? */
@@ -64,4 +71,4 @@ struct dir_connection_t {
 #endif /* defined(MEASUREMENTS_21206) */
 };
 
-#endif
+#endif /* !defined(DIR_CONNECTION_ST_H) */
diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c
index 9e6f72e9ac..b177fe5201 100644
--- a/src/feature/dircommon/directory.c
+++ b/src/feature/dircommon/directory.c
@@ -1,12 +1,16 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
 
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/channeltls.h"
 #include "feature/dircache/dircache.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -15,6 +19,10 @@
 #include "feature/stats/geoip_stats.h"
 #include "lib/compress/compress.h"
 
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 
@@ -167,6 +175,82 @@ connection_dir_is_encrypted(const dir_connection_t *conn)
   return TO_CONN(conn)->linked;
 }
 
+/** Return true iff the given directory connection <b>dir_conn</b> is
+ * anonymous, that is, it is on a circuit via a public relay and not directly
+ * from a client or bridge.
+ *
+ * For client circuits via relays: true for 2-hop+ paths.
+ * For client circuits via bridges: true for 3-hop+ paths.
+ *
+ * This first test if the connection is encrypted since it is a strong
+ * requirement for anonymity. */
+bool
+connection_dir_is_anonymous(const dir_connection_t *dir_conn)
+{
+  const connection_t *conn, *linked_conn;
+  const edge_connection_t *edge_conn;
+  const circuit_t *circ;
+
+  tor_assert(dir_conn);
+
+  if (!connection_dir_is_encrypted(dir_conn)) {
+    return false;
+  }
+
+  /*
+   * Buckle up, we'll do a deep dive into the connection in order to get the
+   * final connection channel of that connection in order to figure out if
+   * this is a client or relay link.
+   *
+   * We go: dir_conn -> linked_conn -> edge_conn -> on_circuit -> p_chan.
+   */
+
+  conn = TO_CONN(dir_conn);
+  linked_conn = conn->linked_conn;
+
+  /* The dir connection should be connected to an edge connection. It can not
+   * be closed or marked for close. */
+  if (linked_conn == NULL || linked_conn->magic != EDGE_CONNECTION_MAGIC ||
+      conn->linked_conn_is_closed || conn->linked_conn->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not linked to edge");
+    return false;
+  }
+
+  edge_conn = TO_EDGE_CONN((connection_t *) linked_conn);
+  circ = edge_conn->on_circuit;
+
+  /* Can't be a circuit we initiated and without a circuit, no channel. */
+  if (circ == NULL || CIRCUIT_IS_ORIGIN(circ)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not on OR circuit");
+    return false;
+  }
+
+  /* It is possible that the circuit was closed because one of the channel was
+   * closed or a DESTROY cell was received. Either way, this connection can
+   * not continue so return that it is not anonymous since we can not know for
+   * sure if it is. */
+  if (circ->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "circuit marked for close");
+    return false;
+  }
+
+  /* Get the previous channel to learn if it is a client or relay link. We
+   * BUG() because if the circuit is not mark for close, we ought to have a
+   * p_chan else we have a code flow issue. */
+  if (BUG(CONST_TO_OR_CIRCUIT(circ)->p_chan == NULL)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "no p_chan on circuit");
+    return false;
+  }
+
+  /* Will be true if the channel is an unauthenticated peer which is only true
+   * for clients and bridges. */
+  return !channel_is_client(CONST_TO_OR_CIRCUIT(circ)->p_chan);
+}
+
 /** Parse an HTTP request line at the start of a headers string.  On failure,
  * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
  * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
@@ -618,34 +702,3 @@ dir_split_resource_into_fingerprints(const char *resource,
   smartlist_free(fp_tmp);
   return 0;
 }
-
-/** As dir_split_resource_into_fingerprints, but instead fills
- * <b>spool_out</b> with a list of spoolable_resource_t for the resource
- * identified through <b>source</b>. */
-int
-dir_split_resource_into_spoolable(const char *resource,
-                                  dir_spool_source_t source,
-                                  smartlist_t *spool_out,
-                                  int *compressed_out,
-                                  int flags)
-{
-  smartlist_t *fingerprints = smartlist_new();
-
-  tor_assert(flags & (DSR_HEX|DSR_BASE64));
-  const size_t digest_len =
-    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
-
-  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
-                                               compressed_out, flags);
-  /* This is not a very efficient implementation XXXX */
-  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
-    spooled_resource_t *spooled =
-      spooled_resource_new(source, digest, digest_len);
-    if (spooled)
-      smartlist_add(spool_out, spooled);
-    tor_free(digest);
-  } SMARTLIST_FOREACH_END(digest);
-
-  smartlist_free(fingerprints);
-  return r;
-}
diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h
index ba3f8c1b0e..0f26cdeff9 100644
--- a/src/feature/dircommon/directory.h
+++ b/src/feature/dircommon/directory.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -94,6 +94,7 @@ int parse_http_command(const char *headers,
 char *http_get_header(const char *headers, const char *which);
 
 int connection_dir_is_encrypted(const dir_connection_t *conn);
+bool connection_dir_is_anonymous(const dir_connection_t *conn);
 int connection_dir_reached_eof(dir_connection_t *conn);
 int connection_dir_process_inbuf(dir_connection_t *conn);
 int connection_dir_finished_flushing(dir_connection_t *conn);
@@ -107,12 +108,6 @@ void connection_dir_about_to_close(dir_connection_t *dir_conn);
 int dir_split_resource_into_fingerprints(const char *resource,
                                      smartlist_t *fp_out, int *compressed_out,
                                      int flags);
-enum dir_spool_source_t;
-int dir_split_resource_into_spoolable(const char *resource,
-                                      enum dir_spool_source_t source,
-                                      smartlist_t *spool_out,
-                                      int *compressed_out,
-                                      int flags);
 int dir_split_resource_into_fingerprint_pairs(const char *res,
                                               smartlist_t *pairs_out);
 char *directory_dump_request_log(void);
diff --git a/src/feature/dircommon/feature_dircommon.md b/src/feature/dircommon/feature_dircommon.md
new file mode 100644
index 0000000000..359049ecd8
--- /dev/null
+++ b/src/feature/dircommon/feature_dircommon.md
@@ -0,0 +1,7 @@
+@dir /feature/dircommon
+@brief feature/dircommon: Directory client and server shared code
+
+This module has the code that directory clients (anybody who download
+information about relays) and directory servers (anybody who serves such
+information) share in common.
+
diff --git a/src/feature/dircommon/fp_pair.c b/src/feature/dircommon/fp_pair.c
index 284600df77..87e1c253bd 100644
--- a/src/feature/dircommon/fp_pair.c
+++ b/src/feature/dircommon/fp_pair.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,14 +22,14 @@
 
 /* Define fp_pair_map_t structures */
 
-struct fp_pair_map_entry_s {
-  HT_ENTRY(fp_pair_map_entry_s) node;
+struct fp_pair_map_entry_t {
+  HT_ENTRY(fp_pair_map_entry_t) node;
   void *val;
   fp_pair_t key;
 };
 
-struct fp_pair_map_s {
-  HT_HEAD(fp_pair_map_impl, fp_pair_map_entry_s) head;
+struct fp_pair_map_t {
+  HT_HEAD(fp_pair_map_impl, fp_pair_map_entry_t) head;
 };
 
 /*
@@ -56,11 +56,11 @@ fp_pair_map_entry_hash(const fp_pair_map_entry_t *a)
  * Hash table functions for fp_pair_map_t
  */
 
-HT_PROTOTYPE(fp_pair_map_impl, fp_pair_map_entry_s, node,
-             fp_pair_map_entry_hash, fp_pair_map_entries_eq)
-HT_GENERATE2(fp_pair_map_impl, fp_pair_map_entry_s, node,
+HT_PROTOTYPE(fp_pair_map_impl, fp_pair_map_entry_t, node,
+             fp_pair_map_entry_hash, fp_pair_map_entries_eq);
+HT_GENERATE2(fp_pair_map_impl, fp_pair_map_entry_t, node,
              fp_pair_map_entry_hash, fp_pair_map_entries_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 /** Constructor to create a new empty map from fp_pair_t to void *
  */
@@ -312,4 +312,3 @@ fp_pair_map_assert_ok(const fp_pair_map_t *map)
 {
   tor_assert(!fp_pair_map_impl_HT_REP_IS_BAD_(&(map->head)));
 }
-
diff --git a/src/feature/dircommon/fp_pair.h b/src/feature/dircommon/fp_pair.h
index 5041583e88..ae71ea7b71 100644
--- a/src/feature/dircommon/fp_pair.h
+++ b/src/feature/dircommon/fp_pair.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,8 +19,8 @@ typedef struct {
  * Declare fp_pair_map_t functions and structs
  */
 
-typedef struct fp_pair_map_entry_s fp_pair_map_entry_t;
-typedef struct fp_pair_map_s fp_pair_map_t;
+typedef struct fp_pair_map_entry_t fp_pair_map_entry_t;
+typedef struct fp_pair_map_t fp_pair_map_t;
 typedef fp_pair_map_entry_t *fp_pair_map_iter_t;
 
 fp_pair_map_t * fp_pair_map_new(void);
diff --git a/src/feature/dircommon/include.am b/src/feature/dircommon/include.am
new file mode 100644
index 0000000000..87850ce183
--- /dev/null
+++ b/src/feature/dircommon/include.am
@@ -0,0 +1,14 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/dircommon/consdiff.c	\
+	src/feature/dircommon/directory.c	\
+	src/feature/dircommon/fp_pair.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dircommon/consdiff.h		\
+	src/feature/dircommon/dir_connection_st.h	\
+	src/feature/dircommon/directory.h		\
+	src/feature/dircommon/fp_pair.h			\
+	src/feature/dircommon/vote_timing_st.h
diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h
index 47b90ab009..103d950f86 100644
--- a/src/feature/dircommon/vote_timing_st.h
+++ b/src/feature/dircommon/vote_timing_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file vote_timing_st.h
+ * @brief Directory voting schedule structure.
+ **/
+
 #ifndef VOTE_TIMING_ST_H
 #define VOTE_TIMING_ST_H
 
@@ -20,5 +25,4 @@ struct vote_timing_t {
   int dist_delay;
 };
 
-#endif
-
+#endif /* !defined(VOTE_TIMING_ST_H) */
diff --git a/src/feature/dirparse/.may_include b/src/feature/dirparse/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dirparse/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dirparse/authcert_members.h b/src/feature/dirparse/authcert_members.h
new file mode 100644
index 0000000000..53eab175d6
--- /dev/null
+++ b/src/feature/dirparse/authcert_members.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file authcert_members.h
+ *
+ * @brief List of tokens common to V3 authority certificates and V3
+ * consensuses.
+ **/
+
+#ifndef TOR_AUTHCERT_MEMBERS_H
+#define TOR_AUTHCERT_MEMBERS_H
+
+// clang-format off
+#define AUTHCERT_MEMBERS                                           \
+  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,      \
+                                                     GE(1),       NO_OBJ ), \
+  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),\
+  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),\
+  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),\
+  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,\
+                                                     NO_ARGS,     NEED_OBJ),\
+  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ)
+// clang-format on
+
+#endif /* !defined(TOR_AUTHCERT_MEMBERS_H) */
diff --git a/src/feature/dirparse/authcert_members.i b/src/feature/dirparse/authcert_members.i
deleted file mode 100644
index 08cffca97a..0000000000
--- a/src/feature/dirparse/authcert_members.i
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * List of tokens common to V3 authority certificates and V3 consensuses.
- */
-  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,
-                                                     GE(1),       NO_OBJ ),
-  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),
-  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),
-  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),
-  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),
-  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),
-  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,
-                                                     NO_ARGS,     NEED_OBJ),
-  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ),
diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c
index 1680bdbf30..deb45c12de 100644
--- a/src/feature/dirparse/authcert_parse.c
+++ b/src/feature/dirparse/authcert_parse.c
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file authcert_parse.c
+ * @brief Authority certificate parsing.
+ **/
+
 #include "core/or/or.h"
 #include "feature/dirparse/authcert_parse.h"
 #include "feature/dirparse/parsecommon.h"
@@ -13,18 +18,22 @@
 #include "lib/memarea/memarea.h"
 
 #include "feature/nodelist/authority_cert_st.h"
+#include "feature/dirparse/authcert_members.h"
 
 /** List of tokens recognized in V3 authority certificates. */
+// clang-format off
 static token_rule_t dir_key_certificate_table[] = {
-#include "feature/dirparse/authcert_members.i"
+  AUTHCERT_MEMBERS,
   T1("fingerprint",      K_FINGERPRINT,              CONCAT_ARGS, NO_OBJ ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
  * the first character after the certificate. */
 authority_cert_t *
-authority_cert_parse_from_string(const char *s, const char **end_of_string)
+authority_cert_parse_from_string(const char *s, size_t maxlen,
+                                 const char **end_of_string)
 {
   /** Reject any certificate at least this big; it is probably an overflow, an
    * attack, a bug, or some other nonsense. */
@@ -35,24 +44,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
   char digest[DIGEST_LEN];
   directory_token_t *tok;
   char fp_declared[DIGEST_LEN];
-  char *eos;
+  const char *eos;
   size_t len;
   int found;
   memarea_t *area = NULL;
+  const char *end_of_s = s + maxlen;
   const char *s_dup = s;
 
-  s = eat_whitespace(s);
-  eos = strstr(s, "\ndir-key-certification");
+  s = eat_whitespace_eos(s, end_of_s);
+  eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification");
   if (! eos) {
     log_warn(LD_DIR, "No signature found on key certificate");
     return NULL;
   }
-  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
+  eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n");
   if (! eos) {
     log_warn(LD_DIR, "No end-of-signature found on key certificate");
     return NULL;
   }
-  eos = strchr(eos+2, '\n');
+  eos = memchr(eos+2, '\n', end_of_s - (eos+2));
   tor_assert(eos);
   ++eos;
   len = eos - s;
@@ -69,7 +79,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
     log_warn(LD_DIR, "Error tokenizing key certificate");
     goto err;
   }
-  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
+  if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version",
                            "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
     goto err;
   tok = smartlist_get(tokens, 0);
diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h
index ca475ad0e3..7f6dd1c02f 100644
--- a/src/feature/dirparse/authcert_parse.h
+++ b/src/feature/dirparse/authcert_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_AUTHCERT_PARSE_H
 
 authority_cert_t *authority_cert_parse_from_string(const char *s,
+                                                   size_t maxlen,
                                                    const char **end_of_string);
 
 #endif /* !defined(TOR_AUTHCERT_PARSE_H) */
diff --git a/src/feature/dirparse/feature_dirparse.md b/src/feature/dirparse/feature_dirparse.md
new file mode 100644
index 0000000000..e4b34668ba
--- /dev/null
+++ b/src/feature/dirparse/feature_dirparse.md
@@ -0,0 +1,8 @@
+@dir /feature/dirparse
+@brief feature/dirparse: Parsing Tor directory objects
+
+We define a number of "directory objects" in
+[dir-spec.txt](https://gitweb.torproject.org/torspec.git/tree/dir-spec.txt),
+all of them using a common line-oriented meta-format.  This module is used by
+other parts of Tor to parse them.
+
diff --git a/src/feature/dirparse/include.am b/src/feature/dirparse/include.am
new file mode 100644
index 0000000000..edca04f6f7
--- /dev/null
+++ b/src/feature/dirparse/include.am
@@ -0,0 +1,25 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/dirparse/authcert_parse.c	\
+	src/feature/dirparse/microdesc_parse.c	\
+	src/feature/dirparse/ns_parse.c		\
+	src/feature/dirparse/parsecommon.c	\
+	src/feature/dirparse/policy_parse.c	\
+	src/feature/dirparse/routerparse.c	\
+	src/feature/dirparse/sigcommon.c	\
+	src/feature/dirparse/signing.c		\
+	src/feature/dirparse/unparseable.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dirparse/authcert_members.h		\
+	src/feature/dirparse/authcert_parse.h		\
+	src/feature/dirparse/microdesc_parse.h		\
+	src/feature/dirparse/ns_parse.h			\
+	src/feature/dirparse/parsecommon.h		\
+	src/feature/dirparse/policy_parse.h		\
+	src/feature/dirparse/routerparse.h		\
+	src/feature/dirparse/sigcommon.h		\
+	src/feature/dirparse/signing.h			\
+	src/feature/dirparse/unparseable.h
diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c
index 5a75af3994..9231080aaa 100644
--- a/src/feature/dirparse/microdesc_parse.c
+++ b/src/feature/dirparse/microdesc_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/relay/router.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -27,17 +28,19 @@
 #include "feature/nodelist/microdesc_st.h"
 
 /** List of tokens recognized in microdescriptors */
+// clang-format off
 static token_rule_t microdesc_token_table[] = {
   T1_START("onion-key",        K_ONION_KEY,        NO_ARGS,     NEED_KEY_1024),
   T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
   T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
   T0N("a",                     K_A,                GE(1),       NO_OBJ ),
-  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("family",                K_FAMILY,           CONCAT_ARGS, NO_OBJ ),
   T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
   T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
   A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** Assuming that s starts with a microdesc, return the start of the
  * *NEXT* one.  Return NULL on "not found." */
@@ -91,6 +94,190 @@ find_start_of_next_microdesc(const char *s, const char *eos)
 #undef NEXT_LINE
 }
 
+static inline int
+policy_is_reject_star_or_null(struct short_policy_t *policy)
+{
+  return !policy || short_policy_is_reject_star(policy);
+}
+
+/**
+ * Return a human-readable description of a given saved_location_t.
+ * Never returns NULL.
+ **/
+static const char *
+saved_location_to_string(saved_location_t where)
+{
+  const char *location;
+  switch (where) {
+    case SAVED_NOWHERE:
+      location = "download or generated string";
+      break;
+    case SAVED_IN_CACHE:
+      location = "cache";
+      break;
+    case SAVED_IN_JOURNAL:
+      location = "journal";
+      break;
+    default:
+      location = "unknown location";
+      break;
+  }
+  return location;
+}
+
+/**
+ * Given a microdescriptor stored in <b>where</b> which starts at <b>s</b>,
+ * which ends at <b>start_of_next_microdescriptor</b>, and which is located
+ * within a larger document beginning at <b>start</b>: Fill in the body,
+ * bodylen, bodylen, saved_location, off, and digest fields of <b>md</b> as
+ * appropriate.
+ *
+ * The body field will be an alias within <b>s</b> if <b>saved_location</b>
+ * is SAVED_IN_CACHE, and will be copied into body and nul-terminated
+ * otherwise.
+ **/
+static int
+microdesc_extract_body(microdesc_t *md,
+                       const char *start,
+                       const char *s, const char *start_of_next_microdesc,
+                       saved_location_t where)
+{
+  const bool copy_body = (where != SAVED_IN_CACHE);
+
+  const char *cp = tor_memstr(s, start_of_next_microdesc-s, "onion-key");
+
+  const bool no_onion_key = (cp == NULL);
+  if (no_onion_key) {
+    cp = s; /* So that we have *some* junk to put in the body */
+  }
+
+  md->bodylen = start_of_next_microdesc - cp;
+  md->saved_location = where;
+  if (copy_body)
+    md->body = tor_memdup_nulterm(cp, md->bodylen);
+  else
+    md->body = (char*)cp;
+  md->off = cp - start;
+
+  crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+
+  return no_onion_key ? -1 : 0;
+}
+
+/**
+ * Parse a microdescriptor which begins at <b>s</b> and ends at
+ * <b>start_of_next_microdesc</b>.  Store its fields into <b>md</b>.  Use
+ * <b>where</b> for generating log information.  If <b>allow_annotations</b>
+ * is true, then one or more annotations may precede the microdescriptor body
+ * proper.  Use <b>area</b> for memory management, clearing it when done.
+ *
+ * On success, return 0; otherwise return -1.
+ **/
+static int
+microdesc_parse_fields(microdesc_t *md,
+                       memarea_t *area,
+                       const char *s, const char *start_of_next_microdesc,
+                       int allow_annotations,
+                       saved_location_t where)
+{
+  smartlist_t *tokens = smartlist_new();
+  int rv = -1;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+  directory_token_t *tok;
+
+  if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                      microdesc_token_table, flags)) {
+    log_warn(LD_DIR, "Unparseable microdescriptor found in %s",
+             saved_location_to_string(where));
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+    if (parse_iso_time(tok->args[0], &md->last_listed)) {
+      log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+      goto err;
+    }
+  }
+
+  tok = find_by_keyword(tokens, K_ONION_KEY);
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_DIR,
+             "Relay's onion key had invalid exponent.");
+    goto err;
+  }
+  md->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  md->onion_pkey_len = tok->object_size;
+  crypto_pk_free(tok->key);
+
+  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+    curve25519_public_key_t k;
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
+      goto err;
+    }
+    md->onion_curve25519_pkey =
+      tor_memdup(&k, sizeof(curve25519_public_key_t));
+  }
+
+  smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
+  if (id_lines) {
+    SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
+      tor_assert(t->n_args >= 2);
+      if (!strcmp(t->args[0], "ed25519")) {
+        if (md->ed25519_identity_pkey) {
+          log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        ed25519_public_key_t k;
+        if (ed25519_public_from_base64(&k, t->args[1])<0) {
+          log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
+      }
+    } SMARTLIST_FOREACH_END(t);
+    smartlist_free(id_lines);
+  }
+
+  {
+    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+    if (a_lines) {
+      find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
+      smartlist_free(a_lines);
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+    md->family = nodefamily_parse(tok->args[0],
+                                  NULL,
+                                  NF_WARN_MALFORMED);
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_P))) {
+    md->exit_policy = parse_short_policy(tok->args[0]);
+  }
+  if ((tok = find_opt_by_keyword(tokens, K_P6))) {
+    md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+  }
+
+  if (policy_is_reject_star_or_null(md->exit_policy) &&
+      policy_is_reject_star_or_null(md->ipv6_exit_policy)) {
+    md->policy_is_reject_star = 1;
+  }
+
+  rv = 0;
+ err:
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  memarea_clear(area);
+  smartlist_free(tokens);
+
+  return rv;
+}
+
 /** Parse as many microdescriptors as are found from the string starting at
  * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
  * annotations we recognize and ignore ones we don't.
@@ -108,16 +295,11 @@ microdescs_parse_from_string(const char *s, const char *eos,
                              saved_location_t where,
                              smartlist_t *invalid_digests_out)
 {
-  smartlist_t *tokens;
   smartlist_t *result;
   microdesc_t *md = NULL;
   memarea_t *area;
   const char *start = s;
   const char *start_of_next_microdesc;
-  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
-  const int copy_body = (where != SAVED_IN_CACHE);
-
-  directory_token_t *tok;
 
   if (!eos)
     eos = s + strlen(s);
@@ -125,143 +307,47 @@ microdescs_parse_from_string(const char *s, const char *eos,
   s = eat_whitespace_eos(s, eos);
   area = memarea_new();
   result = smartlist_new();
-  tokens = smartlist_new();
 
   while (s < eos) {
-    int okay = 0;
+   bool okay = false;
 
     start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
     if (!start_of_next_microdesc)
       start_of_next_microdesc = eos;
 
     md = tor_malloc_zero(sizeof(microdesc_t));
+    uint8_t md_digest[DIGEST256_LEN];
     {
-      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
-                                  "onion-key");
-      const int no_onion_key = (cp == NULL);
-      if (no_onion_key) {
-        cp = s; /* So that we have *some* junk to put in the body */
-      }
+      const bool body_not_found =
+        microdesc_extract_body(md, start, s,
+                               start_of_next_microdesc,
+                               where) < 0;
 
-      md->bodylen = start_of_next_microdesc - cp;
-      md->saved_location = where;
-      if (copy_body)
-        md->body = tor_memdup_nulterm(cp, md->bodylen);
-      else
-        md->body = (char*)cp;
-      md->off = cp - start;
-      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
-      if (no_onion_key) {
+      memcpy(md_digest, md->digest, DIGEST256_LEN);
+      if (body_not_found) {
         log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
         goto next;
       }
     }
 
-    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
-                        microdesc_token_table, flags)) {
-      log_warn(LD_DIR, "Unparseable microdescriptor");
-      goto next;
+    if (microdesc_parse_fields(md, area, s, start_of_next_microdesc,
+                               allow_annotations, where) == 0) {
+      smartlist_add(result, md);
+      md = NULL; // prevent free
+      okay = true;
     }
 
-    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
-      if (parse_iso_time(tok->args[0], &md->last_listed)) {
-        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
-        goto next;
-      }
-    }
-
-    tok = find_by_keyword(tokens, K_ONION_KEY);
-    if (!crypto_pk_public_exponent_ok(tok->key)) {
-      log_warn(LD_DIR,
-               "Relay's onion key had invalid exponent.");
-      goto next;
-    }
-    router_set_rsa_onion_pkey(tok->key, &md->onion_pkey,
-                              &md->onion_pkey_len);
-    crypto_pk_free(tok->key);
-
-    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
-      curve25519_public_key_t k;
-      tor_assert(tok->n_args >= 1);
-      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
-        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
-        goto next;
-      }
-      md->onion_curve25519_pkey =
-        tor_memdup(&k, sizeof(curve25519_public_key_t));
-    }
-
-    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
-    if (id_lines) {
-      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
-        tor_assert(t->n_args >= 2);
-        if (!strcmp(t->args[0], "ed25519")) {
-          if (md->ed25519_identity_pkey) {
-            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          ed25519_public_key_t k;
-          if (ed25519_public_from_base64(&k, t->args[1])<0) {
-            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
-        }
-      } SMARTLIST_FOREACH_END(t);
-      smartlist_free(id_lines);
-    }
-
-    {
-      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
-      if (a_lines) {
-        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
-        smartlist_free(a_lines);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
-      int i;
-      md->family = smartlist_new();
-      for (i=0;i<tok->n_args;++i) {
-        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
-          log_warn(LD_DIR, "Illegal nickname %s in family line",
-                   escaped(tok->args[i]));
-          goto next;
-        }
-        smartlist_add_strdup(md->family, tok->args[i]);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_P))) {
-      md->exit_policy = parse_short_policy(tok->args[0]);
-    }
-    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
-      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
-    }
-
-    smartlist_add(result, md);
-    okay = 1;
-
-    md = NULL;
   next:
     if (! okay && invalid_digests_out) {
       smartlist_add(invalid_digests_out,
-                    tor_memdup(md->digest, DIGEST256_LEN));
+                    tor_memdup(md_digest, DIGEST256_LEN));
     }
     microdesc_free(md);
     md = NULL;
-
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    memarea_clear(area);
-    smartlist_clear(tokens);
     s = start_of_next_microdesc;
   }
 
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
   memarea_drop_all(area);
-  smartlist_free(tokens);
 
   return result;
 }
diff --git a/src/feature/dirparse/microdesc_parse.h b/src/feature/dirparse/microdesc_parse.h
index 23a90084b1..e81126b8cd 100644
--- a/src/feature/dirparse/microdesc_parse.h
+++ b/src/feature/dirparse/microdesc_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@ smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
                                           saved_location_t where,
                                           smartlist_t *invalid_digests_out);
 
-#endif
+#endif /* !defined(TOR_MICRODESC_PARSE_H) */
diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c
index 109eebeb66..ac9325a608 100644
--- a/src/feature/dirparse/ns_parse.c
+++ b/src/feature/dirparse/ns_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,12 +36,14 @@
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/networkstatus_voter_info_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
+#include "feature/dirparse/authcert_members.h"
 
 #undef log
 #include <math.h>
 
 /** List of tokens recognized in the body part of v3 networkstatus
  * documents. */
+// clang-format off
 static token_rule_t rtrstatus_token_table[] = {
   T01("p",                   K_P,               CONCAT_ARGS, NO_OBJ ),
   T1( "r",                   K_R,                   GE(7),   NO_OBJ ),
@@ -55,8 +57,10 @@ static token_rule_t rtrstatus_token_table[] = {
   T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in V3 networkstatus votes. */
+// clang-format off
 static token_rule_t networkstatus_token_table[] = {
   T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
                                                    GE(1),       NO_OBJ ),
@@ -84,7 +88,7 @@ static token_rule_t networkstatus_token_table[] = {
   T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
       CONCAT_ARGS, NO_OBJ ),
 
-#include "feature/dirparse/authcert_members.i"
+  AUTHCERT_MEMBERS,
 
   T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
   T1( "contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
@@ -97,8 +101,10 @@ static token_rule_t networkstatus_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in V3 networkstatus consensuses. */
+// clang-format off
 static token_rule_t networkstatus_consensus_token_table[] = {
   T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
                                                    GE(1),       NO_OBJ ),
@@ -135,14 +141,17 @@ static token_rule_t networkstatus_consensus_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in the footer of v1 directory footers. */
+// clang-format off
 static token_rule_t networkstatus_vote_footer_token_table[] = {
   T01("directory-footer",    K_DIRECTORY_FOOTER,    NO_ARGS,   NO_OBJ ),
   T01("bandwidth-weights",   K_BW_WEIGHTS,          ARGS,      NO_OBJ ),
   T(  "directory-signature", K_DIRECTORY_SIGNATURE, GE(2),     NEED_OBJ ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** Try to find the start and end of the signed portion of a networkstatus
  * document in <b>s</b>. On success, set <b>start_out</b> to the first
@@ -151,10 +160,11 @@ static token_rule_t networkstatus_vote_footer_token_table[] = {
  * -1. */
 int
 router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              size_t len,
                                               const char **start_out,
                                               const char **end_out)
 {
-  return router_get_hash_impl_helper(s, strlen(s),
+  return router_get_hash_impl_helper(s, len,
                                      "network-status-version",
                                      "\ndirectory-signature",
                                      ' ', LOG_INFO,
@@ -166,12 +176,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s,
  * signed portion can be identified.  Return 0 on success, -1 on failure. */
 int
 router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                           const char *s)
+                                           const char *s, size_t len)
 {
   const char *start, *end;
-  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+  if (router_get_networkstatus_v3_signed_boundaries(s, len,
+                                                    &start, &end) < 0) {
     start = s;
-    end = s + strlen(s);
+    end = s + len;
   }
   tor_assert(start);
   tor_assert(end);
@@ -182,9 +193,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
 /** Set <b>digests</b> to all the digests of the consensus document in
  * <b>s</b> */
 int
-router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
+router_get_networkstatus_v3_hashes(const char *s, size_t len,
+                                   common_digests_t *digests)
 {
-  return router_get_hashes_impl(s,strlen(s),digests,
+  return router_get_hashes_impl(s, len, digests,
                                 "network-status-version",
                                 "\ndirectory-signature",
                                 ' ');
@@ -195,13 +207,13 @@ router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
  * return the start of the directory footer, or the next directory signature.
  * If none is found, return the end of the string. */
 static inline const char *
-find_start_of_next_routerstatus(const char *s)
+find_start_of_next_routerstatus(const char *s, const char *s_eos)
 {
   const char *eos, *footer, *sig;
-  if ((eos = strstr(s, "\nr ")))
+  if ((eos = tor_memstr(s, s_eos - s, "\nr ")))
     ++eos;
   else
-    eos = s + strlen(s);
+    eos = s_eos;
 
   footer = tor_memstr(s, eos-s, "\ndirectory-footer");
   sig = tor_memstr(s, eos-s, "\ndirectory-signature");
@@ -289,7 +301,8 @@ routerstatus_parse_guardfraction(const char *guardfraction_str,
  **/
 STATIC routerstatus_t *
 routerstatus_parse_entry_from_string(memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *s_eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
@@ -308,7 +321,7 @@ routerstatus_parse_entry_from_string(memarea_t *area,
     flav = FLAV_NS;
   tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
 
-  eos = find_start_of_next_routerstatus(*s);
+  eos = find_start_of_next_routerstatus(*s, s_eos);
 
   if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing router status");
@@ -430,6 +443,8 @@ routerstatus_parse_entry_from_string(memarea_t *area,
         rs->is_hs_dir = 1;
       } else if (!strcmp(tok->args[i], "V2Dir")) {
         rs->is_v2_dir = 1;
+      } else if (!strcmp(tok->args[i], "StaleDesc")) {
+        rs->is_staledesc = 1;
       }
     }
     /* These are implied true by having been included in a consensus made
@@ -1051,7 +1066,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
 /** Parse a v3 networkstatus vote, opinion, or consensus (depending on
  * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
 networkstatus_t *
-networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
+networkstatus_parse_vote_from_string(const char *s,
+                                     size_t s_len,
+                                     const char **eos_out,
                                      networkstatus_type_t ns_type)
 {
   smartlist_t *tokens = smartlist_new();
@@ -1067,20 +1084,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   memarea_t *area = NULL, *rs_area = NULL;
   consensus_flavor_t flav = FLAV_NS;
   char *last_kwd=NULL;
+  const char *eos = s + s_len;
 
   tor_assert(s);
 
   if (eos_out)
     *eos_out = NULL;
 
-  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
-      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
+  if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed,
+                                                 s, s_len)<0) {
     log_warn(LD_DIR, "Unable to compute digest of network-status");
     goto err;
   }
 
   area = memarea_new();
-  end_of_header = find_start_of_next_routerstatus(s);
+  end_of_header = find_start_of_next_routerstatus(s, eos);
   if (tokenize_string(area, s, end_of_header, tokens,
                       (ns_type == NS_TYPE_CONSENSUS) ?
                       networkstatus_consensus_token_table :
@@ -1111,10 +1130,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   if (ns_type != NS_TYPE_CONSENSUS) {
     const char *end_of_cert = NULL;
-    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
+    if (!(cert = tor_memstr(s, end_of_header - s,
+                            "\ndir-key-certificate-version")))
       goto err;
     ++cert;
-    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
+    ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert,
+                                                &end_of_cert);
     if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
       goto err;
   }
@@ -1424,10 +1445,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   s = end_of_header;
   ns->routerstatus_list = smartlist_new();
 
-  while (!strcmpstart(s, "r ")) {
+  while (eos - s >= 2 && fast_memeq(s, "r ", 2)) {
     if (ns->type != NS_TYPE_CONSENSUS) {
       vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
+      if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns,
                                                rs, 0, 0)) {
         smartlist_add(ns->routerstatus_list, rs);
       } else {
@@ -1435,7 +1456,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
       }
     } else {
       routerstatus_t *rs;
-      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
+      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos,
+                                                     rs_tokens,
                                                      NULL, NULL,
                                                      ns->consensus_method,
                                                      flav))) {
@@ -1465,7 +1487,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
                             vrs) {
       if (! vrs->has_ed25519_listing ||
-          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
+          fast_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
         continue;
       if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
         log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
@@ -1480,10 +1502,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   /* Parse footer; check signature. */
   footer_tokens = smartlist_new();
-  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
+  if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version ")))
     ++end_of_footer;
   else
-    end_of_footer = s + strlen(s);
+    end_of_footer = eos;
   if (tokenize_string(area,s, end_of_footer, footer_tokens,
                       networkstatus_vote_footer_token_table, 0)) {
     log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h
index 10a6f9cefc..6a1ea85c92 100644
--- a/src/feature/dirparse/ns_parse.h
+++ b/src/feature/dirparse/ns_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,18 +12,19 @@
 #ifndef TOR_NS_PARSE_H
 #define TOR_NS_PARSE_H
 
-int router_get_networkstatus_v3_hashes(const char *s,
+int router_get_networkstatus_v3_hashes(const char *s, size_t len,
                                        common_digests_t *digests);
-int router_get_networkstatus_v3_signed_boundaries(const char *s,
+int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len,
                                                   const char **start_out,
                                                   const char **end_out);
 int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                               const char *s);
+                                               const char *s, size_t len);
 int compare_vote_routerstatus_entries(const void **_a, const void **_b);
 
 int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
 enum networkstatus_type_t;
 networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
+                                           size_t len,
                                            const char **eos_out,
                                            enum networkstatus_type_t ns_type);
 
@@ -35,11 +36,12 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
 struct memarea_t;
 STATIC routerstatus_t *routerstatus_parse_entry_from_string(
                                      struct memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
                                      consensus_flavor_t flav);
-#endif
+#endif /* defined(NS_PARSE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NS_PARSE_H) */
diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c
index 1664a77bbe..ab465c4d7f 100644
--- a/src/feature/dirparse/parsecommon.c
+++ b/src/feature/dirparse/parsecommon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,6 +15,7 @@
 #include "lib/string/printf.h"
 #include "lib/memarea/memarea.h"
 #include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/ctime/di_ops.h"
 
 #include <string.h>
 
@@ -169,7 +170,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok,
   char *cp = mem;
   int j = 0;
   char *args[MAX_ARGS];
-  memset(args, 0, sizeof(args));
   while (*cp) {
     if (j == MAX_ARGS)
       return -1;
@@ -251,6 +251,16 @@ token_check_object(memarea_t *area, const char *kwd,
   return tok;
 }
 
+/** Return true iff the <b>memlen</b>-byte chunk of memory at
+ * <b>memlen</b> is the same length as <b>token</b>, and their
+ * contents are equal. */
+static bool
+mem_eq_token(const void *mem, size_t memlen, const char *token)
+{
+  size_t len = strlen(token);
+  return memlen == len && fast_memeq(mem, token, len);
+}
+
 /** Helper function: read the next token from *s, advance *s to the end of the
  * token, and return the parsed token.  Parse *<b>s</b> according to the list
  * of tokens in <b>table</b>.
@@ -266,7 +276,7 @@ get_next_token(memarea_t *area,
    * attack, a bug, or some other nonsense. */
 #define MAX_LINE_LENGTH (128*1024)
 
-  const char *next, *eol, *obstart;
+  const char *next, *eol;
   size_t obname_len;
   int i;
   directory_token_t *tok;
@@ -290,7 +300,7 @@ get_next_token(memarea_t *area,
 
   next = find_whitespace_eos(*s, eol);
 
-  if (!strcmp_len(*s, "opt", next-*s)) {
+  if (mem_eq_token(*s, next-*s, "opt")) {
     /* Skip past an "opt" at the start of the line. */
     *s = eat_whitespace_eos_no_nl(next, eol);
     next = find_whitespace_eos(*s, eol);
@@ -301,7 +311,7 @@ get_next_token(memarea_t *area,
   /* Search the table for the appropriate entry.  (I tried a binary search
    * instead, but it wasn't any faster.) */
   for (i = 0; table[i].t ; ++i) {
-    if (!strcmp_len(*s, table[i].t, next-*s)) {
+    if (mem_eq_token(*s, next-*s, table[i].t)) {
       /* We've found the keyword. */
       kwd = table[i].t;
       tok->tp = table[i].v;
@@ -352,9 +362,8 @@ get_next_token(memarea_t *area,
   if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
     goto check_object;
 
-  obstart = *s; /* Set obstart to start of object spec */
   if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
-      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      !mem_eq_token(eol-5, 5, "-----") ||   /* nuls or invalid endings */
       (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
     RET_ERR("Malformed object: bad begin line");
   }
@@ -373,8 +382,8 @@ get_next_token(memarea_t *area,
     eol = eos;
   /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
   if ((size_t)(eol-next) != 9+obname_len+5 ||
-      strcmp_len(next+9, tok->object_type, obname_len) ||
-      strcmp_len(eol-5, "-----", 5)) {
+      !mem_eq_token(next+9, obname_len, tok->object_type) ||
+      !mem_eq_token(eol-5, 5, "-----")) {
     tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
              tok->object_type);
     ebuf[sizeof(ebuf)-1] = '\0';
@@ -383,28 +392,32 @@ get_next_token(memarea_t *area,
   if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
     RET_ERR("Couldn't parse object: missing footer or object much too big.");
 
+  {
+    int r;
+    size_t maxsize = base64_decode_maxsize(next-*s);
+    tok->object_body = ALLOC(maxsize);
+    r = base64_decode(tok->object_body, maxsize, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+
   if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
     if (o_syn != NEED_KEY && o_syn != NEED_KEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected public key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size);
+    if (! tok->key)
       RET_ERR("Couldn't parse public key.");
   } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
     if (o_syn != NEED_SKEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected private key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_private_key1024_from_string(tok->key,
-                                                   obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode_private(tok->object_body,
+                                             tok->object_size,
+                                             1024);
+    if (! tok->key)
       RET_ERR("Couldn't parse private key.");
-  } else { /* If it's something else, try to base64-decode it */
-    int r;
-    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
-    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
-    if (r<0)
-      RET_ERR("Malformed object: bad base64-encoded data");
-    tok->object_size = r;
   }
   *s = eol;
 
diff --git a/src/feature/dirparse/parsecommon.h b/src/feature/dirparse/parsecommon.h
index ef74925b26..4db9a89f13 100644
--- a/src/feature/dirparse/parsecommon.h
+++ b/src/feature/dirparse/parsecommon.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/policy_parse.c b/src/feature/dirparse/policy_parse.c
index 7562ae409b..28cd174686 100644
--- a/src/feature/dirparse/policy_parse.c
+++ b/src/feature/dirparse/policy_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,7 +9,7 @@
  * \brief Code to parse address policies.
  **/
 
-#define  EXPOSE_ROUTERDESC_TOKEN_TABLE
+#define  ROUTERDESC_TOKEN_TABLE_PRIVATE
 
 #include "core/or/or.h"
 
diff --git a/src/feature/dirparse/policy_parse.h b/src/feature/dirparse/policy_parse.h
index e09ee5559f..7764069e66 100644
--- a/src/feature/dirparse/policy_parse.h
+++ b/src/feature/dirparse/policy_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c
index e44fbf77f9..8828a0f97a 100644
--- a/src/feature/dirparse/routerparse.c
+++ b/src/feature/dirparse/routerparse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -50,7 +50,7 @@
  * </ul>
  **/
 
-#define EXPOSE_ROUTERDESC_TOKEN_TABLE
+#define ROUTERDESC_TOKEN_TABLE_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
@@ -81,6 +81,7 @@
 /****************************************************************************/
 
 /** List of tokens recognized in router descriptors */
+// clang-format off
 const token_rule_t routerdesc_token_table[] = {
   T0N("reject",              K_REJECT,              ARGS,    NO_OBJ ),
   T0N("accept",              K_ACCEPT,              ARGS,    NO_OBJ ),
@@ -123,8 +124,10 @@ const token_rule_t routerdesc_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in extra-info documents. */
+// clang-format off
 static token_rule_t extrainfo_token_table[] = {
   T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
   T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
@@ -162,6 +165,7 @@ static token_rule_t extrainfo_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 #undef T
 
@@ -591,8 +595,8 @@ router_parse_entry_from_string(const char *s, const char *end,
              "Relay's onion key had invalid exponent.");
     goto err;
   }
-  router_set_rsa_onion_pkey(tok->key, &router->onion_pkey,
-                            &router->onion_pkey_len);
+  router->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  router->onion_pkey_len = tok->object_size;
   crypto_pk_free(tok->key);
 
   if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
diff --git a/src/feature/dirparse/routerparse.h b/src/feature/dirparse/routerparse.h
index f9a13f2168..519044e9b0 100644
--- a/src/feature/dirparse/routerparse.h
+++ b/src/feature/dirparse/routerparse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,7 +40,8 @@ int find_single_ipv6_orport(const smartlist_t *list,
 void routerparse_init(void);
 void routerparse_free_all(void);
 
-#ifdef EXPOSE_ROUTERDESC_TOKEN_TABLE
+#ifdef ROUTERDESC_TOKEN_TABLE_PRIVATE
+#include "feature/dirparse/parsecommon.h"
 extern const struct token_rule_t routerdesc_token_table[];
 #endif
 
diff --git a/src/feature/dirparse/sigcommon.c b/src/feature/dirparse/sigcommon.c
index 2019e09918..8b970d7d1f 100644
--- a/src/feature/dirparse/sigcommon.c
+++ b/src/feature/dirparse/sigcommon.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/sigcommon.h b/src/feature/dirparse/sigcommon.h
index fdd8e839a9..c2ed9df494 100644
--- a/src/feature/dirparse/sigcommon.h
+++ b/src/feature/dirparse/sigcommon.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -43,6 +43,6 @@ MOCK_DECL(STATIC int, signed_digest_equals,
 MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
                            const char *start, size_t len,
                            digest_algorithm_t alg));
-#endif
+#endif /* defined(SIGCOMMON_PRIVATE) */
 
 #endif /* !defined(TOR_SIGCOMMON_H) */
diff --git a/src/feature/dirparse/signing.c b/src/feature/dirparse/signing.c
index 3ab40c3807..e420e5b6b9 100644
--- a/src/feature/dirparse/signing.c
+++ b/src/feature/dirparse/signing.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/signing.h b/src/feature/dirparse/signing.h
index 2e3699baf8..7ca34bb14a 100644
--- a/src/feature/dirparse/signing.h
+++ b/src/feature/dirparse/signing.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
                                    const char *digest,
                                    size_t digest_len,
                                    crypto_pk_t *private_key);
-#endif
+#endif /* !defined(TOR_SIGNING_H) */
diff --git a/src/feature/dirparse/unparseable.c b/src/feature/dirparse/unparseable.c
index 1d623fe701..a91148a661 100644
--- a/src/feature/dirparse/unparseable.c
+++ b/src/feature/dirparse/unparseable.c
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file unparseable.c
+ * @brief Dump unparseable objects to disk.
+ **/
+
 #define UNPARSEABLE_PRIVATE
 
 #include "core/or/or.h"
diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h
index 853fe8cb0f..cff91c82cc 100644
--- a/src/feature/dirparse/unparseable.h
+++ b/src/feature/dirparse/unparseable.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,7 +26,7 @@ void dump_desc_init(void);
   log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
             name, (unsigned long)alloc, (unsigned long)used);     \
   STMT_END
-#else /* !(defined(DEBUG_AREA_ALLOC)) */
+#else /* !defined(DEBUG_AREA_ALLOC) */
 #define DUMP_AREA(a,name) STMT_NIL
 #endif /* defined(DEBUG_AREA_ALLOC) */
 
@@ -51,6 +51,6 @@ EXTERN(struct smartlist_t *, descs_dumped)
 MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
     (const char *dirname, const char *f));
 STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-#endif
+#endif /* defined(UNPARSEABLE_PRIVATE) */
 
 #endif /* !defined(TOR_UNPARSEABLE_H) */
diff --git a/src/feature/feature.md b/src/feature/feature.md
new file mode 100644
index 0000000000..acc3487e55
--- /dev/null
+++ b/src/feature/feature.md
@@ -0,0 +1,7 @@
+@dir /feature
+@brief feature: domain-specific modules
+
+The "feature" directory has modules that Tor uses only for a particular
+role or service, such as maintaining/using an onion service, operating as a
+relay or a client, or being a directory authority.
+
diff --git a/src/feature/hibernate/.may_include b/src/feature/hibernate/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/hibernate/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/hibernate/feature_hibernate.md b/src/feature/hibernate/feature_hibernate.md
new file mode 100644
index 0000000000..0eb5ffea0d
--- /dev/null
+++ b/src/feature/hibernate/feature_hibernate.md
@@ -0,0 +1,14 @@
+@dir /feature/hibernate
+@brief feature/hibernate: Bandwidth accounting and hibernation (!)
+
+This module implements two features that are only somewhat related, and
+should probably be separated in the future.  One feature is bandwidth
+accounting (making sure we use no more than so many gigabytes in a day) and
+hibernation (avoiding network activity while we have used up all/most of our
+configured gigabytes).  The other feature is clean shutdown, where we stop
+accepting new connections for a while and give the old ones time to close.
+
+The two features are related only in the sense that "soft hibernation" (being
+almost out of ) is very close to the "shutting down" state.  But it would be
+better in the long run to make the two completely separate.
+
diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c
index 09932c97ac..82c33659aa 100644
--- a/src/feature/hibernate/hibernate.c
+++ b/src/feature/hibernate/hibernate.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,8 +35,9 @@ hibernating, phase 2:
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "feature/hibernate/hibernate.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/relay/router.h"
@@ -56,7 +57,7 @@ hibernating, phase 2:
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-# endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
@@ -66,8 +67,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL;
 /** If are hibernating, when do we plan to wake up? Set to 0 if we
  * aren't hibernating. */
 static time_t hibernate_end_time = 0;
-/** If we are shutting down, when do we plan finally exit? Set to 0 if
- * we aren't shutting down. */
+/** If we are shutting down, when do we plan to finally exit? Set to 0 if we
+ * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed
+ * to happen from mainloop_schedule_shutdown() now.) */
 static time_t shutdown_time = 0;
 
 /** A timed event that we'll use when it's time to wake up from
@@ -560,7 +562,7 @@ time_to_record_bandwidth_usage(time_t now)
   /* Note every 600 sec */
 #define NOTE_INTERVAL (600)
   /* Or every 20 megabytes */
-#define NOTE_BYTES 20*(1024*1024)
+#define NOTE_BYTES (20*1024*1024)
   static uint64_t last_read_bytes_noted = 0;
   static uint64_t last_written_bytes_noted = 0;
   static time_t last_time_noted = 0;
@@ -813,7 +815,7 @@ hibernate_soft_limit_reached(void)
    * We want to stop accepting connections when ALL of the following are true:
    *   - We expect to use up the remaining bytes in under 3 hours
    *   - We have used up 95% of our bytes.
-   *   - We have less than 500MB of bytes left.
+   *   - We have less than 500MBytes left.
    */
   uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT);
   if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) {
@@ -831,8 +833,6 @@ hibernate_soft_limit_reached(void)
   return get_accounting_bytes() >= soft_limit;
 }
 
-#define TOR_USEC_PER_SEC (1000000)
-
 /** Called when we get a SIGINT, or when bandwidth soft limit is
  * reached. Puts us into "loose hibernation": we don't accept new
  * connections, but we continue handling old ones. */
@@ -867,7 +867,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
     log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new "
                "connections, and will shut down in %d seconds. Interrupt "
                "again to exit now.", options->ShutdownWaitLength);
-    shutdown_time = time(NULL) + options->ShutdownWaitLength;
+    /* We add an arbitrary delay here so that even if something goes wrong
+     * with the mainloop shutdown code, we can still shutdown from
+     * consider_hibernation() if we call it... but so that the
+     * mainloop_schedule_shutdown() mechanism will be the first one called.
+     */
+    shutdown_time = time(NULL) + options->ShutdownWaitLength + 5;
+    mainloop_schedule_shutdown(options->ShutdownWaitLength);
 #ifdef HAVE_SYSTEMD
     /* tell systemd that we may need more than the default 90 seconds to shut
      * down so they don't kill us. add some extra time to actually finish
@@ -887,7 +893,7 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
      */
     sd_notifyf(0, "EXTEND_TIMEOUT_USEC=%" PRIu64,
             ((uint64_t)(options->ShutdownWaitLength) + 30) * TOR_USEC_PER_SEC);
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
   } else { /* soft limit reached */
     hibernate_end_time = interval_end_time;
   }
@@ -1096,11 +1102,12 @@ consider_hibernation(time_t now)
   hibernate_state_t prev_state = hibernate_state;
 
   /* If we're in 'exiting' mode, then we just shut down after the interval
-   * elapses. */
+   * elapses.  The mainloop was supposed to catch this via
+   * mainloop_schedule_shutdown(), but apparently it didn't. */
   if (hibernate_state == HIBERNATE_STATE_EXITING) {
     tor_assert(shutdown_time);
     if (shutdown_time <= now) {
-      log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+      log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting.");
       tor_shutdown_event_loop_and_exit(0);
     }
     return; /* if exiting soon, don't worry about bandwidth limits */
@@ -1112,7 +1119,7 @@ consider_hibernation(time_t now)
     if (hibernate_end_time > now && accounting_enabled) {
       /* If we're hibernating, don't wake up until it's time, regardless of
        * whether we're in a new interval. */
-      return ;
+      return;
     } else {
       hibernate_end_time_elapsed(now);
     }
@@ -1240,8 +1247,6 @@ on_hibernate_state_change(hibernate_state_t prev_state)
   if (prev_state != HIBERNATE_STATE_INITIAL) {
     rescan_periodic_events(get_options());
   }
-
-  reschedule_per_second_timer();
 }
 
 /** Free all resources held by the accounting module */
diff --git a/src/feature/hibernate/hibernate.h b/src/feature/hibernate/hibernate.h
index 3309ef0ce3..2383658b20 100644
--- a/src/feature/hibernate/hibernate.h
+++ b/src/feature/hibernate/hibernate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,6 +32,7 @@ int getinfo_helper_accounting(control_connection_t *conn,
                               const char **errmsg);
 uint64_t get_accounting_max_total(void);
 void accounting_free_all(void);
+bool accounting_tor_is_dormant(void);
 
 #ifdef HIBERNATE_PRIVATE
 /** Possible values of hibernate_state */
diff --git a/src/feature/hibernate/include.am b/src/feature/hibernate/include.am
new file mode 100644
index 0000000000..355e591392
--- /dev/null
+++ b/src/feature/hibernate/include.am
@@ -0,0 +1,8 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/hibernate/hibernate.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/hibernate/hibernate.h
diff --git a/src/feature/hs/.may_include b/src/feature/hs/.may_include
new file mode 100644
index 0000000000..11c5ffbb14
--- /dev/null
+++ b/src/feature/hs/.may_include
@@ -0,0 +1,2 @@
+*.h
+*.inc
diff --git a/src/feature/hs/feature_hs.md b/src/feature/hs/feature_hs.md
new file mode 100644
index 0000000000..299d07e014
--- /dev/null
+++ b/src/feature/hs/feature_hs.md
@@ -0,0 +1,8 @@
+@dir /feature/hs
+@brief feature/hs: v3 (current) onion service protocol
+
+This directory implements the v3 onion service protocol,
+as specified in
+[rend-spec-v3.txt](https://gitweb.torproject.org/torspec.git/tree/rend-spec-v3.txt).
+
+
diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c
index 042ec55fa4..ef5e88e947 100644
--- a/src/feature/hs/hs_cache.c
+++ b/src/feature/hs/hs_cache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,12 +28,27 @@
 static int cached_client_descriptor_has_expired(time_t now,
            const hs_cache_client_descriptor_t *cached_desc);
 
+/** Helper function: Return true iff the cache entry has a decrypted
+ * descriptor.
+ *
+ * A NULL desc object in the entry means that we were not able to decrypt the
+ * descriptor because we are likely lacking client authorization.  It is still
+ * a valid entry but some operations can't be done without the decrypted
+ * descriptor thus this function MUST be used to safe guard access to the
+ * decrypted desc object. */
+static inline bool
+entry_has_decrypted_descriptor(const hs_cache_client_descriptor_t *entry)
+{
+  tor_assert(entry);
+  return (entry->desc != NULL);
+}
+
 /********************** Directory HS cache ******************/
 
-/* Directory descriptor cache. Map indexed by blinded key. */
+/** Directory descriptor cache. Map indexed by blinded key. */
 static digest256map_t *hs_cache_v3_dir;
 
-/* Remove a given descriptor from our cache. */
+/** Remove a given descriptor from our cache. */
 static void
 remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
 {
@@ -41,7 +56,7 @@ remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
   digest256map_remove(hs_cache_v3_dir, desc->key);
 }
 
-/* Store a given descriptor in our cache. */
+/** Store a given descriptor in our cache. */
 static void
 store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
 {
@@ -49,7 +64,7 @@ store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
   digest256map_set(hs_cache_v3_dir, desc->key, desc);
 }
 
-/* Query our cache and return the entry or NULL if not found. */
+/** Query our cache and return the entry or NULL if not found. */
 static hs_cache_dir_descriptor_t *
 lookup_v3_desc_as_dir(const uint8_t *key)
 {
@@ -60,7 +75,7 @@ lookup_v3_desc_as_dir(const uint8_t *key)
 #define cache_dir_desc_free(val) \
   FREE_AND_NULL(hs_cache_dir_descriptor_t, cache_dir_desc_free_, (val))
 
-/* Free a directory descriptor object. */
+/** Free a directory descriptor object. */
 static void
 cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
 {
@@ -72,7 +87,7 @@ cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
   tor_free(desc);
 }
 
-/* Helper function: Use by the free all function using the digest256map
+/** Helper function: Use by the free all function using the digest256map
  * interface to cache entries. */
 static void
 cache_dir_desc_free_void(void *ptr)
@@ -80,7 +95,7 @@ cache_dir_desc_free_void(void *ptr)
   cache_dir_desc_free_(ptr);
 }
 
-/* Create a new directory cache descriptor object from a encoded descriptor.
+/** Create a new directory cache descriptor object from a encoded descriptor.
  * On success, return the heap-allocated cache object, otherwise return NULL if
  * we can't decode the descriptor. */
 static hs_cache_dir_descriptor_t *
@@ -110,7 +125,7 @@ cache_dir_desc_new(const char *desc)
   return NULL;
 }
 
-/* Return the size of a cache entry in bytes. */
+/** Return the size of a cache entry in bytes. */
 static size_t
 cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
 {
@@ -118,7 +133,7 @@ cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
           + strlen(entry->encoded_desc));
 }
 
-/* Try to store a valid version 3 descriptor in the directory cache. Return 0
+/** Try to store a valid version 3 descriptor in the directory cache. Return 0
  * on success else a negative value is returned indicating that we have a
  * newer version in our cache. On error, caller is responsible to free the
  * given descriptor desc. */
@@ -168,7 +183,7 @@ cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)
   return -1;
 }
 
-/* Using the query which is the base64 encoded blinded key of a version 3
+/** Using the query which is the base64 encoded blinded key of a version 3
  * descriptor, lookup in our directory cache the entry. If found, 1 is
  * returned and desc_out is populated with a newly allocated string being the
  * encoded descriptor. If not found, 0 is returned and desc_out is untouched.
@@ -203,7 +218,7 @@ cache_lookup_v3_as_dir(const char *query, const char **desc_out)
   return -1;
 }
 
-/* Clean the v3 cache by removing any entry that has expired using the
+/** Clean the v3 cache by removing any entry that has expired using the
  * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning
  * process will use the lifetime found in the plaintext data section. Return
  * the number of bytes cleaned. */
@@ -253,7 +268,7 @@ cache_clean_v3_as_dir(time_t now, time_t global_cutoff)
   return bytes_removed;
 }
 
-/* Given an encoded descriptor, store it in the directory cache depending on
+/** Given an encoded descriptor, store it in the directory cache depending on
  * which version it is. Return a negative value on error. On success, 0 is
  * returned. */
 int
@@ -288,7 +303,7 @@ hs_cache_store_as_dir(const char *desc)
   return -1;
 }
 
-/* Using the query, lookup in our directory cache the entry. If found, 1 is
+/** Using the query, lookup in our directory cache the entry. If found, 1 is
  * returned and desc_out is populated with a newly allocated string being
  * the encoded descriptor. If not found, 0 is returned and desc_out is
  * untouched. On error, a negative value is returned and desc_out is
@@ -313,7 +328,7 @@ hs_cache_lookup_as_dir(uint32_t version, const char *query,
   return found;
 }
 
-/* Clean all directory caches using the current time now. */
+/** Clean all directory caches using the current time now. */
 void
 hs_cache_clean_as_dir(time_t now)
 {
@@ -330,23 +345,38 @@ hs_cache_clean_as_dir(time_t now)
 
 /********************** Client-side HS cache ******************/
 
-/* Client-side HS descriptor cache. Map indexed by service identity key. */
+/** Client-side HS descriptor cache. Map indexed by service identity key. */
 static digest256map_t *hs_cache_v3_client;
 
-/* Client-side introduction point state cache. Map indexed by service public
+/** Client-side introduction point state cache. Map indexed by service public
  * identity key (onion address). It contains hs_cache_client_intro_state_t
  * objects all related to a specific service. */
 static digest256map_t *hs_cache_client_intro_state;
 
-/* Return the size of a client cache entry in bytes. */
+/** Return the size of a client cache entry in bytes. */
 static size_t
 cache_get_client_entry_size(const hs_cache_client_descriptor_t *entry)
 {
-  return sizeof(*entry) +
-         strlen(entry->encoded_desc) + hs_desc_obj_size(entry->desc);
+  size_t size = 0;
+
+  if (entry == NULL) {
+    goto end;
+  }
+  size += sizeof(*entry);
+
+  if (entry->encoded_desc) {
+    size += strlen(entry->encoded_desc);
+  }
+
+  if (entry_has_decrypted_descriptor(entry)) {
+    size += hs_desc_obj_size(entry->desc);
+  }
+
+ end:
+  return size;
 }
 
-/* Remove a given descriptor from our cache. */
+/** Remove a given descriptor from our cache. */
 static void
 remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
 {
@@ -356,7 +386,7 @@ remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
   rend_cache_decrement_allocation(cache_get_client_entry_size(desc));
 }
 
-/* Store a given descriptor in our cache. */
+/** Store a given descriptor in our cache. */
 static void
 store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
 {
@@ -366,7 +396,7 @@ store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
   rend_cache_increment_allocation(cache_get_client_entry_size(desc));
 }
 
-/* Query our cache and return the entry or NULL if not found or if expired. */
+/** Query our cache and return the entry or NULL if not found or if expired. */
 STATIC hs_cache_client_descriptor_t *
 lookup_v3_desc_as_client(const uint8_t *key)
 {
@@ -389,15 +419,17 @@ lookup_v3_desc_as_client(const uint8_t *key)
   return cached_desc;
 }
 
-/* Parse the encoded descriptor in <b>desc_str</b> using
- * <b>service_identity_pk<b> to decrypt it first.
+/** Parse the encoded descriptor in <b>desc_str</b> using
+ * <b>service_identity_pk</b> to decrypt it first.
  *
  * If everything goes well, allocate and return a new
  * hs_cache_client_descriptor_t object. In case of error, return NULL. */
 static hs_cache_client_descriptor_t *
 cache_client_desc_new(const char *desc_str,
-                      const ed25519_public_key_t *service_identity_pk)
+                      const ed25519_public_key_t *service_identity_pk,
+                      hs_desc_decode_status_t *decode_status_out)
 {
+  hs_desc_decode_status_t ret;
   hs_descriptor_t *desc = NULL;
   hs_cache_client_descriptor_t *client_desc = NULL;
 
@@ -405,10 +437,24 @@ cache_client_desc_new(const char *desc_str,
   tor_assert(service_identity_pk);
 
   /* Decode the descriptor we just fetched. */
-  if (hs_client_decode_descriptor(desc_str, service_identity_pk, &desc) < 0) {
+  ret = hs_client_decode_descriptor(desc_str, service_identity_pk, &desc);
+  if (ret != HS_DESC_DECODE_OK &&
+      ret != HS_DESC_DECODE_NEED_CLIENT_AUTH &&
+      ret != HS_DESC_DECODE_BAD_CLIENT_AUTH) {
+    /* In the case of a missing or bad client authorization, we'll keep the
+     * descriptor in the cache because those credentials can arrive later. */
     goto end;
   }
-  tor_assert(desc);
+  /* Make sure we do have a descriptor if decoding was successful. */
+  if (ret == HS_DESC_DECODE_OK) {
+    tor_assert(desc);
+  } else {
+    if (BUG(desc != NULL)) {
+      /* We are not suppose to have a descriptor if the decoding code is not
+       * indicating success. Just in case, bail early to recover. */
+      goto end;
+    }
+  }
 
   /* All is good: make a cache object for this descriptor */
   client_desc = tor_malloc_zero(sizeof(hs_cache_client_descriptor_t));
@@ -421,6 +467,9 @@ cache_client_desc_new(const char *desc_str,
   client_desc->encoded_desc = tor_strdup(desc_str);
 
  end:
+  if (decode_status_out) {
+    *decode_status_out = ret;
+  }
   return client_desc;
 }
 
@@ -449,7 +498,7 @@ cache_client_desc_free_void(void *ptr)
   cache_client_desc_free(desc);
 }
 
-/* Return a newly allocated and initialized hs_cache_intro_state_t object. */
+/** Return a newly allocated and initialized hs_cache_intro_state_t object. */
 static hs_cache_intro_state_t *
 cache_intro_state_new(void)
 {
@@ -461,21 +510,21 @@ cache_intro_state_new(void)
 #define cache_intro_state_free(val) \
   FREE_AND_NULL(hs_cache_intro_state_t, cache_intro_state_free_, (val))
 
-/* Free an hs_cache_intro_state_t object. */
+/** Free an hs_cache_intro_state_t object. */
 static void
 cache_intro_state_free_(hs_cache_intro_state_t *state)
 {
   tor_free(state);
 }
 
-/* Helper function: used by the free all function. */
+/** Helper function: used by the free all function. */
 static void
 cache_intro_state_free_void(void *state)
 {
   cache_intro_state_free_(state);
 }
 
-/* Return a newly allocated and initialized hs_cache_client_intro_state_t
+/** Return a newly allocated and initialized hs_cache_client_intro_state_t
  * object. */
 static hs_cache_client_intro_state_t *
 cache_client_intro_state_new(void)
@@ -489,7 +538,7 @@ cache_client_intro_state_new(void)
   FREE_AND_NULL(hs_cache_client_intro_state_t,          \
                 cache_client_intro_state_free_, (val))
 
-/* Free a cache_client_intro_state object. */
+/** Free a cache_client_intro_state object. */
 static void
 cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
 {
@@ -500,14 +549,14 @@ cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
   tor_free(cache);
 }
 
-/* Helper function: used by the free all function. */
+/** Helper function: used by the free all function. */
 static void
 cache_client_intro_state_free_void(void *entry)
 {
   cache_client_intro_state_free_(entry);
 }
 
-/* For the given service identity key service_pk and an introduction
+/** For the given service identity key service_pk and an introduction
  * authentication key auth_key, lookup the intro state object. Return 1 if
  * found and put it in entry if not NULL. Return 0 if not found and entry is
  * untouched. */
@@ -542,7 +591,7 @@ cache_client_intro_state_lookup(const ed25519_public_key_t *service_pk,
   return 0;
 }
 
-/* Note the given failure in state. */
+/** Note the given failure in state. */
 static void
 cache_client_intro_state_note(hs_cache_intro_state_t *state,
                               rend_intro_point_failure_t failure)
@@ -564,7 +613,7 @@ cache_client_intro_state_note(hs_cache_intro_state_t *state,
   }
 }
 
-/* For the given service identity key service_pk and an introduction
+/** For the given service identity key service_pk and an introduction
  * authentication key auth_key, add an entry in the client intro state cache
  * If no entry exists for the service, it will create one. If state is non
  * NULL, it will point to the new intro state entry. */
@@ -598,7 +647,7 @@ cache_client_intro_state_add(const ed25519_public_key_t *service_pk,
   }
 }
 
-/* Remove every intro point state entry from cache that has been created
+/** Remove every intro point state entry from cache that has been created
  * before or at the cutoff. */
 static void
 cache_client_intro_state_clean(time_t cutoff,
@@ -615,7 +664,7 @@ cache_client_intro_state_clean(time_t cutoff,
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Return true iff no intro points are in this cache. */
+/** Return true iff no intro points are in this cache. */
 static int
 cache_client_intro_state_is_empty(const hs_cache_client_intro_state_t *cache)
 {
@@ -636,9 +685,25 @@ cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
   tor_assert(client_desc);
 
   /* Check if we already have a descriptor from this HS in cache. If we do,
-   * check if this descriptor is newer than the cached one */
+   * check if this descriptor is newer than the cached one only if we have a
+   * decoded descriptor. We do keep non-decoded descriptor that requires
+   * client authorization. */
   cache_entry = lookup_v3_desc_as_client(client_desc->key.pubkey);
   if (cache_entry != NULL) {
+    /* If the current or the new cache entry don't have a decrypted descriptor
+     * (missing client authorization), we always replace the current one with
+     * the new one. Reason is that we can't inspect the revision counter
+     * within the plaintext data so we blindly replace. */
+    if (!entry_has_decrypted_descriptor(cache_entry) ||
+        !entry_has_decrypted_descriptor(client_desc)) {
+      remove_v3_desc_as_client(cache_entry);
+      cache_client_desc_free(cache_entry);
+      goto store;
+    }
+
+    /* From this point on, we know that the decrypted descriptor is in the
+     * current entry and new object thus safe to access. */
+
     /* If we have an entry in our cache that has a revision counter greater
      * than the one we just fetched, discard the one we fetched. */
     if (cache_entry->desc->plaintext_data.revision_counter >
@@ -658,6 +723,7 @@ cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
     cache_client_desc_free(cache_entry);
   }
 
+ store:
   /* Store descriptor in cache */
   store_v3_desc_as_client(client_desc);
 
@@ -665,7 +731,7 @@ cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
   return 0;
 }
 
-/* Return true iff the cached client descriptor at <b>cached_desc</b has
+/** Return true iff the cached client descriptor at <b>cached_desc</b> has
  * expired. */
 static int
 cached_client_descriptor_has_expired(time_t now,
@@ -690,7 +756,7 @@ cached_client_descriptor_has_expired(time_t now,
   return 0;
 }
 
-/* clean the client cache using now as the current time. Return the total size
+/** clean the client cache using now as the current time. Return the total size
  * of removed bytes from the cache. */
 static size_t
 cache_clean_v3_as_client(time_t now)
@@ -713,6 +779,15 @@ cache_clean_v3_as_client(time_t now)
     MAP_DEL_CURRENT(key);
     entry_size = cache_get_client_entry_size(entry);
     bytes_removed += entry_size;
+
+    /* We just removed an old descriptor. We need to close all intro circuits
+     * if the descriptor is decrypted so we don't have leftovers that can be
+     * selected while lacking a descriptor. Circuits are selected by intro
+     * authentication key thus we need the descriptor. We leave the rendezvous
+     * circuits opened because they could be in use. */
+    if (entry_has_decrypted_descriptor(entry)) {
+      hs_client_close_intro_circuits_from_desc(entry->desc);
+    }
     /* Entry is not in the cache anymore, destroy it. */
     cache_client_desc_free(entry);
     /* Update our OOM. We didn't use the remove() function because we are in
@@ -750,7 +825,9 @@ hs_cache_lookup_encoded_as_client(const ed25519_public_key_t *key)
 }
 
 /** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
- *  its HS descriptor if it's stored in our cache, or NULL if not. */
+ *  its HS descriptor if it's stored in our cache, or NULL if not or if the
+ *  descriptor was never decrypted. The later can happen if we are waiting for
+ *  client authorization to be added. */
 const hs_descriptor_t *
 hs_cache_lookup_as_client(const ed25519_public_key_t *key)
 {
@@ -759,27 +836,41 @@ hs_cache_lookup_as_client(const ed25519_public_key_t *key)
   tor_assert(key);
 
   cached_desc = lookup_v3_desc_as_client(key->pubkey);
-  if (cached_desc) {
-    tor_assert(cached_desc->desc);
+  if (cached_desc && entry_has_decrypted_descriptor(cached_desc)) {
     return cached_desc->desc;
   }
 
   return NULL;
 }
 
-/** Public API: Given an encoded descriptor, store it in the client HS
- *  cache. Return -1 on error, 0 on success .*/
-int
+/** Public API: Given an encoded descriptor, store it in the client HS cache.
+ *  Return a decode status which changes how we handle the SOCKS connection
+ *  depending on its value:
+ *
+ *  HS_DESC_DECODE_OK: Returned on success. Descriptor was properly decoded
+ *                     and is now stored.
+ *
+ *  HS_DESC_DECODE_NEED_CLIENT_AUTH: Client authorization is needed but the
+ *                                   descriptor was still stored.
+ *
+ *  HS_DESC_DECODE_BAD_CLIENT_AUTH: Client authorization for this descriptor
+ *                                  was not usable but the descriptor was
+ *                                  still stored.
+ *
+ *  Any other codes means indicate where the error occured and the descriptor
+ *  was not stored. */
+hs_desc_decode_status_t
 hs_cache_store_as_client(const char *desc_str,
                          const ed25519_public_key_t *identity_pk)
 {
+  hs_desc_decode_status_t ret;
   hs_cache_client_descriptor_t *client_desc = NULL;
 
   tor_assert(desc_str);
   tor_assert(identity_pk);
 
   /* Create client cache descriptor object */
-  client_desc = cache_client_desc_new(desc_str, identity_pk);
+  client_desc = cache_client_desc_new(desc_str, identity_pk, &ret);
   if (!client_desc) {
     log_warn(LD_GENERAL, "HSDesc parsing failed!");
     log_debug(LD_GENERAL, "Failed to parse HSDesc: %s.", escaped(desc_str));
@@ -788,17 +879,54 @@ hs_cache_store_as_client(const char *desc_str,
 
   /* Push it to the cache */
   if (cache_store_as_client(client_desc) < 0) {
+    ret = HS_DESC_DECODE_GENERIC_ERROR;
     goto err;
   }
 
-  return 0;
+  return ret;
 
  err:
   cache_client_desc_free(client_desc);
-  return -1;
+  return ret;
 }
 
-/* Clean all client caches using the current time now. */
+/** Remove and free a client cache descriptor entry for the given onion
+ * service ed25519 public key. If the descriptor is decoded, the intro
+ * circuits are closed if any.
+ *
+ * This does nothing if no descriptor exists for the given key. */
+void
+hs_cache_remove_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (!cached_desc) {
+    return;
+  }
+  /* If we have a decrypted/decoded descriptor, attempt to close its
+   * introduction circuit(s). We shouldn't have circuit(s) without a
+   * descriptor else it will lead to a failure. */
+  if (entry_has_decrypted_descriptor(cached_desc)) {
+    hs_client_close_intro_circuits_from_desc(cached_desc->desc);
+  }
+  /* Remove and free. */
+  remove_v3_desc_as_client(cached_desc);
+  cache_client_desc_free(cached_desc);
+
+  /* Logging. */
+  {
+    char key_b64[BASE64_DIGEST256_LEN + 1];
+    digest256_to_base64(key_b64, (const char *) key);
+    log_info(LD_REND, "Onion service v3 descriptor '%s' removed "
+                      "from client cache",
+             safe_str_client(key_b64));
+  }
+}
+
+/** Clean all client caches using the current time now. */
 void
 hs_cache_clean_as_client(time_t now)
 {
@@ -809,7 +937,7 @@ hs_cache_clean_as_client(time_t now)
   cache_clean_v3_as_client(now);
 }
 
-/* Purge the client descriptor cache. */
+/** Purge the client descriptor cache. */
 void
 hs_cache_purge_as_client(void)
 {
@@ -826,7 +954,7 @@ hs_cache_purge_as_client(void)
   log_info(LD_REND, "Hidden service client descriptor cache purged.");
 }
 
-/* For a given service identity public key and an introduction authentication
+/** For a given service identity public key and an introduction authentication
  * key, note the given failure in the client intro state cache. */
 void
 hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
@@ -848,7 +976,7 @@ hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
   cache_client_intro_state_note(entry, failure);
 }
 
-/* For a given service identity public key and an introduction authentication
+/** For a given service identity public key and an introduction authentication
  * key, return true iff it is present in the failure cache. */
 const hs_cache_intro_state_t *
 hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
@@ -859,7 +987,7 @@ hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
   return state;
 }
 
-/* Cleanup the client introduction state cache. */
+/** Cleanup the client introduction state cache. */
 void
 hs_cache_client_intro_state_clean(time_t now)
 {
@@ -879,7 +1007,7 @@ hs_cache_client_intro_state_clean(time_t now)
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Purge the client introduction state cache. */
+/** Purge the client introduction state cache. */
 void
 hs_cache_client_intro_state_purge(void)
 {
@@ -893,9 +1021,41 @@ hs_cache_client_intro_state_purge(void)
                     "cache purged.");
 }
 
+/* This is called when new client authorization was added to the global state.
+ * It attemps to decode the descriptor of the given service identity key.
+ *
+ * Return true if decoding was successful else false. */
+bool
+hs_cache_client_new_auth_parse(const ed25519_public_key_t *service_pk)
+{
+  bool ret = false;
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(service_pk);
+
+  if (!hs_cache_v3_client) {
+    return false;
+  }
+
+  cached_desc = lookup_v3_desc_as_client(service_pk->pubkey);
+  if (cached_desc == NULL || entry_has_decrypted_descriptor(cached_desc)) {
+    /* No entry for that service or the descriptor is already decoded. */
+    goto end;
+  }
+
+  /* Attempt a decode. If we are successful, inform the caller. */
+  if (hs_client_decode_descriptor(cached_desc->encoded_desc, service_pk,
+                                  &cached_desc->desc) == HS_DESC_DECODE_OK) {
+    ret = true;
+  }
+
+ end:
+  return ret;
+}
+
 /**************** Generics *********************************/
 
-/* Do a round of OOM cleanup on all directory caches. Return the amount of
+/** Do a round of OOM cleanup on all directory caches. Return the amount of
  * removed bytes. It is possible that the returned value is lower than
  * min_remove_bytes if the caches get emptied out so the caller should be
  * aware of this. */
@@ -949,7 +1109,7 @@ hs_cache_handle_oom(time_t now, size_t min_remove_bytes)
   return bytes_removed;
 }
 
-/* Return the maximum size of a v3 HS descriptor. */
+/** Return the maximum size of a v3 HS descriptor. */
 unsigned int
 hs_cache_get_max_descriptor_size(void)
 {
@@ -958,7 +1118,7 @@ hs_cache_get_max_descriptor_size(void)
                                             HS_DESC_MAX_LEN, 1, INT32_MAX);
 }
 
-/* Initialize the hidden service cache subsystem. */
+/** Initialize the hidden service cache subsystem. */
 void
 hs_cache_init(void)
 {
@@ -973,7 +1133,7 @@ hs_cache_init(void)
   hs_cache_client_intro_state = digest256map_new();
 }
 
-/* Cleanup the hidden service cache subsystem. */
+/** Cleanup the hidden service cache subsystem. */
 void
 hs_cache_free_all(void)
 {
diff --git a/src/feature/hs/hs_cache.h b/src/feature/hs/hs_cache.h
index 079d31d437..bb3c77f224 100644
--- a/src/feature/hs/hs_cache.h
+++ b/src/feature/hs/hs_cache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,47 +18,47 @@
 
 struct ed25519_public_key_t;
 
-/* This is the maximum time an introduction point state object can stay in the
+/** This is the maximum time an introduction point state object can stay in the
  * client cache in seconds (2 mins or 120 seconds). */
 #define HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE (2 * 60)
 
-/* Introduction point state. */
+/** Introduction point state. */
 typedef struct hs_cache_intro_state_t {
-  /* When this entry was created and put in the cache. */
+  /** When this entry was created and put in the cache. */
   time_t created_ts;
 
-  /* Did it suffered a generic error? */
+  /** Did it suffered a generic error? */
   unsigned int error : 1;
 
-  /* Did it timed out? */
+  /** Did it timed out? */
   unsigned int timed_out : 1;
 
-  /* How many times we tried to reached it and it was unreachable. */
+  /** How many times we tried to reached it and it was unreachable. */
   uint32_t unreachable_count;
 } hs_cache_intro_state_t;
 
 typedef struct hs_cache_client_intro_state_t {
-  /* Contains hs_cache_intro_state_t object indexed by introduction point
+  /** Contains hs_cache_intro_state_t object indexed by introduction point
    * authentication key. */
   digest256map_t *intro_points;
 } hs_cache_client_intro_state_t;
 
-/* Descriptor representation on the directory side which is a subset of
+/** Descriptor representation on the directory side which is a subset of
  * information that the HSDir can decode and serve it. */
 typedef struct hs_cache_dir_descriptor_t {
-  /* This object is indexed using the blinded pubkey located in the plaintext
+  /** This object is indexed using the blinded pubkey located in the plaintext
    * data which is populated only once the descriptor has been successfully
    * decoded and validated. This simply points to that pubkey. */
   const uint8_t *key;
 
-  /* When does this entry has been created. Used to expire entries. */
+  /** When does this entry has been created. Used to expire entries. */
   time_t created_ts;
 
-  /* Descriptor plaintext information. Obviously, we can't decrypt the
+  /** Descriptor plaintext information. Obviously, we can't decrypt the
    * encrypted part of the descriptor. */
   hs_desc_plaintext_data_t *plaintext_data;
 
-  /* Encoded descriptor which is basically in text form. It's a NUL terminated
+  /** Encoded descriptor which is basically in text form. It's a NUL terminated
    * string thus safe to strlen(). */
   char *encoded_desc;
 } hs_cache_dir_descriptor_t;
@@ -83,8 +83,9 @@ const hs_descriptor_t *
 hs_cache_lookup_as_client(const struct ed25519_public_key_t *key);
 const char *
 hs_cache_lookup_encoded_as_client(const struct ed25519_public_key_t *key);
-int hs_cache_store_as_client(const char *desc_str,
-                             const struct ed25519_public_key_t *identity_pk);
+hs_desc_decode_status_t hs_cache_store_as_client(const char *desc_str,
+                           const struct ed25519_public_key_t *identity_pk);
+void hs_cache_remove_as_client(const struct ed25519_public_key_t *key);
 void hs_cache_clean_as_client(time_t now);
 void hs_cache_purge_as_client(void);
 
@@ -99,24 +100,28 @@ const hs_cache_intro_state_t *hs_cache_client_intro_state_find(
 void hs_cache_client_intro_state_clean(time_t now);
 void hs_cache_client_intro_state_purge(void);
 
+bool hs_cache_client_new_auth_parse(const ed25519_public_key_t *service_pk);
+
 #ifdef HS_CACHE_PRIVATE
 #include "lib/crypt_ops/crypto_ed25519.h"
 
 /** Represents a locally cached HS descriptor on a hidden service client. */
 typedef struct hs_cache_client_descriptor_t {
-  /* This object is indexed using the service identity public key */
+  /** This object is indexed using the service identity public key */
   struct ed25519_public_key_t key;
 
-  /* When will this entry expire? We expire cached client descriptors in the
+  /** When will this entry expire? We expire cached client descriptors in the
    * start of the next time period, since that's when clients need to start
    * using the next blinded key of the service. */
   time_t expiration_ts;
 
-  /* The cached descriptor, this object is the owner. It can't be NULL. A
-   * cache object without a valid descriptor is not possible. */
+  /** The cached decoded descriptor, this object is the owner. This can be
+   * NULL if the descriptor couldn't be decoded due to missing or bad client
+   * authorization. It can be decoded later from the encoded_desc object if
+   * the proper client authorization is given tor. */
   hs_descriptor_t *desc;
 
-  /* Encoded descriptor in string form. Can't be NULL. */
+  /** Encoded descriptor in string form. Can't be NULL. */
   char *encoded_desc;
 } hs_cache_client_descriptor_t;
 
diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c
index 613ffe7260..fc9f4a2654 100644
--- a/src/feature/hs/hs_cell.c
+++ b/src/feature/hs/hs_cell.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #include "feature/hs_common/replaycache.h"
 
 #include "feature/hs/hs_cell.h"
+#include "feature/hs/hs_ob.h"
 #include "core/crypto/hs_ntor.h"
 
 #include "core/or/origin_circuit_st.h"
@@ -24,7 +25,7 @@
 #include "trunnel/hs/cell_introduce1.h"
 #include "trunnel/hs/cell_rendezvous.h"
 
-/* Compute the MAC of an INTRODUCE cell in mac_out. The encoded_cell param is
+/** Compute the MAC of an INTRODUCE cell in mac_out. The encoded_cell param is
  * the cell content up to the ENCRYPTED section of length encoded_cell_len.
  * The encrypted param is the start of the ENCRYPTED section of length
  * encrypted_len. The mac_key is the key needed for the computation of the MAC
@@ -67,14 +68,17 @@ compute_introduce_mac(const uint8_t *encoded_cell, size_t encoded_cell_len,
   memwipe(mac_msg, 0, sizeof(mac_msg));
 }
 
-/* From a set of keys, subcredential and the ENCRYPTED section of an
- * INTRODUCE2 cell, return a newly allocated intro cell keys structure.
- * Finally, the client public key is copied in client_pk. On error, return
- * NULL. */
+/**
+ * From a set of keys, a list of subcredentials, and the ENCRYPTED section of
+ * an INTRODUCE2 cell, return an array of newly allocated intro cell keys
+ * structures.  Finally, the client public key is copied in client_pk. On
+ * error, return NULL.
+ **/
 static hs_ntor_intro_cell_keys_t *
 get_introduce2_key_material(const ed25519_public_key_t *auth_key,
                             const curve25519_keypair_t *enc_key,
-                            const uint8_t *subcredential,
+                            size_t n_subcredentials,
+                            const hs_subcredential_t *subcredentials,
                             const uint8_t *encrypted_section,
                             curve25519_public_key_t *client_pk)
 {
@@ -82,17 +86,19 @@ get_introduce2_key_material(const ed25519_public_key_t *auth_key,
 
   tor_assert(auth_key);
   tor_assert(enc_key);
-  tor_assert(subcredential);
+  tor_assert(n_subcredentials > 0);
+  tor_assert(subcredentials);
   tor_assert(encrypted_section);
   tor_assert(client_pk);
 
-  keys = tor_malloc_zero(sizeof(*keys));
+  keys = tor_calloc(n_subcredentials, sizeof(hs_ntor_intro_cell_keys_t));
 
   /* First bytes of the ENCRYPTED section are the client public key. */
   memcpy(client_pk->public_key, encrypted_section, CURVE25519_PUBKEY_LEN);
 
-  if (hs_ntor_service_get_introduce1_keys(auth_key, enc_key, client_pk,
-                                          subcredential, keys) < 0) {
+  if (hs_ntor_service_get_introduce1_keys_multi(auth_key, enc_key, client_pk,
+                                                n_subcredentials,
+                                                subcredentials, keys) < 0) {
     /* Don't rely on the caller to wipe this on error. */
     memwipe(client_pk, 0, sizeof(curve25519_public_key_t));
     tor_free(keys);
@@ -101,7 +107,7 @@ get_introduce2_key_material(const ed25519_public_key_t *auth_key,
   return keys;
 }
 
-/* Using the given encryption key, decrypt the encrypted_section of length
+/** Using the given encryption key, decrypt the encrypted_section of length
  * encrypted_section_len of an INTRODUCE2 cell and return a newly allocated
  * buffer containing the decrypted data. On decryption failure, NULL is
  * returned. */
@@ -136,7 +142,7 @@ decrypt_introduce2(const uint8_t *enc_key, const uint8_t *encrypted_section,
   return decrypted;
 }
 
-/* Given a pointer to the decrypted data of the ENCRYPTED section of an
+/** Given a pointer to the decrypted data of the ENCRYPTED section of an
  * INTRODUCE2 cell of length decrypted_len, parse and validate the cell
  * content. Return a newly allocated cell structure or NULL on error. The
  * circuit and service object are only used for logging purposes. */
@@ -188,7 +194,7 @@ parse_introduce2_encrypted(const uint8_t *decrypted_data,
   return NULL;
 }
 
-/* Build a legacy ESTABLISH_INTRO cell with the given circuit nonce and RSA
+/** Build a legacy ESTABLISH_INTRO cell with the given circuit nonce and RSA
  * encryption key. The encoded cell is put in cell_out that MUST at least be
  * of the size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on
  * success else a negative value and cell_out is untouched. */
@@ -210,7 +216,7 @@ build_legacy_establish_intro(const char *circ_nonce, crypto_pk_t *enc_key,
   return cell_len;
 }
 
-/* Parse an INTRODUCE2 cell from payload of size payload_len for the given
+/** Parse an INTRODUCE2 cell from payload of size payload_len for the given
  * service and circuit which are used only for logging purposes. The resulting
  * parsed cell is put in cell_ptr_out.
  *
@@ -249,7 +255,7 @@ parse_introduce2_cell(const hs_service_t *service,
   return -1;
 }
 
-/* Set the onion public key onion_pk in cell, the encrypted section of an
+/** Set the onion public key onion_pk in cell, the encrypted section of an
  * INTRODUCE1 cell. */
 static void
 introduce1_set_encrypted_onion_key(trn_cell_introduce_encrypted_t *cell,
@@ -266,7 +272,7 @@ introduce1_set_encrypted_onion_key(trn_cell_introduce_encrypted_t *cell,
          trn_cell_introduce_encrypted_getlen_onion_key(cell));
 }
 
-/* Set the link specifiers in lspecs in cell, the encrypted section of an
+/** Set the link specifiers in lspecs in cell, the encrypted section of an
  * INTRODUCE1 cell. */
 static void
 introduce1_set_encrypted_link_spec(trn_cell_introduce_encrypted_t *cell,
@@ -286,7 +292,7 @@ introduce1_set_encrypted_link_spec(trn_cell_introduce_encrypted_t *cell,
                     trn_cell_introduce_encrypted_add_nspecs(cell, ls));
 }
 
-/* Set padding in the enc_cell only if needed that is the total length of both
+/** Set padding in the enc_cell only if needed that is the total length of both
  * sections are below the mininum required for an INTRODUCE1 cell. */
 static void
 introduce1_set_encrypted_padding(const trn_cell_introduce1_t *cell,
@@ -306,7 +312,7 @@ introduce1_set_encrypted_padding(const trn_cell_introduce1_t *cell,
   }
 }
 
-/* Encrypt the ENCRYPTED payload and encode it in the cell using the enc_cell
+/** Encrypt the ENCRYPTED payload and encode it in the cell using the enc_cell
  * and the INTRODUCE1 data.
  *
  * This can't fail but it is very important that the caller sets every field
@@ -394,7 +400,7 @@ introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell,
   tor_free(encrypted);
 }
 
-/* Using the INTRODUCE1 data, setup the ENCRYPTED section in cell. This means
+/** Using the INTRODUCE1 data, setup the ENCRYPTED section in cell. This means
  * set it, encrypt it and encode it. */
 static void
 introduce1_set_encrypted(trn_cell_introduce1_t *cell,
@@ -435,7 +441,7 @@ introduce1_set_encrypted(trn_cell_introduce1_t *cell,
   trn_cell_introduce_encrypted_free(enc_cell);
 }
 
-/* Set the authentication key in the INTRODUCE1 cell from the given data. */
+/** Set the authentication key in the INTRODUCE1 cell from the given data. */
 static void
 introduce1_set_auth_key(trn_cell_introduce1_t *cell,
                         const hs_cell_introduce1_data_t *data)
@@ -451,7 +457,7 @@ introduce1_set_auth_key(trn_cell_introduce1_t *cell,
          data->auth_pk->pubkey, trn_cell_introduce1_getlen_auth_key(cell));
 }
 
-/* Set the legacy ID field in the INTRODUCE1 cell from the given data. */
+/** Set the legacy ID field in the INTRODUCE1 cell from the given data. */
 static void
 introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
                          const hs_cell_introduce1_data_t *data)
@@ -473,26 +479,150 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
   }
 }
 
+/** Build and add to the given DoS cell extension the given parameter type and
+ * value. */
+static void
+build_establish_intro_dos_param(trn_cell_extension_dos_t *dos_ext,
+                                uint8_t param_type, uint64_t param_value)
+{
+  trn_cell_extension_dos_param_t *dos_param =
+    trn_cell_extension_dos_param_new();
+
+  /* Extra safety. We should never send an unknown parameter type. */
+  tor_assert(param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC ||
+             param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+
+  trn_cell_extension_dos_param_set_type(dos_param, param_type);
+  trn_cell_extension_dos_param_set_value(dos_param, param_value);
+  trn_cell_extension_dos_add_params(dos_ext, dos_param);
+
+  /* Not freeing the trunnel object because it is now owned by dos_ext. */
+}
+
+/** Build the DoS defense cell extension and put it in the given extensions
+ * object. Return 0 on success, -1 on failure.  (Right now, failure is only
+ * possible if there is a bug.) */
+static int
+build_establish_intro_dos_extension(const hs_service_config_t *service_config,
+                                    trn_cell_extension_t *extensions)
+{
+  ssize_t ret;
+  size_t dos_ext_encoded_len;
+  uint8_t *field_array;
+  trn_cell_extension_field_t *field = NULL;
+  trn_cell_extension_dos_t *dos_ext = NULL;
+
+  tor_assert(service_config);
+  tor_assert(extensions);
+
+  /* We are creating a cell extension field of the type DoS. */
+  field = trn_cell_extension_field_new();
+  trn_cell_extension_field_set_field_type(field,
+                                          TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+
+  /* Build DoS extension field. We will put in two parameters. */
+  dos_ext = trn_cell_extension_dos_new();
+  trn_cell_extension_dos_set_n_params(dos_ext, 2);
+
+  /* Build DoS parameter INTRO2 rate per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC,
+                                  service_config->intro_dos_rate_per_sec);
+  /* Build DoS parameter INTRO2 burst per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC,
+                                  service_config->intro_dos_burst_per_sec);
+
+  /* Set the field with the encoded DoS extension. */
+  ret = trn_cell_extension_dos_encoded_len(dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  dos_ext_encoded_len = ret;
+  /* Set length field and the field array size length. */
+  trn_cell_extension_field_set_field_len(field, dos_ext_encoded_len);
+  trn_cell_extension_field_setlen_field(field, dos_ext_encoded_len);
+  /* Encode the DoS extension into the cell extension field. */
+  field_array = trn_cell_extension_field_getarray_field(field);
+  ret = trn_cell_extension_dos_encode(field_array,
+                 trn_cell_extension_field_getlen_field(field), dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  tor_assert(ret == (ssize_t) dos_ext_encoded_len);
+
+  /* Finally, encode field into the cell extension. */
+  trn_cell_extension_add_fields(extensions, field);
+
+  /* We've just add an extension field to the cell extensions so increment the
+   * total number. */
+  trn_cell_extension_set_num(extensions,
+                             trn_cell_extension_get_num(extensions) + 1);
+
+  /* Cleanup. DoS extension has been encoded at this point. */
+  trn_cell_extension_dos_free(dos_ext);
+
+  return 0;
+
+ err:
+  trn_cell_extension_field_free(field);
+  trn_cell_extension_dos_free(dos_ext);
+  return -1;
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
-/* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
+/** Allocate and build all the ESTABLISH_INTRO cell extension. The given
+ * extensions pointer is always set to a valid cell extension object. */
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip)
+{
+  int ret;
+  trn_cell_extension_t *extensions;
+
+  tor_assert(service_config);
+  tor_assert(ip);
+
+  extensions = trn_cell_extension_new();
+  trn_cell_extension_set_num(extensions, 0);
+
+  /* If the defense has been enabled service side (by the operator with a
+   * torrc option) and the intro point does support it. */
+  if (service_config->has_dos_defense_enabled &&
+      ip->support_intro2_dos_defense) {
+    /* This function takes care to increment the number of extensions. */
+    ret = build_establish_intro_dos_extension(service_config, extensions);
+    if (ret < 0) {
+      /* Return no extensions on error. */
+      goto end;
+    }
+  }
+
+ end:
+  return extensions;
+}
+
+/** Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
  * object. The encoded cell is put in cell_out that MUST at least be of the
  * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else
  * a negative value and cell_out is untouched. This function also supports
  * legacy cell creation. */
 ssize_t
 hs_cell_build_establish_intro(const char *circ_nonce,
+                              const hs_service_config_t *service_config,
                               const hs_service_intro_point_t *ip,
                               uint8_t *cell_out)
 {
   ssize_t cell_len = -1;
   uint16_t sig_len = ED25519_SIG_LEN;
-  trn_cell_extension_t *ext;
   trn_cell_establish_intro_t *cell = NULL;
+  trn_cell_extension_t *extensions;
 
   tor_assert(circ_nonce);
+  tor_assert(service_config);
   tor_assert(ip);
 
   /* Quickly handle the legacy IP. */
@@ -505,11 +635,12 @@ hs_cell_build_establish_intro(const char *circ_nonce,
     goto done;
   }
 
+  /* Build the extensions, if any. */
+  extensions = build_establish_intro_extensions(service_config, ip);
+
   /* Set extension data. None used here. */
-  ext = trn_cell_extension_new();
-  trn_cell_extension_set_num(ext, 0);
   cell = trn_cell_establish_intro_new();
-  trn_cell_establish_intro_set_extensions(cell, ext);
+  trn_cell_establish_intro_set_extensions(cell, extensions);
   /* Set signature size. Array is then allocated in the cell. We need to do
    * this early so we can use trunnel API to get the signature length. */
   trn_cell_establish_intro_set_sig_len(cell, sig_len);
@@ -600,7 +731,7 @@ hs_cell_build_establish_intro(const char *circ_nonce,
   return cell_len;
 }
 
-/* Parse the INTRO_ESTABLISHED cell in the payload of size payload_len. If we
+/** Parse the INTRO_ESTABLISHED cell in the payload of size payload_len. If we
  * are successful at parsing it, return the length of the parsed cell else a
  * negative value on error. */
 ssize_t
@@ -622,7 +753,75 @@ hs_cell_parse_intro_established(const uint8_t *payload, size_t payload_len)
   return ret;
 }
 
-/* Parse the INTRODUCE2 cell using data which contains everything we need to
+/** For the encrypted INTRO2 cell in <b>encrypted_section</b>, use the crypto
+ * material in <b>data</b> to compute the right ntor keys. Also validate the
+ * INTRO2 MAC to ensure that the keys are the right ones.
+ *
+ * Return NULL on failure to either produce the key material or on MAC
+ * validation. Else return a newly allocated intro keys object. */
+static hs_ntor_intro_cell_keys_t *
+get_introduce2_keys_and_verify_mac(hs_cell_introduce2_data_t *data,
+                                   const uint8_t *encrypted_section,
+                                   size_t encrypted_section_len)
+{
+  hs_ntor_intro_cell_keys_t *intro_keys = NULL;
+  hs_ntor_intro_cell_keys_t *intro_keys_result = NULL;
+
+  /* Build the key material out of the key material found in the cell. */
+  intro_keys = get_introduce2_key_material(data->auth_pk, data->enc_kp,
+                                           data->n_subcredentials,
+                                           data->subcredentials,
+                                           encrypted_section,
+                                           &data->client_pk);
+  if (intro_keys == NULL) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to "
+             "compute key material");
+    return NULL;
+  }
+
+  /* Make sure we are not about to underflow. */
+  if (BUG(encrypted_section_len < DIGEST256_LEN)) {
+    return NULL;
+  }
+
+  /* Validate MAC from the cell and our computed key material. The MAC field
+   * in the cell is at the end of the encrypted section. */
+  intro_keys_result = tor_malloc_zero(sizeof(*intro_keys_result));
+  for (unsigned i = 0; i < data->n_subcredentials; ++i) {
+    uint8_t mac[DIGEST256_LEN];
+
+    /* The MAC field is at the very end of the ENCRYPTED section. */
+    size_t mac_offset = encrypted_section_len - sizeof(mac);
+    /* Compute the MAC. Use the entire encoded payload with a length up to the
+     * ENCRYPTED section. */
+    compute_introduce_mac(data->payload,
+                          data->payload_len - encrypted_section_len,
+                          encrypted_section, encrypted_section_len,
+                          intro_keys[i].mac_key,
+                          sizeof(intro_keys[i].mac_key),
+                          mac, sizeof(mac));
+    /* Time-invariant conditional copy: if the MAC is what we expected, then
+     * set intro_keys_result to intro_keys[i]. Otherwise, don't: but don't
+     * leak which one it was! */
+    bool equal = tor_memeq(mac, encrypted_section + mac_offset, sizeof(mac));
+    memcpy_if_true_timei(equal, intro_keys_result, &intro_keys[i],
+                         sizeof(*intro_keys_result));
+  }
+
+  /* We no longer need intro_keys. */
+  memwipe(intro_keys, 0,
+          sizeof(hs_ntor_intro_cell_keys_t) * data->n_subcredentials);
+  tor_free(intro_keys);
+
+  if (safe_mem_is_zero(intro_keys_result, sizeof(*intro_keys_result))) {
+    log_info(LD_REND, "Invalid MAC validation for INTRODUCE2 cell");
+    tor_free(intro_keys_result); /* sets intro_keys_result to NULL */
+  }
+
+  return intro_keys_result;
+}
+
+/** Parse the INTRODUCE2 cell using data which contains everything we need to
  * do so and contains the destination buffers of information we extract and
  * compute from the cell. Return 0 on success else a negative value. The
  * service and circ are only used for logging purposes. */
@@ -670,47 +869,29 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
   /* Check our replay cache for this introduction point. */
   if (replaycache_add_test_and_elapsed(data->replay_cache, encrypted_section,
                                        encrypted_section_len, &elapsed)) {
-    log_warn(LD_REND, "Possible replay detected! An INTRODUCE2 cell with the"
+    log_warn(LD_REND, "Possible replay detected! An INTRODUCE2 cell with the "
                       "same ENCRYPTED section was seen %ld seconds ago. "
                       "Dropping cell.", (long int) elapsed);
     goto done;
   }
 
-  /* Build the key material out of the key material found in the cell. */
-  intro_keys = get_introduce2_key_material(data->auth_pk, data->enc_kp,
-                                           data->subcredential,
-                                           encrypted_section,
-                                           &data->client_pk);
-  if (intro_keys == NULL) {
-    log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to "
-                      "compute key material on circuit %u for service %s",
-             TO_CIRCUIT(circ)->n_circ_id,
+  /* First bytes of the ENCRYPTED section are the client public key (they are
+   * guaranteed to exist because of the length check above). We are gonna use
+   * the client public key to compute the ntor keys and decrypt the payload:
+   */
+  memcpy(&data->client_pk.public_key, encrypted_section,
+         CURVE25519_PUBKEY_LEN);
+
+  /* Get the right INTRODUCE2 ntor keys and verify the cell MAC */
+  intro_keys = get_introduce2_keys_and_verify_mac(data, encrypted_section,
+                                                  encrypted_section_len);
+  if (!intro_keys) {
+    log_warn(LD_REND, "Could not get valid INTRO2 keys on circuit %u "
+             "for service %s", TO_CIRCUIT(circ)->n_circ_id,
              safe_str_client(service->onion_address));
     goto done;
   }
 
-  /* Validate MAC from the cell and our computed key material. The MAC field
-   * in the cell is at the end of the encrypted section. */
-  {
-    uint8_t mac[DIGEST256_LEN];
-    /* The MAC field is at the very end of the ENCRYPTED section. */
-    size_t mac_offset = encrypted_section_len - sizeof(mac);
-    /* Compute the MAC. Use the entire encoded payload with a length up to the
-     * ENCRYPTED section. */
-    compute_introduce_mac(data->payload,
-                          data->payload_len - encrypted_section_len,
-                          encrypted_section, encrypted_section_len,
-                          intro_keys->mac_key, sizeof(intro_keys->mac_key),
-                          mac, sizeof(mac));
-    if (tor_memcmp(mac, encrypted_section + mac_offset, sizeof(mac))) {
-      log_info(LD_REND, "Invalid MAC validation for INTRODUCE2 cell on "
-                        "circuit %u for service %s",
-               TO_CIRCUIT(circ)->n_circ_id,
-               safe_str_client(service->onion_address));
-      goto done;
-    }
-  }
-
   {
     /* The ENCRYPTED_DATA section starts just after the CLIENT_PK. */
     const uint8_t *encrypted_data =
@@ -758,7 +939,14 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
        idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) {
     link_specifier_t *lspec =
       trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx);
-    smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec));
+    if (BUG(!lspec)) {
+      goto done;
+    }
+    link_specifier_t *lspec_dup = link_specifier_dup(lspec);
+    if (BUG(!lspec_dup)) {
+      goto done;
+    }
+    smartlist_add(data->link_specifiers, lspec_dup);
   }
 
   /* Success. */
@@ -776,7 +964,7 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
   return ret;
 }
 
-/* Build a RENDEZVOUS1 cell with the given rendezvous cookie and handshake
+/** Build a RENDEZVOUS1 cell with the given rendezvous cookie and handshake
  * info. The encoded cell is put in cell_out and the length of the data is
  * returned. This can't fail. */
 ssize_t
@@ -810,7 +998,7 @@ hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
   return cell_len;
 }
 
-/* Build an INTRODUCE1 cell from the given data. The encoded cell is put in
+/** Build an INTRODUCE1 cell from the given data. The encoded cell is put in
  * cell_out which must be of at least size RELAY_PAYLOAD_SIZE. On success, the
  * encoded length is returned else a negative value and the content of
  * cell_out should be ignored. */
@@ -851,7 +1039,7 @@ hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
   return cell_len;
 }
 
-/* Build an ESTABLISH_RENDEZVOUS cell from the given rendezvous_cookie. The
+/** Build an ESTABLISH_RENDEZVOUS cell from the given rendezvous_cookie. The
  * encoded cell is put in cell_out which must be of at least
  * RELAY_PAYLOAD_SIZE. On success, the encoded length is returned and the
  * caller should clear up the content of the cell.
@@ -868,7 +1056,7 @@ hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
   return HS_REND_COOKIE_LEN;
 }
 
-/* Handle an INTRODUCE_ACK cell encoded in payload of length payload_len.
+/** Handle an INTRODUCE_ACK cell encoded in payload of length payload_len.
  * Return the status code on success else a negative value if the cell as not
  * decodable. */
 int
@@ -903,7 +1091,7 @@ hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len)
   return ret;
 }
 
-/* Handle a RENDEZVOUS2 cell encoded in payload of length payload_len. On
+/** Handle a RENDEZVOUS2 cell encoded in payload of length payload_len. On
  * success, handshake_info contains the data in the HANDSHAKE_INFO field, and
  * 0 is returned. On error, a negative value is returned. */
 int
@@ -935,7 +1123,7 @@ hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
   return ret;
 }
 
-/* Clear the given INTRODUCE1 data structure data. */
+/** Clear the given INTRODUCE1 data structure data. */
 void
 hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
 {
@@ -949,4 +1137,3 @@ hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
   /* The data object has no ownership of any members. */
   memwipe(data, 0, sizeof(hs_cell_introduce1_data_t));
 }
-
diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h
index 9569de535e..2b28c44c50 100644
--- a/src/feature/hs/hs_cell.h
+++ b/src/feature/hs/hs_cell.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,35 +12,37 @@
 #include "core/or/or.h"
 #include "feature/hs/hs_service.h"
 
-/* An INTRODUCE1 cell requires at least this amount of bytes (see section
+/** An INTRODUCE1 cell requires at least this amount of bytes (see section
  * 3.2.2 of the specification). Below this value, the cell must be padded. */
 #define HS_CELL_INTRODUCE1_MIN_SIZE 246
 
-/* This data structure contains data that we need to build an INTRODUCE1 cell
+struct hs_subcredential_t;
+
+/** This data structure contains data that we need to build an INTRODUCE1 cell
  * used by the INTRODUCE1 build function. */
 typedef struct hs_cell_introduce1_data_t {
-  /* Is this a legacy introduction point? */
+  /** Is this a legacy introduction point? */
   unsigned int is_legacy : 1;
-  /* (Legacy only) The encryption key for a legacy intro point. Only set if
+  /** (Legacy only) The encryption key for a legacy intro point. Only set if
    * is_legacy is true. */
   const crypto_pk_t *legacy_key;
-  /* Introduction point authentication public key. */
+  /** Introduction point authentication public key. */
   const ed25519_public_key_t *auth_pk;
-  /* Introduction point encryption public key. */
+  /** Introduction point encryption public key. */
   const curve25519_public_key_t *enc_pk;
-  /* Subcredentials of the service. */
-  const uint8_t *subcredential;
-  /* Onion public key for the ntor handshake. */
+  /** Subcredentials of the service. */
+  const struct hs_subcredential_t *subcredential;
+  /** Onion public key for the ntor handshake. */
   const curve25519_public_key_t *onion_pk;
-  /* Rendezvous cookie. */
+  /** Rendezvous cookie. */
   const uint8_t *rendezvous_cookie;
-  /* Public key put before the encrypted data (CLIENT_PK). */
+  /** Public key put before the encrypted data (CLIENT_PK). */
   const curve25519_keypair_t *client_kp;
-  /* Rendezvous point link specifiers. */
+  /** Rendezvous point link specifiers. */
   smartlist_t *link_specifiers;
 } hs_cell_introduce1_data_t;
 
-/* This data structure contains data that we need to parse an INTRODUCE2 cell
+/** This data structure contains data that we need to parse an INTRODUCE2 cell
  * which is used by the INTRODUCE2 cell parsing function. On a successful
  * parsing, the onion_pk and rendezvous_cookie will be populated with the
  * computed key material from the cell data. This structure is only used during
@@ -48,37 +50,43 @@ typedef struct hs_cell_introduce1_data_t {
 typedef struct hs_cell_introduce2_data_t {
   /*** Immutable Section: Set on structure init. ***/
 
-  /* Introduction point authentication public key. Pointer owned by the
+  /** Introduction point authentication public key. Pointer owned by the
      introduction point object through which we received the INTRO2 cell. */
   const ed25519_public_key_t *auth_pk;
-  /* Introduction point encryption keypair for the ntor handshake. Pointer
+  /** Introduction point encryption keypair for the ntor handshake. Pointer
      owned by the introduction point object through which we received the
      INTRO2 cell*/
   const curve25519_keypair_t *enc_kp;
-  /* Subcredentials of the service. Pointer owned by the descriptor that owns
-     the introduction point through which we received the INTRO2 cell. */
-  const uint8_t *subcredential;
-  /* Payload of the received encoded cell. */
+  /**
+   * Length of the subcredentials array below.
+   **/
+  size_t n_subcredentials;
+  /** Array of <b>n_subcredentials</b> subcredentials for the service. Pointer
+   * owned by the descriptor that owns the introduction point through which we
+   * received the INTRO2 cell. */
+  const struct hs_subcredential_t *subcredentials;
+  /** Payload of the received encoded cell. */
   const uint8_t *payload;
-  /* Size of the payload of the received encoded cell. */
+  /** Size of the payload of the received encoded cell. */
   size_t payload_len;
 
   /*** Mutable Section: Set upon parsing INTRODUCE2 cell. ***/
 
-  /* Onion public key computed using the INTRODUCE2 encrypted section. */
+  /** Onion public key computed using the INTRODUCE2 encrypted section. */
   curve25519_public_key_t onion_pk;
-  /* Rendezvous cookie taken from the INTRODUCE2 encrypted section. */
+  /** Rendezvous cookie taken from the INTRODUCE2 encrypted section. */
   uint8_t rendezvous_cookie[REND_COOKIE_LEN];
-  /* Client public key from the INTRODUCE2 encrypted section. */
+  /** Client public key from the INTRODUCE2 encrypted section. */
   curve25519_public_key_t client_pk;
-  /* Link specifiers of the rendezvous point. Contains link_specifier_t. */
+  /** Link specifiers of the rendezvous point. Contains link_specifier_t. */
   smartlist_t *link_specifiers;
-  /* Replay cache of the introduction point. */
+  /** Replay cache of the introduction point. */
   replaycache_t *replay_cache;
 } hs_cell_introduce2_data_t;
 
 /* Build cell API. */
 ssize_t hs_cell_build_establish_intro(const char *circ_nonce,
+                                      const hs_service_config_t *config,
                                       const hs_service_intro_point_t *ip,
                                       uint8_t *cell_out);
 ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
@@ -105,5 +113,14 @@ int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
 /* Util API. */
 void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data);
 
-#endif /* !defined(TOR_HS_CELL_H) */
+#ifdef TOR_UNIT_TESTS
+
+#include "trunnel/hs/cell_common.h"
 
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_HS_CELL_H) */
diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c
index 8acfcbd65b..447f664f81 100644
--- a/src/feature/hs/hs_circuit.c
+++ b/src/feature/hs/hs_circuit.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,15 +15,19 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_ob.h"
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_service.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/rend/rendservice.h"
+#include "feature/rend/rendclient.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
@@ -39,7 +43,7 @@
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
 
-/* A circuit is about to become an e2e rendezvous circuit. Check
+/** A circuit is about to become an e2e rendezvous circuit. Check
  * <b>circ_purpose</b> and ensure that it's properly set. Return true iff
  * circuit purpose is properly set, otherwise return false. */
 static int
@@ -66,7 +70,7 @@ circuit_purpose_is_correct_for_rend(unsigned int circ_purpose,
   return 1;
 }
 
-/* Create and return a crypt path for the final hop of a v3 prop224 rendezvous
+/** Create and return a crypt path for the final hop of a v3 prop224 rendezvous
  * circuit. Initialize the crypt path crypto using the output material from the
  * ntor key exchange at <b>ntor_key_seed</b>.
  *
@@ -89,7 +93,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
   cpath = tor_malloc_zero(sizeof(crypt_path_t));
   cpath->magic = CRYPT_PATH_MAGIC;
 
-  if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys),
+  if (cpath_init_circuit_crypto(cpath, (char*)keys, sizeof(keys),
                                 is_service_side, 1) < 0) {
     tor_free(cpath);
     goto err;
@@ -100,7 +104,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
   return cpath;
 }
 
-/* We are a v2 legacy HS client: Create and return a crypt path for the hidden
+/** We are a v2 legacy HS client: Create and return a crypt path for the hidden
  * service on the other side of the rendezvous circuit <b>circ</b>. Initialize
  * the crypt path crypto using the body of the RENDEZVOUS1 cell at
  * <b>rend_cell_body</b> (which must be at least DH1024_KEY_LEN+DIGEST_LEN
@@ -126,7 +130,7 @@ create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
     goto err;
   }
   /* ... and set up cpath. */
-  if (circuit_init_cpath_crypto(hop,
+  if (cpath_init_circuit_crypto(hop,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 0, 0) < 0)
     goto err;
@@ -151,7 +155,7 @@ create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
   return hop;
 }
 
-/* Append the final <b>hop</b> to the cpath of the rend <b>circ</b>, and mark
+/** Append the final <b>hop</b> to the cpath of the rend <b>circ</b>, and mark
  * <b>circ</b> ready for use to transfer HS relay cells. */
 static void
 finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
@@ -177,7 +181,7 @@ finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
   circ->hs_circ_has_timed_out = 0;
 
   /* Append the hop to the cpath of this circuit */
-  onion_append_to_cpath(&circ->cpath, hop);
+  cpath_extend_linked_list(&circ->cpath, hop);
 
   /* In legacy code, 'pending_final_cpath' points to the final hop we just
    * appended to the cpath. We set the original pointer to NULL so that we
@@ -192,7 +196,7 @@ finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
   }
 }
 
-/* For a given circuit and a service introduction point object, register the
+/** For a given circuit and a service introduction point object, register the
  * intro circuit to the circuitmap. This supports legacy intro point. */
 static void
 register_intro_circ(const hs_service_intro_point_t *ip,
@@ -210,7 +214,7 @@ register_intro_circ(const hs_service_intro_point_t *ip,
   }
 }
 
-/* Return the number of opened introduction circuit for the given circuit that
+/** Return the number of opened introduction circuit for the given circuit that
  * is matching its identity key. */
 static unsigned int
 count_opened_desc_intro_point_circuits(const hs_service_t *service,
@@ -242,7 +246,7 @@ count_opened_desc_intro_point_circuits(const hs_service_t *service,
   return count;
 }
 
-/* From a given service, rendezvous cookie and handshake info, create a
+/** From a given service, rendezvous cookie and handshake info, create a
  * rendezvous point circuit identifier. This can't fail. */
 STATIC hs_ident_circuit_t *
 create_rp_circuit_identifier(const hs_service_t *service,
@@ -258,8 +262,7 @@ create_rp_circuit_identifier(const hs_service_t *service,
   tor_assert(server_pk);
   tor_assert(keys);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_RENDEZVOUS);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */
   memcpy(ident->rendezvous_cookie, rendezvous_cookie,
          sizeof(ident->rendezvous_cookie));
@@ -282,7 +285,7 @@ create_rp_circuit_identifier(const hs_service_t *service,
   return ident;
 }
 
-/* From a given service and service intro point, create an introduction point
+/** From a given service and service intro point, create an introduction point
  * circuit identifier. This can't fail. */
 static hs_ident_circuit_t *
 create_intro_circuit_identifier(const hs_service_t *service,
@@ -293,14 +296,13 @@ create_intro_circuit_identifier(const hs_service_t *service,
   tor_assert(service);
   tor_assert(ip);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_INTRO);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey);
 
   return ident;
 }
 
-/* For a given introduction point and an introduction circuit, send the
+/** For a given introduction point and an introduction circuit, send the
  * ESTABLISH_INTRO cell. The service object is used for logging. This can fail
  * and if so, the circuit is closed and the intro point object is flagged
  * that the circuit is not established anymore which is important for the
@@ -318,7 +320,7 @@ send_establish_intro(const hs_service_t *service,
 
   /* Encode establish intro cell. */
   cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce,
-                                           ip, payload);
+                                           &service->config, ip, payload);
   if (cell_len < 0) {
     log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s "
                       "on circuit %u. Closing circuit.",
@@ -350,7 +352,7 @@ send_establish_intro(const hs_service_t *service,
   memwipe(payload, 0, sizeof(payload));
 }
 
-/* Return a string constant describing the anonymity of service. */
+/** Return a string constant describing the anonymity of service. */
 static const char *
 get_service_anonymity_string(const hs_service_t *service)
 {
@@ -361,15 +363,15 @@ get_service_anonymity_string(const hs_service_t *service)
   }
 }
 
-/* For a given service, the ntor onion key and a rendezvous cookie, launch a
+/** For a given service, the ntor onion key and a rendezvous cookie, launch a
  * circuit to the rendezvous point specified by the link specifiers. On
  * success, a circuit identifier is attached to the circuit with the needed
  * data. This function will try to open a circuit for a maximum value of
  * MAX_REND_FAILURES then it will give up. */
-static void
-launch_rendezvous_point_circuit(const hs_service_t *service,
-                                const hs_service_intro_point_t *ip,
-                                const hs_cell_introduce2_data_t *data)
+MOCK_IMPL(STATIC void,
+launch_rendezvous_point_circuit,(const hs_service_t *service,
+                                 const hs_service_intro_point_t *ip,
+                                 const hs_cell_introduce2_data_t *data))
 {
   int circ_needs_uptime;
   time_t now = time(NULL);
@@ -388,10 +390,7 @@ launch_rendezvous_point_circuit(const hs_service_t *service,
                                         &data->onion_pk,
                                         service->config.is_single_onion);
   if (info == NULL) {
-    /* We are done here, we can't extend to the rendezvous point.
-     * If you're running an IPv6-only v3 single onion service on 0.3.2 or with
-     * 0.3.2 clients, and somehow disable the option check, it will fail here.
-     */
+    /* We are done here, we can't extend to the rendezvous point. */
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Not enough info to open a circuit to a rendezvous point for "
            "%s service %s.",
@@ -473,7 +472,7 @@ launch_rendezvous_point_circuit(const hs_service_t *service,
   extend_info_free(info);
 }
 
-/* Return true iff the given service rendezvous circuit circ is allowed for a
+/** Return true iff the given service rendezvous circuit circ is allowed for a
  * relaunch to the rendezvous point. */
 static int
 can_relaunch_service_rendezvous_point(const origin_circuit_t *circ)
@@ -520,7 +519,7 @@ can_relaunch_service_rendezvous_point(const origin_circuit_t *circ)
   return 0;
 }
 
-/* Retry the rendezvous point of circ by launching a new circuit to it. */
+/** Retry the rendezvous point of circ by launching a new circuit to it. */
 static void
 retry_service_rendezvous_point(const origin_circuit_t *circ)
 {
@@ -569,82 +568,7 @@ retry_service_rendezvous_point(const origin_circuit_t *circ)
   return;
 }
 
-/* Add all possible link specifiers in node to lspecs:
- *  - legacy ID is mandatory thus MUST be present in node;
- *  - include ed25519 link specifier if present in the node, and the node
- *    supports ed25519 link authentication, even if its link versions are not
- *    compatible with us;
- *  - include IPv4 link specifier, if the primary address is not IPv4, log a
- *    BUG() warning, and return an empty smartlist;
- *  - include IPv6 link specifier if present in the node. */
-static void
-get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
-{
-  link_specifier_t *ls;
-  tor_addr_port_t ap;
-
-  tor_assert(node);
-  tor_assert(lspecs);
-
-  /* Get the relay's IPv4 address. */
-  node_get_prim_orport(node, &ap);
-
-  /* We expect the node's primary address to be a valid IPv4 address.
-   * This conforms to the protocol, which requires either an IPv4 or IPv6
-   * address (or both). */
-  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
-      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
-    return;
-  }
-
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_IPV4);
-  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
-  link_specifier_set_un_ipv4_port(ls, ap.port);
-  /* Four bytes IPv4 and two bytes port. */
-  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
-                            sizeof(ap.port));
-  smartlist_add(lspecs, ls);
-
-  /* Legacy ID is mandatory and will always be present in node. */
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
-  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
-         link_specifier_getlen_un_legacy_id(ls));
-  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
-  smartlist_add(lspecs, ls);
-
-  /* ed25519 ID is only included if the node has it, and the node declares a
-     protocol version that supports ed25519 link authentication, even if that
-     link version is not compatible with us. (We are sending the ed25519 key
-     to another tor, which may support different link versions.) */
-  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
-      node_supports_ed25519_link_authentication(node, 0)) {
-    ls = link_specifier_new();
-    link_specifier_set_ls_type(ls, LS_ED25519_ID);
-    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
-           link_specifier_getlen_un_ed25519_id(ls));
-    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
-    smartlist_add(lspecs, ls);
-  }
-
-  /* Check for IPv6. If so, include it as well. */
-  if (node_has_ipv6_orport(node)) {
-    ls = link_specifier_new();
-    node_get_pref_ipv6_orport(node, &ap);
-    link_specifier_set_ls_type(ls, LS_IPV6);
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
-    smartlist_add(lspecs, ls);
-  }
-}
-
-/* Using the given descriptor intro point ip, the node of the
+/** Using the given descriptor intro point ip, the node of the
  * rendezvous point rp_node and the service's subcredential, populate the
  * already allocated intro1_data object with the needed key material and link
  * specifiers.
@@ -655,7 +579,7 @@ get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
 static int
 setup_introduce1_data(const hs_desc_intro_point_t *ip,
                       const node_t *rp_node,
-                      const uint8_t *subcredential,
+                      const hs_subcredential_t *subcredential,
                       hs_cell_introduce1_data_t *intro1_data)
 {
   int ret = -1;
@@ -666,10 +590,9 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip,
   tor_assert(subcredential);
   tor_assert(intro1_data);
 
-  /* Build the link specifiers from the extend information of the rendezvous
-   * circuit that we've picked previously. */
-  rp_lspecs = smartlist_new();
-  get_lspecs_from_node(rp_node, rp_lspecs);
+  /* Build the link specifiers from the node at the end of the rendezvous
+   * circuit that we opened for this introduction. */
+  rp_lspecs = node_get_link_specifier_smartlist(rp_node, 0);
   if (smartlist_len(rp_lspecs) == 0) {
     /* We can't rendezvous without link specifiers. */
     smartlist_free(rp_lspecs);
@@ -698,11 +621,41 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip,
   return ret;
 }
 
+/** Helper: cleanup function for client circuit. This is for every HS version.
+ * It is called from hs_circ_cleanup_on_close() entry point. */
+static void
+cleanup_on_close_client_circ(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circuit_is_hs_v3(circ)) {
+    hs_client_circuit_cleanup_on_close(circ);
+  }
+  /* It is possible the circuit has an HS purpose but no identifier (rend_data
+   * or hs_ident). Thus possible that this passes through. */
+}
+
+/** Helper: cleanup function for client circuit. This is for every HS version.
+ * It is called from hs_circ_cleanup_on_free() entry point. */
+static void
+cleanup_on_free_client_circ(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circuit_is_hs_v2(circ)) {
+    rend_client_circuit_cleanup_on_free(circ);
+  } else if (circuit_is_hs_v3(circ)) {
+    hs_client_circuit_cleanup_on_free(circ);
+  }
+  /* It is possible the circuit has an HS purpose but no identifier (rend_data
+   * or hs_ident). Thus possible that this passes through. */
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
-/* Return an introduction point circuit matching the given intro point object.
+/** Return an introduction point circuit matching the given intro point object.
  * NULL is returned is no such circuit can be found. */
 origin_circuit_t *
 hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip)
@@ -717,7 +670,29 @@ hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip)
   }
 }
 
-/* Called when we fail building a rendezvous circuit at some point other than
+/** Return an introduction point established circuit matching the given intro
+ * point object. The circuit purpose has to be CIRCUIT_PURPOSE_S_INTRO. NULL
+ * is returned is no such circuit can be found. */
+origin_circuit_t *
+hs_circ_service_get_established_intro_circ(const hs_service_intro_point_t *ip)
+{
+  origin_circuit_t *circ;
+
+  tor_assert(ip);
+
+  if (ip->base.is_only_legacy) {
+    circ = hs_circuitmap_get_intro_circ_v2_service_side(ip->legacy_key_digest);
+  } else {
+    circ = hs_circuitmap_get_intro_circ_v3_service_side(
+                                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* Only return circuit if it is established. */
+  return (circ && TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO) ?
+          circ : NULL;
+}
+
+/** Called when we fail building a rendezvous circuit at some point other than
  * the last hop: launches a new circuit to the same rendezvous point. This
  * supports legacy service.
  *
@@ -757,7 +732,7 @@ hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ)
   return;
 }
 
-/* For a given service and a service intro point, launch a circuit to the
+/** For a given service and a service intro point, launch a circuit to the
  * extend info ei. If the service is a single onion, and direct_conn is true,
  * a one-hop circuit will be requested.
  *
@@ -818,7 +793,7 @@ hs_circ_launch_intro_point(hs_service_t *service,
   return ret;
 }
 
-/* Called when a service introduction point circuit is done building. Given
+/** Called when a service introduction point circuit is done building. Given
  * the service and intro point object, this function will send the
  * ESTABLISH_INTRO cell on the circuit. Return 0 on success. Return 1 if the
  * circuit has been repurposed to General because we already have too many
@@ -887,7 +862,7 @@ hs_circ_service_intro_has_opened(hs_service_t *service,
   return ret;
 }
 
-/* Called when a service rendezvous point circuit is done building. Given the
+/** Called when a service rendezvous point circuit is done building. Given the
  * service and the circuit, this function will send a RENDEZVOUS1 cell on the
  * circuit using the information in the circuit identifier. If the cell can't
  * be sent, the circuit is closed. */
@@ -953,7 +928,7 @@ hs_circ_service_rp_has_opened(const hs_service_t *service,
   memwipe(payload, 0, sizeof(payload));
 }
 
-/* Circ has been expecting an INTRO_ESTABLISHED cell that just arrived. Handle
+/** Circ has been expecting an INTRO_ESTABLISHED cell that just arrived. Handle
  * the INTRO_ESTABLISHED cell payload of length payload_len arriving on the
  * given introduction circuit circ. The service is only used for logging
  * purposes. Return 0 on success else a negative value. */
@@ -998,7 +973,43 @@ hs_circ_handle_intro_established(const hs_service_t *service,
   return ret;
 }
 
-/* We just received an INTRODUCE2 cell on the established introduction circuit
+/**
+ *  Go into <b>data</b> and add the right subcredential to be able to handle
+ *  this incoming cell.
+ *
+ *  <b>desc_subcred</b> is the subcredential of the descriptor that corresponds
+ *  to the intro point that received this intro request. This subcredential
+ *  should be used if we are not an onionbalance instance.
+ *
+ *  Return 0 if everything went well, or -1 in case of internal error.
+ */
+static int
+get_subcredential_for_handling_intro2_cell(const hs_service_t *service,
+                                        hs_cell_introduce2_data_t *data,
+                                        const hs_subcredential_t *desc_subcred)
+{
+  /* Handle the simple case first: We are not an onionbalance instance and we
+   * should just use the regular descriptor subcredential */
+  if (!hs_ob_service_is_instance(service)) {
+    data->n_subcredentials = 1;
+    data->subcredentials = desc_subcred;
+    return 0;
+  }
+
+  /* This should not happen since we should have made onionbalance
+   * subcredentials when we created our descriptors. */
+  if (BUG(!service->state.ob_subcreds)) {
+    return -1;
+  }
+
+  /* We are an onionbalance instance: */
+  data->n_subcredentials = service->state.n_ob_subcreds;
+  data->subcredentials = service->state.ob_subcreds;
+
+  return 0;
+}
+
+/** We just received an INTRODUCE2 cell on the established introduction circuit
  * circ.  Handle the INTRODUCE2 payload of size payload_len for the given
  * circuit and service. This cell is associated with the intro point object ip
  * and the subcredential. Return 0 on success else a negative value. */
@@ -1006,7 +1017,7 @@ int
 hs_circ_handle_introduce2(const hs_service_t *service,
                           const origin_circuit_t *circ,
                           hs_service_intro_point_t *ip,
-                          const uint8_t *subcredential,
+                          const hs_subcredential_t *subcredential,
                           const uint8_t *payload, size_t payload_len)
 {
   int ret = -1;
@@ -1023,12 +1034,16 @@ hs_circ_handle_introduce2(const hs_service_t *service,
    * parsed, decrypted and key material computed correctly. */
   data.auth_pk = &ip->auth_key_kp.pubkey;
   data.enc_kp = &ip->enc_key_kp;
-  data.subcredential = subcredential;
   data.payload = payload;
   data.payload_len = payload_len;
   data.link_specifiers = smartlist_new();
   data.replay_cache = ip->replay_cache;
 
+  if (get_subcredential_for_handling_intro2_cell(service,
+                                                 &data, subcredential)) {
+    goto done;
+  }
+
   if (hs_cell_parse_introduce2(&data, circ, service) < 0) {
     goto done;
   }
@@ -1060,14 +1075,12 @@ hs_circ_handle_introduce2(const hs_service_t *service,
   ret = 0;
 
  done:
-  SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec,
-                    link_specifier_free(lspec));
-  smartlist_free(data.link_specifiers);
+  link_specifier_smartlist_free(data.link_specifiers);
   memwipe(&data, 0, sizeof(data));
   return ret;
 }
 
-/* Circuit <b>circ</b> just finished the rend ntor key exchange. Use the key
+/** Circuit <b>circ</b> just finished the rend ntor key exchange. Use the key
  * exchange output material at <b>ntor_key_seed</b> and setup <b>circ</b> to
  * serve as a rendezvous end-to-end circuit between the client and the
  * service. If <b>is_service_side</b> is set, then we are the hidden service
@@ -1097,7 +1110,7 @@ hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
   return 0;
 }
 
-/* We are a v2 legacy HS client and we just received a RENDEZVOUS1 cell
+/** We are a v2 legacy HS client and we just received a RENDEZVOUS1 cell
  * <b>rend_cell_body</b> on <b>circ</b>. Finish up the DH key exchange and then
  * extend the crypt path of <b>circ</b> so that the hidden service is on the
  * other side. */
@@ -1122,7 +1135,7 @@ hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
   return 0;
 }
 
-/* Given the introduction circuit intro_circ, the rendezvous circuit
+/** Given the introduction circuit intro_circ, the rendezvous circuit
  * rend_circ, a descriptor intro point object ip and the service's
  * subcredential, send an INTRODUCE1 cell on intro_circ.
  *
@@ -1134,7 +1147,7 @@ int
 hs_circ_send_introduce1(origin_circuit_t *intro_circ,
                         origin_circuit_t *rend_circ,
                         const hs_desc_intro_point_t *ip,
-                        const uint8_t *subcredential)
+                        const hs_subcredential_t *subcredential)
 {
   int ret = -1;
   ssize_t payload_len;
@@ -1207,7 +1220,7 @@ hs_circ_send_introduce1(origin_circuit_t *intro_circ,
   return ret;
 }
 
-/* Send an ESTABLISH_RENDEZVOUS cell along the rendezvous circuit circ. On
+/** Send an ESTABLISH_RENDEZVOUS cell along the rendezvous circuit circ. On
  * success, 0 is returned else -1 and the circuit is marked for close. */
 int
 hs_circ_send_establish_rendezvous(origin_circuit_t *circ)
@@ -1258,30 +1271,136 @@ hs_circ_send_establish_rendezvous(origin_circuit_t *circ)
   return -1;
 }
 
-/* We are about to close or free this <b>circ</b>. Clean it up from any
- * related HS data structures. This function can be called multiple times
- * safely for the same circuit. */
+/** Circuit cleanup strategy:
+ *
+ *  What follows is a series of functions that notifies the HS subsystem of 3
+ *  different circuit cleanup phase: close, free and repurpose.
+ *
+ *  Tor can call any of those in any orders so they have to be safe between
+ *  each other. In other words, the free should never depend on close to be
+ *  called before.
+ *
+ *  The "on_close()" is called from circuit_mark_for_close() which is
+ *  considered the tor fast path and thus as little work as possible should
+ *  done in that function. Currently, we only remove the circuit from the HS
+ *  circuit map and move on.
+ *
+ *  The "on_free()" is called from circuit circuit_free_() and it is very
+ *  important that at the end of the function, no state or objects related to
+ *  this circuit remains alive.
+ *
+ *  The "on_repurpose()" is called from circuit_change_purpose() for which we
+ *  simply remove it from the HS circuit map. We do not have other cleanup
+ *  requirements after that.
+ *
+ *  NOTE: The onion service code, specifically the service code, cleans up
+ *  lingering objects or state if any of its circuit disappear which is why
+ *  our cleanup strategy doesn't involve any service specific actions. As long
+ *  as the circuit is removed from the HS circuit map, it won't be used.
+ */
+
+/** We are about to close this <b>circ</b>. Clean it up from any related HS
+ * data structures. This function can be called multiple times safely for the
+ * same circuit. */
 void
-hs_circ_cleanup(circuit_t *circ)
+hs_circ_cleanup_on_close(circuit_t *circ)
 {
   tor_assert(circ);
 
-  /* If it's a service-side intro circ, notify the HS subsystem for the intro
-   * point circuit closing so it can be dealt with cleanly. */
-  if (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
-      circ->purpose == CIRCUIT_PURPOSE_S_INTRO) {
-    hs_service_intro_circ_has_closed(TO_ORIGIN_CIRCUIT(circ));
+  if (circuit_purpose_is_hs_client(circ->purpose)) {
+    cleanup_on_close_client_circ(circ);
   }
 
-  /* Clear HS circuitmap token for this circ (if any). Very important to be
-   * done after the HS subsystem has been notified of the close else the
-   * circuit will not be found.
-   *
-   * We do this at the close if possible because from that point on, the
-   * circuit is good as dead. We can't rely on removing it in the circuit
-   * free() function because we open a race window between the close and free
-   * where we can't register a new circuit for the same intro point. */
+  /* On close, we simply remove it from the circuit map. It can not be used
+   * anymore. We keep this code path fast and lean. */
+
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+}
+
+/** We are about to free this <b>circ</b>. Clean it up from any related HS
+ * data structures. This function can be called multiple times safely for the
+ * same circuit. */
+void
+hs_circ_cleanup_on_free(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* NOTE: Bulk of the work of cleaning up a circuit is done here. */
+
+  if (circuit_purpose_is_hs_client(circ->purpose)) {
+    cleanup_on_free_client_circ(circ);
+  }
+
+  /* We have no assurance that the given HS circuit has been closed before and
+   * thus removed from the HS map. This actually happens in unit tests. */
   if (circ->hs_token) {
     hs_circuitmap_remove_circuit(circ);
   }
 }
+
+/** We are about to repurpose this <b>circ</b>. Clean it up from any related
+ * HS data structures. This function can be called multiple times safely for
+ * the same circuit. */
+void
+hs_circ_cleanup_on_repurpose(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* On repurpose, we simply remove it from the circuit map but we do not do
+   * the on_free actions since we don't treat a repurpose as something we need
+   * to report in the client cache failure. */
+
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+}
+
+/** Return true iff the given established client rendezvous circuit was sent
+ * into the INTRODUCE1 cell. This is called so we can take a decision on
+ * expiring or not the circuit.
+ *
+ * The caller MUST make sure the circuit is an established client rendezvous
+ * circuit (purpose: CIRCUIT_PURPOSE_C_REND_READY).
+ *
+ * This function supports all onion service versions. */
+bool
+hs_circ_is_rend_sent_in_intro1(const origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  /* This can only be called for a rendezvous circuit that is an established
+   * confirmed rendezsvous circuit but without an introduction ACK. */
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_REND_READY);
+
+  /* The v2 and v3 circuit are handled differently:
+   *
+   * v2: A circ's pending_final_cpath field is non-NULL iff it is a rend circ
+   * and we have tried to send an INTRODUCE1 cell specifying it. Thus, if the
+   * pending_final_cpath field *is* NULL, then we want to not spare it.
+   *
+   * v3: When the INTRODUCE1 cell is sent, the introduction encryption public
+   * key is copied in the rendezvous circuit hs identifier. If it is a valid
+   * key, we know that this circuit is waiting the ACK on the introduction
+   * circuit. We want to _not_ spare the circuit if the key was never set. */
+
+  if (circ->rend_data) {
+    /* v2. */
+    if (circ->build_state && circ->build_state->pending_final_cpath != NULL) {
+      return true;
+    }
+  } else if (circ->hs_ident) {
+    /* v3. */
+    if (curve25519_public_key_is_ok(&circ->hs_ident->intro_enc_pk)) {
+      return true;
+    }
+  } else {
+    /* A circuit with an HS purpose without an hs_ident or rend_data in theory
+     * can not happen. In case, scream loudly and return false to the caller
+     * that the rendezvous was not sent in the INTRO1 cell. */
+    tor_assert_nonfatal_unreached();
+  }
+
+  /* The rendezvous has not been specified in the INTRODUCE1 cell. */
+  return false;
+}
diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h
index e168b301f1..22e936e685 100644
--- a/src/feature/hs/hs_circuit.h
+++ b/src/feature/hs/hs_circuit.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,8 +14,10 @@
 
 #include "feature/hs/hs_service.h"
 
-/* Cleanup function when the circuit is closed or/and freed. */
-void hs_circ_cleanup(circuit_t *circ);
+/* Cleanup function when the circuit is closed or freed. */
+void hs_circ_cleanup_on_close(circuit_t *circ);
+void hs_circ_cleanup_on_free(circuit_t *circ);
+void hs_circ_cleanup_on_repurpose(circuit_t *circ);
 
 /* Circuit API. */
 int hs_circ_service_intro_has_opened(hs_service_t *service,
@@ -35,6 +37,8 @@ void hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ);
 
 origin_circuit_t *hs_circ_service_get_intro_circ(
                                       const hs_service_intro_point_t *ip);
+origin_circuit_t *hs_circ_service_get_established_intro_circ(
+                                      const hs_service_intro_point_t *ip);
 
 /* Cell API. */
 int hs_circ_handle_intro_established(const hs_service_t *service,
@@ -42,15 +46,16 @@ int hs_circ_handle_intro_established(const hs_service_t *service,
                                      origin_circuit_t *circ,
                                      const uint8_t *payload,
                                      size_t payload_len);
+struct hs_subcredential_t;
 int hs_circ_handle_introduce2(const hs_service_t *service,
                               const origin_circuit_t *circ,
                               hs_service_intro_point_t *ip,
-                              const uint8_t *subcredential,
+                              const struct hs_subcredential_t *subcredential,
                               const uint8_t *payload, size_t payload_len);
 int hs_circ_send_introduce1(origin_circuit_t *intro_circ,
                             origin_circuit_t *rend_circ,
                             const hs_desc_intro_point_t *ip,
-                            const uint8_t *subcredential);
+                            const struct hs_subcredential_t *subcredential);
 int hs_circ_send_establish_rendezvous(origin_circuit_t *circ);
 
 /* e2e circuit API. */
@@ -62,15 +67,24 @@ int hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
 int hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
                                           const uint8_t *rend_cell_body);
 
+bool hs_circ_is_rend_sent_in_intro1(const origin_circuit_t *circ);
+
 #ifdef HS_CIRCUIT_PRIVATE
 
+struct hs_ntor_rend_cell_keys_t;
+
 STATIC hs_ident_circuit_t *
 create_rp_circuit_identifier(const hs_service_t *service,
                              const uint8_t *rendezvous_cookie,
                              const curve25519_public_key_t *server_pk,
-                             const hs_ntor_rend_cell_keys_t *keys);
+                             const struct hs_ntor_rend_cell_keys_t *keys);
+
+struct hs_cell_introduce2_data_t;
+MOCK_DECL(STATIC void,
+launch_rendezvous_point_circuit,(const hs_service_t *service,
+                                 const hs_service_intro_point_t *ip,
+                                const struct hs_cell_introduce2_data_t *data));
 
 #endif /* defined(HS_CIRCUIT_PRIVATE) */
 
 #endif /* !defined(TOR_HS_CIRCUIT_H) */
-
diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c
index 5480d5eb84..466a02de39 100644
--- a/src/feature/hs/hs_circuitmap.c
+++ b/src/feature/hs/hs_circuitmap.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,13 +23,13 @@
 
 /************************** HS circuitmap code *******************************/
 
-/* This is the hidden service circuitmap. It's a hash table that maps
+/** This is the hidden service circuitmap. It's a hash table that maps
    introduction and rendezvous tokens to specific circuits such that given a
    token it's easy to find the corresponding circuit. */
 static struct hs_circuitmap_ht *the_hs_circuitmap = NULL;
 
-/* This is a helper function used by the hash table code (HT_). It returns 1 if
- * two circuits have the same HS token. */
+/** This is a helper function used by the hash table code (HT_). It returns 1
+ * if two circuits have the same HS token. */
 static int
 hs_circuits_have_same_token(const circuit_t *first_circuit,
                             const circuit_t *second_circuit)
@@ -60,8 +60,9 @@ hs_circuits_have_same_token(const circuit_t *first_circuit,
                    first_token->token_len);
 }
 
-/* This is a helper function for the hash table code (HT_). It hashes a circuit
- * HS token into an unsigned int for use as a key by the hash table routines.*/
+/** This is a helper function for the hash table code (HT_). It hashes a
+ * circuit HS token into an unsigned int for use as a key by the hash table
+ * routines.*/
 static inline unsigned int
 hs_circuit_hash_token(const circuit_t *circuit)
 {
@@ -71,19 +72,19 @@ hs_circuit_hash_token(const circuit_t *circuit)
                                circuit->hs_token->token_len);
 }
 
-/* Register the circuitmap hash table */
+/** Register the circuitmap hash table */
 HT_PROTOTYPE(hs_circuitmap_ht, // The name of the hashtable struct
              circuit_t,    // The name of the element struct,
              hs_circuitmap_node,        // The name of HT_ENTRY member
-             hs_circuit_hash_token, hs_circuits_have_same_token)
+             hs_circuit_hash_token, hs_circuits_have_same_token);
 
 HT_GENERATE2(hs_circuitmap_ht, circuit_t, hs_circuitmap_node,
              hs_circuit_hash_token, hs_circuits_have_same_token,
-             0.6, tor_reallocarray, tor_free_)
+             0.6, tor_reallocarray, tor_free_);
 
 #ifdef TOR_UNIT_TESTS
 
-/* Return the global HS circuitmap. Used by unittests. */
+/** Return the global HS circuitmap. Used by unittests. */
 hs_circuitmap_ht *
 get_hs_circuitmap(void)
 {
@@ -136,7 +137,7 @@ get_circuit_with_token(hs_token_t *search_token)
   return HT_FIND(hs_circuitmap_ht, the_hs_circuitmap, &search_circ);
 }
 
-/* Helper function that registers <b>circ</b> with <b>token</b> on the HS
+/** Helper function that registers <b>circ</b> with <b>token</b> on the HS
    circuitmap. This function steals reference of <b>token</b>. */
 static void
 hs_circuitmap_register_impl(circuit_t *circ, hs_token_t *token)
@@ -186,7 +187,7 @@ hs_circuitmap_register_circuit(circuit_t *circ,
   hs_circuitmap_register_impl(circ, hs_token);
 }
 
-/* Helper function for hs_circuitmap_get_origin_circuit() and
+/** Helper function for hs_circuitmap_get_origin_circuit() and
  * hs_circuitmap_get_or_circuit(). Because only circuit_t are indexed in the
  * circuitmap, this function returns object type so the specialized functions
  * using this helper can upcast it to the right type.
@@ -220,7 +221,7 @@ hs_circuitmap_get_circuit_impl(hs_token_type_t type,
   return found_circ;
 }
 
-/* Helper function: Query circuitmap for origin circuit with <b>token</b> of
+/** Helper function: Query circuitmap for origin circuit with <b>token</b> of
  * size <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose
  * equal to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked
  * for close. Return NULL if no such circuit is found. */
@@ -244,7 +245,7 @@ hs_circuitmap_get_origin_circuit(hs_token_type_t type,
   return TO_ORIGIN_CIRCUIT(circ);
 }
 
-/* Helper function: Query circuitmap for OR circuit with <b>token</b> of size
+/** Helper function: Query circuitmap for OR circuit with <b>token</b> of size
  * <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose equal
  * to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked for
  * close. Return NULL if no such circuit is found. */
@@ -272,7 +273,34 @@ hs_circuitmap_get_or_circuit(hs_token_type_t type,
 
 /**** Public relay-side getters: */
 
-/* Public function: Return a v3 introduction circuit to this relay with
+/** Public function: Return v2 and v3 introduction circuit to this relay.
+ * Always return a newly allocated list for which it is the caller's
+ * responsability to free it. */
+smartlist_t *
+hs_circuitmap_get_all_intro_circ_relay_side(void)
+{
+  circuit_t **iter;
+  smartlist_t *circuit_list = smartlist_new();
+
+  HT_FOREACH(iter, hs_circuitmap_ht, the_hs_circuitmap) {
+    circuit_t *circ = *iter;
+
+    /* An origin circuit or purpose is wrong or the hs token is not set to be
+     * a v2 or v3 intro relay side type, we ignore the circuit. Else, we have
+     * a match so add it to our list. */
+    if (CIRCUIT_IS_ORIGIN(circ) ||
+        circ->purpose != CIRCUIT_PURPOSE_INTRO_POINT ||
+        (circ->hs_token->type != HS_TOKEN_INTRO_V3_RELAY_SIDE &&
+         circ->hs_token->type != HS_TOKEN_INTRO_V2_RELAY_SIDE)) {
+      continue;
+    }
+    smartlist_add(circuit_list, circ);
+  }
+
+  return circuit_list;
+}
+
+/** Public function: Return a v3 introduction circuit to this relay with
  * <b>auth_key</b>. Return NULL if no such circuit is found in the
  * circuitmap. */
 or_circuit_t *
@@ -284,7 +312,7 @@ hs_circuitmap_get_intro_circ_v3_relay_side(
                                       CIRCUIT_PURPOSE_INTRO_POINT);
 }
 
-/* Public function: Return v2 introduction circuit to this relay with
+/** Public function: Return v2 introduction circuit to this relay with
  * <b>digest</b>. Return NULL if no such circuit is found in the circuitmap. */
 or_circuit_t *
 hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
@@ -294,7 +322,7 @@ hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
                                       CIRCUIT_PURPOSE_INTRO_POINT);
 }
 
-/* Public function: Return rendezvous circuit to this relay with rendezvous
+/** Public function: Return rendezvous circuit to this relay with rendezvous
  * <b>cookie</b>. Return NULL if no such circuit is found in the circuitmap. */
 or_circuit_t *
 hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
@@ -306,7 +334,7 @@ hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
 
 /** Public relay-side setters: */
 
-/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+/** Public function: Register rendezvous circuit with key <b>cookie</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
@@ -316,7 +344,7 @@ hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
                                  HS_TOKEN_REND_RELAY_SIDE,
                                  REND_TOKEN_LEN, cookie);
 }
-/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+/** Public function: Register v2 intro circuit with key <b>digest</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
@@ -327,7 +355,7 @@ hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
                                  REND_TOKEN_LEN, digest);
 }
 
-/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+/** Public function: Register v3 intro circuit with key <b>auth_key</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
@@ -340,7 +368,7 @@ hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
 
 /**** Public servide-side getters: */
 
-/* Public function: Return v3 introduction circuit with <b>auth_key</b>
+/** Public function: Return v3 introduction circuit with <b>auth_key</b>
  * originating from this hidden service. Return NULL if no such circuit is
  * found in the circuitmap. */
 origin_circuit_t *
@@ -365,9 +393,9 @@ hs_circuitmap_get_intro_circ_v3_service_side(const
   return circ;
 }
 
-/* Public function: Return v2 introduction circuit originating from this hidden
- * service with <b>digest</b>. Return NULL if no such circuit is found in the
- * circuitmap. */
+/** Public function: Return v2 introduction circuit originating from this
+ * hidden service with <b>digest</b>. Return NULL if no such circuit is found
+ * in the circuitmap. */
 origin_circuit_t *
 hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
 {
@@ -389,7 +417,7 @@ hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
   return circ;
 }
 
-/* Public function: Return rendezvous circuit originating from this hidden
+/** Public function: Return rendezvous circuit originating from this hidden
  * service with rendezvous <b>cookie</b>. Return NULL if no such circuit is
  * found in the circuitmap. */
 origin_circuit_t *
@@ -412,7 +440,7 @@ hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie)
   return circ;
 }
 
-/* Public function: Return client-side rendezvous circuit with rendezvous
+/** Public function: Return client-side rendezvous circuit with rendezvous
  * <b>cookie</b>. It will look for circuits with the following purposes:
 
  * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
@@ -445,7 +473,7 @@ hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie)
   return circ;
 }
 
-/*  Public function: Return client-side established rendezvous circuit with
+/**  Public function: Return client-side established rendezvous circuit with
  *  rendezvous <b>cookie</b>. It will look for circuits with the following
  *  purposes:
  *
@@ -487,7 +515,7 @@ hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie)
 
 /**** Public servide-side setters: */
 
-/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+/** Public function: Register v2 intro circuit with key <b>digest</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
@@ -498,7 +526,7 @@ hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
                                  REND_TOKEN_LEN, digest);
 }
 
-/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+/** Public function: Register v3 intro circuit with key <b>auth_key</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
@@ -509,7 +537,7 @@ hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
                                  ED25519_PUBKEY_LEN, auth_key->pubkey);
 }
 
-/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+/** Public function: Register rendezvous circuit with key <b>cookie</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
@@ -520,7 +548,7 @@ hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
                                  REND_TOKEN_LEN, cookie);
 }
 
-/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+/** Public function: Register rendezvous circuit with key <b>cookie</b> to the
  * client-side circuitmap. */
 void
 hs_circuitmap_register_rend_circ_client_side(origin_circuit_t *or_circ,
@@ -564,7 +592,7 @@ hs_circuitmap_remove_circuit(circuit_t *circ)
   circ->hs_token = NULL;
 }
 
-/* Public function: Initialize the global HS circuitmap. */
+/** Public function: Initialize the global HS circuitmap. */
 void
 hs_circuitmap_init(void)
 {
@@ -574,7 +602,7 @@ hs_circuitmap_init(void)
   HT_INIT(hs_circuitmap_ht, the_hs_circuitmap);
 }
 
-/* Public function: Free all memory allocated by the global HS circuitmap. */
+/** Public function: Free all memory allocated by the global HS circuitmap. */
 void
 hs_circuitmap_free_all(void)
 {
diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h
index c1bbb1ff1c..df3e7a6e7e 100644
--- a/src/feature/hs/hs_circuitmap.h
+++ b/src/feature/hs/hs_circuitmap.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@ typedef HT_HEAD(hs_circuitmap_ht, circuit_t) hs_circuitmap_ht;
 typedef struct hs_token_t hs_token_t;
 struct or_circuit_t;
 struct origin_circuit_t;
+struct ed25519_public_key_t;
 
 /** Public HS circuitmap API: */
 
@@ -21,7 +22,7 @@ struct origin_circuit_t;
 
 struct or_circuit_t *
 hs_circuitmap_get_intro_circ_v3_relay_side(const
-                                           ed25519_public_key_t *auth_key);
+                                   struct ed25519_public_key_t *auth_key);
 struct or_circuit_t *
 hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest);
 struct or_circuit_t *
@@ -32,13 +33,15 @@ void hs_circuitmap_register_rend_circ_relay_side(struct or_circuit_t *circ,
 void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
                                                      const uint8_t *digest);
 void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
-                                         const ed25519_public_key_t *auth_key);
+                                 const struct ed25519_public_key_t *auth_key);
+
+smartlist_t *hs_circuitmap_get_all_intro_circ_relay_side(void);
 
 /** Public service-side API: */
 
 struct origin_circuit_t *
 hs_circuitmap_get_intro_circ_v3_service_side(const
-                                             ed25519_public_key_t *auth_key);
+                                     struct ed25519_public_key_t *auth_key);
 struct origin_circuit_t *
 hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest);
 struct origin_circuit_t *
@@ -52,8 +55,8 @@ void hs_circuitmap_register_intro_circ_v2_service_side(
                                         struct origin_circuit_t *circ,
                                         const uint8_t *digest);
 void hs_circuitmap_register_intro_circ_v3_service_side(
-                                        struct origin_circuit_t *circ,
-                                        const ed25519_public_key_t *auth_key);
+                                 struct origin_circuit_t *circ,
+                                 const struct ed25519_public_key_t *auth_key);
 void hs_circuitmap_register_rend_circ_service_side(
                                         struct origin_circuit_t *circ,
                                         const uint8_t *cookie);
diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c
index e25919ecb7..0f6109195b 100644
--- a/src/feature/hs/hs_client.c
+++ b/src/feature/hs/hs_client.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -43,14 +43,15 @@
 #include "core/or/entry_connection_st.h"
 #include "core/or/extend_info_st.h"
 #include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
 
-/* Client-side authorizations for hidden services; map of service identity
+/** Client-side authorizations for hidden services; map of service identity
  * public key to hs_client_service_authorization_t *. */
 static digest256map_t *client_auths = NULL;
 
 #include "trunnel/hs/cell_introduce1.h"
 
-/* Return a human-readable string for the client fetch status code. */
+/** Return a human-readable string for the client fetch status code. */
 static const char *
 fetch_status_to_string(hs_client_fetch_status_t status)
 {
@@ -74,7 +75,7 @@ fetch_status_to_string(hs_client_fetch_status_t status)
   }
 }
 
-/* Return true iff tor should close the SOCKS request(s) for the descriptor
+/** Return true iff tor should close the SOCKS request(s) for the descriptor
  * fetch that ended up with this given status code. */
 static int
 fetch_status_should_close_socks(hs_client_fetch_status_t status)
@@ -101,12 +102,51 @@ fetch_status_should_close_socks(hs_client_fetch_status_t status)
   return 1;
 }
 
+/* Return a newly allocated list of all the entry connections that matches the
+ * given service identity pk. If service_identity_pk is NULL, all entry
+ * connections with an hs_ident are returned.
+ *
+ * Caller must free the returned list but does NOT have ownership of the
+ * object inside thus they have to remain untouched. */
+static smartlist_t *
+find_entry_conns(const ed25519_public_key_t *service_identity_pk)
+{
+  time_t now = time(NULL);
+  smartlist_t *conns = NULL, *entry_conns = NULL;
+
+  entry_conns = smartlist_new();
+
+  conns = connection_list_by_type_state(CONN_TYPE_AP,
+                                        AP_CONN_STATE_RENDDESC_WAIT);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we just fetched its descriptor. */
+    if (!edge_conn->hs_ident ||
+        (service_identity_pk &&
+         !ed25519_pubkey_eq(service_identity_pk,
+                            &edge_conn->hs_ident->identity_pk))) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+
+    /* Validated! Add the entry connection to the list. */
+    smartlist_add(entry_conns, entry_conn);
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(conns);
+  return entry_conns;
+}
+
 /* Cancel all descriptor fetches currently in progress. */
 static void
 cancel_descriptor_fetches(void)
 {
   smartlist_t *conns =
-    connection_list_by_type_state(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+    connection_list_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
   SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
     const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
     if (BUG(ident == NULL)) {
@@ -125,7 +165,7 @@ cancel_descriptor_fetches(void)
   log_info(LD_REND, "Hidden service client descriptor fetches cancelled.");
 }
 
-/* Get all connections that are waiting on a circuit and flag them back to
+/** Get all connections that are waiting on a circuit and flag them back to
  * waiting for a hidden service descriptor for the given service key
  * service_identity_pk. */
 static void
@@ -152,7 +192,7 @@ flag_all_conn_wait_desc(const ed25519_public_key_t *service_identity_pk)
   smartlist_free(conns);
 }
 
-/* Remove tracked HSDir requests from our history for this hidden service
+/** Remove tracked HSDir requests from our history for this hidden service
  * identity public key. */
 static void
 purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
@@ -168,14 +208,12 @@ purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
    * some point and we don't care about those anymore. */
   hs_build_blinded_pubkey(identity_pk, NULL, 0,
                           hs_get_time_period_num(0), &blinded_pk);
-  if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   /* Purge last hidden service request from cache for this blinded key. */
   hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk);
 }
 
-/* Return true iff there is at least one pending directory descriptor request
+/** Return true iff there is at least one pending directory descriptor request
  * for the service identity_pk. */
 static int
 directory_request_is_pending(const ed25519_public_key_t *identity_pk)
@@ -203,7 +241,7 @@ directory_request_is_pending(const ed25519_public_key_t *identity_pk)
   return ret;
 }
 
-/* Helper function that changes the state of an entry connection to waiting
+/** Helper function that changes the state of an entry connection to waiting
  * for a circuit. For this to work properly, the connection timestamps are set
  * to now and the connection is then marked as pending for a circuit. */
 static void
@@ -223,7 +261,7 @@ mark_conn_as_waiting_for_circuit(connection_t *conn, time_t now)
   connection_ap_mark_as_pending_circuit(TO_ENTRY_CONN(conn));
 }
 
-/* We failed to fetch a descriptor for the service with <b>identity_pk</b>
+/** We failed to fetch a descriptor for the service with <b>identity_pk</b>
  * because of <b>status</b>. Find all pending SOCKS connections for this
  * service that are waiting on the descriptor and close them with
  * <b>reason</b>. */
@@ -233,26 +271,13 @@ close_all_socks_conns_waiting_for_desc(const ed25519_public_key_t *identity_pk,
                                        int reason)
 {
   unsigned int count = 0;
-  time_t now = approx_time();
-  smartlist_t *conns =
-    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+  smartlist_t *entry_conns = find_entry_conns(identity_pk);
 
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
-    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
-
-    /* Only consider the entry connections that matches the service for which
-     * we tried to get the descriptor */
-    if (!edge_conn->hs_ident ||
-        !ed25519_pubkey_eq(identity_pk,
-                           &edge_conn->hs_ident->identity_pk)) {
-      continue;
-    }
-    assert_connection_ok(base_conn, now);
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
     /* Unattach the entry connection which will close for the reason. */
     connection_mark_unattached_ap(entry_conn, reason);
     count++;
-  } SMARTLIST_FOREACH_END(base_conn);
+  } SMARTLIST_FOREACH_END(entry_conn);
 
   if (count > 0) {
     char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
@@ -265,26 +290,26 @@ close_all_socks_conns_waiting_for_desc(const ed25519_public_key_t *identity_pk,
   }
 
   /* No ownership of the object(s) in this list. */
-  smartlist_free(conns);
+  smartlist_free(entry_conns);
 }
 
-/* Find all pending SOCKS connection waiting for a descriptor and retry them
+/** Find all pending SOCKS connection waiting for a descriptor and retry them
  * all. This is called when the directory information changed. */
 STATIC void
 retry_all_socks_conn_waiting_for_desc(void)
 {
-  smartlist_t *conns =
-    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+  smartlist_t *entry_conns = find_entry_conns(NULL);
 
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
     hs_client_fetch_status_t status;
-    const edge_connection_t *edge_conn =
-      ENTRY_TO_EDGE_CONN(TO_ENTRY_CONN(base_conn));
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+    connection_t *base_conn = &edge_conn->base_;
 
     /* Ignore non HS or non v3 connection. */
     if (edge_conn->hs_ident == NULL) {
       continue;
     }
+
     /* In this loop, we will possibly try to fetch a descriptor for the
      * pending connections because we just got more directory information.
      * However, the refetch process can cleanup all SOCKS request to the same
@@ -318,13 +343,13 @@ retry_all_socks_conn_waiting_for_desc(void)
      * closed or we are still missing directory information. Leave the
      * connection in renddesc wait state so when we get more info, we'll be
      * able to try it again. */
-  } SMARTLIST_FOREACH_END(base_conn);
+  } SMARTLIST_FOREACH_END(entry_conn);
 
   /* We don't have ownership of those objects. */
-  smartlist_free(conns);
+  smartlist_free(entry_conns);
 }
 
-/* A v3 HS circuit successfully connected to the hidden service. Update the
+/** A v3 HS circuit successfully connected to the hidden service. Update the
  * stream state at <b>hs_conn_ident</b> appropriately. */
 static void
 note_connection_attempt_succeeded(const hs_ident_edge_conn_t *hs_conn_ident)
@@ -346,7 +371,7 @@ note_connection_attempt_succeeded(const hs_ident_edge_conn_t *hs_conn_ident)
    * will be reset and thus possible to be retried. */
 }
 
-/* Given the pubkey of a hidden service in <b>onion_identity_pk</b>, fetch its
+/** Given the pubkey of a hidden service in <b>onion_identity_pk</b>, fetch its
  * descriptor by launching a dir connection to <b>hsdir</b>. Return a
  * hs_client_fetch_status_t status code depending on how it went. */
 static hs_client_fetch_status_t
@@ -357,7 +382,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   ed25519_public_key_t blinded_pubkey;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   hs_ident_dir_conn_t hs_conn_dir_ident;
-  int retval;
 
   tor_assert(hsdir);
   tor_assert(onion_identity_pk);
@@ -366,10 +390,7 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return HS_CLIENT_FETCH_ERROR;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Copy onion pk to a dir_ident so that we attach it to the dir conn */
   hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey,
@@ -408,7 +429,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
 STATIC routerstatus_t *
 pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 {
-  int retval;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   uint64_t current_time_period = hs_get_time_period_num(0);
   smartlist_t *responsible_hsdirs = NULL;
@@ -421,10 +441,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return NULL;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Get responsible hsdirs of service for this time period */
   responsible_hsdirs = smartlist_new();
@@ -437,7 +454,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 
   /* Pick an HSDir from the responsible ones. The ownership of
    * responsible_hsdirs is given to this function so no need to free it. */
-  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey);
+  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey, NULL);
 
   return hsdir_rs;
 }
@@ -462,7 +479,25 @@ fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk))
   return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs);
 }
 
-/* Make sure that the given v3 origin circuit circ is a valid correct
+/** With a given <b>onion_identity_pk</b>, fetch its descriptor. If
+ * <b>hsdirs</b> is specified, use the directory servers specified in the list.
+ * Else, use a random server. */
+void
+hs_client_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  if (hsdirs != NULL) {
+    SMARTLIST_FOREACH_BEGIN(hsdirs, const routerstatus_t *, hsdir) {
+      directory_launch_v3_desc_fetch(onion_identity_pk, hsdir);
+    } SMARTLIST_FOREACH_END(hsdir);
+  } else {
+    fetch_v3_desc(onion_identity_pk);
+  }
+}
+
+/** Make sure that the given v3 origin circuit circ is a valid correct
  * introduction circuit. This will BUG() on any problems and hard assert if
  * the anonymity of the circuit is not ok. Return 0 on success else -1 where
  * the circuit should be mark for closed immediately. */
@@ -491,7 +526,7 @@ intro_circ_is_ok(const origin_circuit_t *circ)
   return ret;
 }
 
-/* Find a descriptor intro point object that matches the given ident in the
+/** Find a descriptor intro point object that matches the given ident in the
  * given descriptor desc. Return NULL if not found. */
 static const hs_desc_intro_point_t *
 find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
@@ -514,7 +549,7 @@ find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
   return intro_point;
 }
 
-/* Find a descriptor intro point object from the descriptor object desc that
+/** Find a descriptor intro point object from the descriptor object desc that
  * matches the given legacy identity digest in legacy_id. Return NULL if not
  * found. */
 static hs_desc_intro_point_t *
@@ -531,13 +566,15 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id,
   SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                           hs_desc_intro_point_t *, ip) {
     SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                            const hs_desc_link_specifier_t *, lspec) {
+                            const link_specifier_t *, lspec) {
       /* Not all tor node have an ed25519 identity key so we still rely on the
        * legacy identity digest. */
-      if (lspec->type != LS_LEGACY_ID) {
+      if (link_specifier_get_ls_type(lspec) != LS_LEGACY_ID) {
         continue;
       }
-      if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) {
+      if (fast_memneq(legacy_id,
+                      link_specifier_getconstarray_un_legacy_id(lspec),
+                      DIGEST_LEN)) {
         break;
       }
       /* Found it. */
@@ -550,7 +587,7 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id,
   return ret_ip;
 }
 
-/* Send an INTRODUCE1 cell along the intro circuit and populate the rend
+/** Send an INTRODUCE1 cell along the intro circuit and populate the rend
  * circuit identifier with the needed key material for the e2e encryption.
  * Return 0 on success, -1 if there is a transient error such that an action
  * has been taken to recover and -2 if there is a permanent error indicating
@@ -610,7 +647,7 @@ send_introduce1(origin_circuit_t *intro_circ,
 
   /* Send the INTRODUCE1 cell. */
   if (hs_circ_send_introduce1(intro_circ, rend_circ, ip,
-                              desc->subcredential) < 0) {
+                              &desc->subcredential) < 0) {
     if (TO_CIRCUIT(intro_circ)->marked_for_close) {
       /* If the introduction circuit was closed, we were unable to send the
        * cell for some reasons. In any case, the intro circuit has to be
@@ -667,9 +704,12 @@ send_introduce1(origin_circuit_t *intro_circ,
   return status;
 }
 
-/* Using the introduction circuit circ, setup the authentication key of the
- * intro point this circuit has extended to. */
-static void
+/** Using the introduction circuit circ, setup the authentication key of the
+ * intro point this circuit has extended to.
+ *
+ * Return 0 if everything went well, otherwise return -1 in the case of errors.
+ */
+static int
 setup_intro_circ_auth_key(origin_circuit_t *circ)
 {
   const hs_descriptor_t *desc;
@@ -683,30 +723,31 @@ setup_intro_circ_auth_key(origin_circuit_t *circ)
      * and the client descriptor cache that gets purged (NEWNYM) or the
      * cleaned up because it expired. Mark the circuit for close so a new
      * descriptor fetch can occur. */
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
-    goto end;
+    goto err;
   }
 
   /* We will go over every intro point and try to find which one is linked to
    * that circuit. Those lists are small so it's not that expensive. */
   ip = find_desc_intro_point_by_legacy_id(
                        circ->build_state->chosen_exit->identity_digest, desc);
-  if (ip) {
-    /* We got it, copy its authentication key to the identifier. */
-    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
-                        &ip->auth_key_cert->signed_key);
-    goto end;
+  if (!ip) {
+    /* Reaching this point means we didn't find any intro point for this
+     * circuit which is not supposed to happen. */
+    log_info(LD_REND,"Could not match opened intro circuit with intro point.");
+    goto err;
   }
 
-  /* Reaching this point means we didn't find any intro point for this circuit
-   * which is not suppose to happen. */
-  tor_assert_nonfatal_unreached();
+  /* We got it, copy its authentication key to the identifier. */
+  ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                      &ip->auth_key_cert->signed_key);
+  return 0;
 
- end:
-  return;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+  return -1;
 }
 
-/* Called when an introduction circuit has opened. */
+/** Called when an introduction circuit has opened. */
 static void
 client_intro_circ_has_opened(origin_circuit_t *circ)
 {
@@ -718,12 +759,14 @@ client_intro_circ_has_opened(origin_circuit_t *circ)
   /* This is an introduction circuit so we'll attach the correct
    * authentication key to the circuit identifier so it can be identified
    * properly later on. */
-  setup_intro_circ_auth_key(circ);
+  if (setup_intro_circ_auth_key(circ) < 0) {
+    return;
+  }
 
   connection_ap_attach_pending(1);
 }
 
-/* Called when a rendezvous circuit has opened. */
+/** Called when a rendezvous circuit has opened. */
 static void
 client_rendezvous_circ_has_opened(origin_circuit_t *circ)
 {
@@ -736,10 +779,16 @@ client_rendezvous_circ_has_opened(origin_circuit_t *circ)
    * the v3 rendezvous protocol */
   if (rp_ei) {
     const node_t *rp_node = node_get_by_id(rp_ei->identity_digest);
-    if (rp_node) {
-      if (BUG(!node_supports_v3_rendezvous_point(rp_node))) {
-        return;
-      }
+    if (rp_node && !node_supports_v3_rendezvous_point(rp_node)) {
+      /* Even tho we checked that this node supported v3 when we created the
+         rendezvous circuit, there is a chance that we might think it does
+         not support v3 anymore. This might happen if we got a new consensus
+         in the meanwhile, where the relay is still listed but its listed
+         descriptor digest has changed and hence we can't access its 'ri' or
+         'md'. */
+      log_info(LD_REND, "Rendezvous node %s did not support v3 after circuit "
+               "has opened.", safe_str_client(extend_info_describe(rp_ei)));
+      return;
     }
   }
 
@@ -757,7 +806,7 @@ client_rendezvous_circ_has_opened(origin_circuit_t *circ)
   }
 }
 
-/* This is an helper function that convert a descriptor intro point object ip
+/** This is an helper function that convert a descriptor intro point object ip
  * to a newly allocated extend_info_t object fully initialized. Return NULL if
  * we can't convert it for which chances are that we are missing or malformed
  * link specifiers. */
@@ -765,28 +814,17 @@ STATIC extend_info_t *
 desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip)
 {
   extend_info_t *ei;
-  smartlist_t *lspecs = smartlist_new();
 
   tor_assert(ip);
 
-  /* We first encode the descriptor link specifiers into the binary
-   * representation which is a trunnel object. */
-  SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                          const hs_desc_link_specifier_t *, desc_lspec) {
-    link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec);
-    smartlist_add(lspecs, lspec);
-  } SMARTLIST_FOREACH_END(desc_lspec);
-
   /* Explicitly put the direct connection option to 0 because this is client
    * side and there is no such thing as a non anonymous client. */
-  ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0);
+  ei = hs_get_extend_info_from_lspecs(ip->link_specifiers, &ip->onion_key, 0);
 
-  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls));
-  smartlist_free(lspecs);
   return ei;
 }
 
-/* Return true iff the intro point ip for the service service_pk is usable.
+/** Return true iff the intro point ip for the service service_pk is usable.
  * This function checks if the intro point is in the client intro state cache
  * and checks at the failures. It is considered usable if:
  *   - No error happened (INTRO_POINT_FAILURE_GENERIC)
@@ -831,7 +869,7 @@ intro_point_is_usable(const ed25519_public_key_t *service_pk,
   return 0;
 }
 
-/* Using a descriptor desc, return a newly allocated extend_info_t object of a
+/** Using a descriptor desc, return a newly allocated extend_info_t object of a
  * randomly picked introduction point from its list. Return NULL if none are
  * usable. */
 STATIC extend_info_t *
@@ -936,7 +974,88 @@ client_get_random_intro(const ed25519_public_key_t *service_pk)
   return ei;
 }
 
-/* For this introduction circuit, we'll look at if we have any usable
+/** Return true iff all intro points for the given service have timed out. */
+static bool
+intro_points_all_timed_out(const ed25519_public_key_t *service_pk)
+{
+  bool ret = false;
+
+  tor_assert(service_pk);
+
+  const hs_descriptor_t *desc = hs_cache_lookup_as_client(service_pk);
+  if (BUG(!desc)) {
+    /* We can't introduce without a descriptor so ending up here means somehow
+     * between the introduction failure and this, the cache entry was removed
+     * which shouldn't be possible in theory. */
+    goto end;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    const hs_cache_intro_state_t *state =
+      hs_cache_client_intro_state_find(service_pk,
+                                       &ip->auth_key_cert->signed_key);
+    if (!state || !state->timed_out) {
+      /* No state or if this intro point has not timed out, we are done since
+       * clearly not all of them have timed out. */
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  /* Exiting the loop here means that all intro points we've looked at have
+   * timed out. Note that we can _not_ have a descriptor without intro points
+   * in the client cache. */
+  ret = true;
+
+ end:
+  return ret;
+}
+
+/** Called when a rendezvous circuit has timed out. Every stream attached to
+ * the circuit will get set with the SOCKS5_HS_REND_FAILED (0xF3) extended
+ * error code so if the connection to the rendezvous point ends up not
+ * working, this code could be sent back as a reason. */
+static void
+socks_mark_rend_circuit_timed_out(const origin_circuit_t *rend_circ)
+{
+  tor_assert(rend_circ);
+
+  /* For each entry connection attached to this rendezvous circuit, report
+   * the error. */
+  for (edge_connection_t *edge = rend_circ->p_streams; edge;
+       edge = edge->next_stream) {
+     entry_connection_t *entry = EDGE_TO_ENTRY_CONN(edge);
+     if (entry->socks_request) {
+       entry->socks_request->socks_extended_error_code =
+         SOCKS5_HS_REND_FAILED;
+     }
+  }
+}
+
+/** Called when introduction has failed meaning there is no more usable
+ * introduction points to be used (either NACKed or failed) for the given
+ * entry connection.
+ *
+ * This function only reports back the SOCKS5_HS_INTRO_FAILED (0xF2) code or
+ * SOCKS5_HS_INTRO_TIMEDOUT (0xF7) if all intros have timed out. The caller
+ * has to make sure to close the entry connections. */
+static void
+socks_mark_introduction_failed(entry_connection_t *conn,
+                                 const ed25519_public_key_t *identity_pk)
+{
+  socks5_reply_status_t code = SOCKS5_HS_INTRO_FAILED;
+
+  tor_assert(conn);
+  tor_assert(conn->socks_request);
+  tor_assert(identity_pk);
+
+  if (intro_points_all_timed_out(identity_pk)) {
+    code = SOCKS5_HS_INTRO_TIMEDOUT;
+  }
+  conn->socks_request->socks_extended_error_code = code;
+}
+
+/** For this introduction circuit, we'll look at if we have any usable
  * introduction point left for this service. If so, we'll use the circuit to
  * re-extend to a new intro point. Else, we'll close the circuit and its
  * corresponding rendezvous circuit. Return 0 if we are re-extending else -1
@@ -953,8 +1072,10 @@ close_or_reextend_intro_circ(origin_circuit_t *intro_circ)
   tor_assert(intro_circ);
 
   desc = hs_cache_lookup_as_client(&intro_circ->hs_ident->identity_pk);
-  if (BUG(desc == NULL)) {
-    /* We can't continue without a descriptor. */
+  if (desc == NULL) {
+    /* We can't continue without a descriptor. This is possible if the cache
+     * was cleaned up between the intro point established and the reception of
+     * the introduce ack. */
     goto close;
   }
   /* We still have the descriptor, great! Let's try to see if we can
@@ -993,7 +1114,7 @@ close_or_reextend_intro_circ(origin_circuit_t *intro_circ)
   return ret;
 }
 
-/* Called when we get an INTRODUCE_ACK success status code. Do the appropriate
+/** Called when we get an INTRODUCE_ACK success status code. Do the appropriate
  * actions for the rendezvous point and finally close intro_circ. */
 static void
 handle_introduce_ack_success(origin_circuit_t *intro_circ)
@@ -1039,7 +1160,7 @@ handle_introduce_ack_success(origin_circuit_t *intro_circ)
   return;
 }
 
-/* Called when we get an INTRODUCE_ACK failure status code. Depending on our
+/** Called when we get an INTRODUCE_ACK failure status code. Depending on our
  * failure cache status, either close the circuit or re-extend to a new
  * introduction point. */
 static void
@@ -1061,7 +1182,7 @@ handle_introduce_ack_bad(origin_circuit_t *circ, int status)
                                    INTRO_POINT_FAILURE_GENERIC);
 }
 
-/* Called when we get an INTRODUCE_ACK on the intro circuit circ. The encoded
+/** Called when we get an INTRODUCE_ACK on the intro circuit circ. The encoded
  * cell is in payload of length payload_len. Return 0 on success else a
  * negative value. The circuit is either close or reuse to re-extend to a new
  * introduction point. */
@@ -1100,7 +1221,7 @@ handle_introduce_ack(origin_circuit_t *circ, const uint8_t *payload,
   return ret;
 }
 
-/* Called when we get a RENDEZVOUS2 cell on the rendezvous circuit circ. The
+/** Called when we get a RENDEZVOUS2 cell on the rendezvous circuit circ. The
  * encoded cell is in payload of length payload_len. Return 0 on success or a
  * negative value on error. On error, the circuit is marked for close. */
 STATIC int
@@ -1162,7 +1283,7 @@ handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
   return ret;
 }
 
-/* Return true iff the client can fetch a descriptor for this service public
+/** Return true iff the client can fetch a descriptor for this service public
  * identity key and status_out if not NULL is untouched. If the client can
  * _not_ fetch the descriptor and if status_out is not NULL, it is set with
  * the fetch status code. */
@@ -1230,7 +1351,27 @@ can_client_refetch_desc(const ed25519_public_key_t *identity_pk,
   return 0;
 }
 
-/* Return the client auth in the map using the service identity public key.
+/** Purge the client authorization cache of all ephemeral entries that is the
+ * entries that are not flagged with CLIENT_AUTH_FLAG_IS_PERMANENT.
+ *
+ * This is called from the hs_client_purge_state() used by a SIGNEWNYM. */
+STATIC void
+purge_ephemeral_client_auth(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(client_auths, key,
+                              hs_client_service_authorization_t *, auth) {
+    /* Cleanup every entry that are _NOT_ permanent that is ephemeral. */
+    if (!(auth->flags & CLIENT_AUTH_FLAG_IS_PERMANENT)) {
+      MAP_DEL_CURRENT(key);
+      client_service_authorization_free(auth);
+    }
+  } DIGESTMAP_FOREACH_END;
+
+  log_info(LD_REND, "Client onion service ephemeral authorization "
+                    "cache has been purged.");
+}
+
+/** Return the client auth in the map using the service identity public key.
  * Return NULL if it does not exist in the map. */
 static hs_client_service_authorization_t *
 find_client_auth(const ed25519_public_key_t *service_identity_pk)
@@ -1243,10 +1384,565 @@ find_client_auth(const ed25519_public_key_t *service_identity_pk)
   return digest256map_get(client_auths, service_identity_pk->pubkey);
 }
 
+/** This is called when a descriptor has arrived following a fetch request and
+ * has been stored in the client cache. The given entry connections, matching
+ * the service identity key, will get attached to the service circuit. */
+static void
+client_desc_has_arrived(const smartlist_t *entry_conns)
+{
+  time_t now = time(NULL);
+
+  tor_assert(entry_conns);
+
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
+    const hs_descriptor_t *desc;
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+    const ed25519_public_key_t *identity_pk =
+      &edge_conn->hs_ident->identity_pk;
+
+    /* We were just called because we stored the descriptor for this service
+     * so not finding a descriptor means we have a bigger problem. */
+    desc = hs_cache_lookup_as_client(identity_pk);
+    if (BUG(desc == NULL)) {
+      goto end;
+    }
+
+    if (!hs_client_any_intro_points_usable(identity_pk, desc)) {
+      log_info(LD_REND, "Hidden service descriptor is unusable. "
+                        "Closing streams.");
+      /* Report the extended socks error code that we were unable to introduce
+       * to the service. */
+      socks_mark_introduction_failed(entry_conn, identity_pk);
+
+      connection_mark_unattached_ap(entry_conn,
+                                    END_STREAM_REASON_RESOLVEFAILED);
+      /* We are unable to use the descriptor so remove the directory request
+       * from the cache so the next connection can try again. */
+      note_connection_attempt_succeeded(edge_conn->hs_ident);
+      continue;
+    }
+
+    log_info(LD_REND, "Descriptor has arrived. Launching circuits.");
+
+    /* Mark connection as waiting for a circuit since we do have a usable
+     * descriptor now. */
+    mark_conn_as_waiting_for_circuit(&edge_conn->base_, now);
+  } SMARTLIST_FOREACH_END(entry_conn);
+
+ end:
+  return;
+}
+
+/** This is called when a descriptor fetch was successful but the descriptor
+ * couldn't be decrypted due to missing or bad client authorization. */
+static void
+client_desc_missing_bad_client_auth(const smartlist_t *entry_conns,
+                                    hs_desc_decode_status_t status)
+{
+  tor_assert(entry_conns);
+
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
+    socks5_reply_status_t code;
+    if (status == HS_DESC_DECODE_BAD_CLIENT_AUTH) {
+      code = SOCKS5_HS_BAD_CLIENT_AUTH;
+    } else if (status == HS_DESC_DECODE_NEED_CLIENT_AUTH) {
+      code = SOCKS5_HS_MISSING_CLIENT_AUTH;
+    } else {
+      /* We should not be called with another type of status. Recover by
+       * sending a generic error. */
+      tor_assert_nonfatal_unreached();
+      code = SOCKS5_GENERAL_ERROR;
+    }
+    entry_conn->socks_request->socks_extended_error_code = code;
+    connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_MISC);
+  } SMARTLIST_FOREACH_END(entry_conn);
+}
+
+/** Called when we get a 200 directory fetch status code. */
+static void
+client_dir_fetch_200(dir_connection_t *dir_conn,
+                     const smartlist_t *entry_conns, const char *body)
+{
+  hs_desc_decode_status_t decode_status;
+
+  tor_assert(dir_conn);
+  tor_assert(entry_conns);
+  tor_assert(body);
+
+  /* We got something: Try storing it in the cache. */
+  decode_status = hs_cache_store_as_client(body,
+                                           &dir_conn->hs_ident->identity_pk);
+  switch (decode_status) {
+  case HS_DESC_DECODE_OK:
+  case HS_DESC_DECODE_NEED_CLIENT_AUTH:
+  case HS_DESC_DECODE_BAD_CLIENT_AUTH:
+    log_info(LD_REND, "Stored hidden service descriptor successfully.");
+    TO_CONN(dir_conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
+    if (decode_status == HS_DESC_DECODE_OK) {
+      client_desc_has_arrived(entry_conns);
+    } else {
+      /* This handles both client auth decode status. */
+      client_desc_missing_bad_client_auth(entry_conns, decode_status);
+      log_info(LD_REND, "Stored hidden service descriptor requires "
+                         "%s client authorization.",
+               decode_status == HS_DESC_DECODE_NEED_CLIENT_AUTH ? "missing"
+                                                                : "new");
+    }
+    /* Fire control port RECEIVED event. */
+    hs_control_desc_event_received(dir_conn->hs_ident,
+                                   dir_conn->identity_digest);
+    hs_control_desc_event_content(dir_conn->hs_ident,
+                                  dir_conn->identity_digest, body);
+    break;
+  case HS_DESC_DECODE_ENCRYPTED_ERROR:
+  case HS_DESC_DECODE_SUPERENC_ERROR:
+  case HS_DESC_DECODE_PLAINTEXT_ERROR:
+  case HS_DESC_DECODE_GENERIC_ERROR:
+  default:
+    log_info(LD_REND, "Failed to store hidden service descriptor. "
+                      "Descriptor decoding status: %d", decode_status);
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(dir_conn->hs_ident,
+                                 dir_conn->identity_digest, "BAD_DESC");
+    hs_control_desc_event_content(dir_conn->hs_ident,
+                                  dir_conn->identity_digest, NULL);
+    break;
+  }
+}
+
+/** Called when we get a 404 directory fetch status code. */
+static void
+client_dir_fetch_404(dir_connection_t *dir_conn,
+                     const smartlist_t *entry_conns)
+{
+  tor_assert(entry_conns);
+
+  /* Not there. We'll retry when connection_about_to_close_connection() tries
+   * to clean this conn up. */
+  log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
+                    "Retrying at another directory.");
+  /* Fire control port FAILED event. */
+  hs_control_desc_event_failed(dir_conn->hs_ident, dir_conn->identity_digest,
+                               "NOT_FOUND");
+  hs_control_desc_event_content(dir_conn->hs_ident, dir_conn->identity_digest,
+                                NULL);
+
+  /* Flag every entry connections that the descriptor was not found. */
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
+    entry_conn->socks_request->socks_extended_error_code =
+      SOCKS5_HS_NOT_FOUND;
+  } SMARTLIST_FOREACH_END(entry_conn);
+}
+
+/** Called when we get a 400 directory fetch status code. */
+static void
+client_dir_fetch_400(dir_connection_t *dir_conn, const char *reason)
+{
+  tor_assert(dir_conn);
+
+  log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+                    "http status 400 (%s). Dirserver didn't like our "
+                    "query? Retrying at another directory.",
+           escaped(reason));
+
+  /* Fire control port FAILED event. */
+  hs_control_desc_event_failed(dir_conn->hs_ident, dir_conn->identity_digest,
+                               "QUERY_REJECTED");
+  hs_control_desc_event_content(dir_conn->hs_ident, dir_conn->identity_digest,
+                                NULL);
+}
+
+/** Called when we get an unexpected directory fetch status code. */
+static void
+client_dir_fetch_unexpected(dir_connection_t *dir_conn, const char *reason,
+                            const int status_code)
+{
+  tor_assert(dir_conn);
+
+  log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+                    "http status %d (%s) response unexpected from HSDir "
+                    "server '%s:%d'. Retrying at another directory.",
+           status_code, escaped(reason), TO_CONN(dir_conn)->address,
+           TO_CONN(dir_conn)->port);
+  /* Fire control port FAILED event. */
+  hs_control_desc_event_failed(dir_conn->hs_ident, dir_conn->identity_digest,
+                               "UNEXPECTED");
+  hs_control_desc_event_content(dir_conn->hs_ident, dir_conn->identity_digest,
+                                NULL);
+}
+
+/** Get the full filename for storing the client auth credentials for the
+ *  service in <b>onion_address</b>. The base directory is <b>dir</b>.
+ *  This function never returns NULL. */
+static char *
+get_client_auth_creds_filename(const char *onion_address,
+                               const char *dir)
+{
+  char *full_fname = NULL;
+  char *fname;
+
+  tor_asprintf(&fname, "%s.auth_private", onion_address);
+  full_fname = hs_path_from_filename(dir, fname);
+  tor_free(fname);
+
+  return full_fname;
+}
+
+/** Permanently store the credentials in <b>creds</b> to disk.
+ *
+ *  Return -1 if there was an error while storing the credentials, otherwise
+ *  return 0.
+ */
+static int
+store_permanent_client_auth_credentials(
+                              const hs_client_service_authorization_t *creds)
+{
+  const or_options_t *options = get_options();
+  char *full_fname = NULL;
+  char *file_contents = NULL;
+  char priv_key_b32[BASE32_NOPAD_LEN(CURVE25519_PUBKEY_LEN)+1];
+  int retval = -1;
+
+  tor_assert(creds->flags & CLIENT_AUTH_FLAG_IS_PERMANENT);
+
+  /* We need ClientOnionAuthDir to be set, otherwise we can't proceed */
+  if (!options->ClientOnionAuthDir) {
+    log_warn(LD_GENERAL, "Can't register permanent client auth credentials "
+             "for %s without ClientOnionAuthDir option. Discarding.",
+             creds->onion_address);
+    goto err;
+  }
+
+  /* Make sure the directory exists and is private enough. */
+  if (check_private_dir(options->ClientOnionAuthDir, 0, options->User) < 0) {
+    goto err;
+  }
+
+  /* Get filename that we should store the credentials */
+  full_fname = get_client_auth_creds_filename(creds->onion_address,
+                                              options->ClientOnionAuthDir);
+
+  /* Encode client private key */
+  base32_encode(priv_key_b32, sizeof(priv_key_b32),
+                (char*)creds->enc_seckey.secret_key,
+                sizeof(creds->enc_seckey.secret_key));
+
+  /* Get the full file contents and write it to disk! */
+  tor_asprintf(&file_contents, "%s:descriptor:x25519:%s",
+               creds->onion_address, priv_key_b32);
+  if (write_str_to_file(full_fname, file_contents, 0) < 0) {
+    log_warn(LD_GENERAL, "Failed to write client auth creds file for %s!",
+             creds->onion_address);
+    goto err;
+  }
+
+  retval = 0;
+
+ err:
+  tor_free(file_contents);
+  tor_free(full_fname);
+
+  return retval;
+}
+
+/** Register the credential <b>creds</b> as part of the client auth subsystem.
+ *
+ * Takes ownership of <b>creds</b>.
+ **/
+hs_client_register_auth_status_t
+hs_client_register_auth_credentials(hs_client_service_authorization_t *creds)
+{
+  ed25519_public_key_t service_identity_pk;
+  hs_client_service_authorization_t *old_creds = NULL;
+  hs_client_register_auth_status_t retval = REGISTER_SUCCESS;
+
+  tor_assert(creds);
+
+  if (!client_auths) {
+    client_auths = digest256map_new();
+  }
+
+  if (hs_parse_address(creds->onion_address, &service_identity_pk,
+                       NULL, NULL) < 0) {
+    client_service_authorization_free(creds);
+    return REGISTER_FAIL_BAD_ADDRESS;
+  }
+
+  /* If we reach this point, the credentials will be stored one way or another:
+   * Make them permanent if the user asked us to. */
+  if (creds->flags & CLIENT_AUTH_FLAG_IS_PERMANENT) {
+    if (store_permanent_client_auth_credentials(creds) < 0) {
+      client_service_authorization_free(creds);
+      return REGISTER_FAIL_PERMANENT_STORAGE;
+    }
+  }
+
+  old_creds = digest256map_get(client_auths, service_identity_pk.pubkey);
+  if (old_creds) {
+    digest256map_remove(client_auths, service_identity_pk.pubkey);
+    client_service_authorization_free(old_creds);
+    retval = REGISTER_SUCCESS_ALREADY_EXISTS;
+  }
+
+  digest256map_set(client_auths, service_identity_pk.pubkey, creds);
+
+  /** Now that we set the new credentials, also try to decrypt any cached
+   *  descriptors. */
+  if (hs_cache_client_new_auth_parse(&service_identity_pk)) {
+    retval = REGISTER_SUCCESS_AND_DECRYPTED;
+  }
+
+  return retval;
+}
+
+/** Load a client authorization file with <b>filename</b> that is stored under
+ *  the global client auth directory, and return a newly-allocated credentials
+ *  object if it parsed well. Otherwise, return NULL.
+ */
+static hs_client_service_authorization_t *
+get_creds_from_client_auth_filename(const char *filename,
+                                    const or_options_t *options)
+{
+  hs_client_service_authorization_t *auth = NULL;
+  char *client_key_file_path = NULL;
+  char *client_key_str = NULL;
+
+  log_info(LD_REND, "Loading a client authorization key file %s...",
+           filename);
+
+  if (!auth_key_filename_is_valid(filename)) {
+    log_notice(LD_REND, "Client authorization unrecognized filename %s. "
+               "File must end in .auth_private. Ignoring.",
+               filename);
+    goto err;
+  }
+
+  /* Create a full path for a file. */
+  client_key_file_path = hs_path_from_filename(options->ClientOnionAuthDir,
+                                               filename);
+
+  client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
+  if (!client_key_str) {
+    log_warn(LD_REND, "The file %s cannot be read.", filename);
+    goto err;
+  }
+
+  auth = parse_auth_file_content(client_key_str);
+  if (!auth) {
+    goto err;
+  }
+
+ err:
+  tor_free(client_key_str);
+  tor_free(client_key_file_path);
+
+  return auth;
+}
+
+/*
+ * Remove the file in <b>filename</b> under the global client auth credential
+ * storage.
+ */
+static void
+remove_client_auth_creds_file(const char *filename)
+{
+  char *creds_file_path = NULL;
+  const or_options_t *options = get_options();
+
+  creds_file_path = hs_path_from_filename(options->ClientOnionAuthDir,
+                                          filename);
+  if (tor_unlink(creds_file_path) != 0) {
+    log_warn(LD_REND, "Failed to remove client auth file (%s).",
+             creds_file_path);
+    goto end;
+  }
+
+  log_warn(LD_REND, "Successfuly removed client auth file (%s).",
+           creds_file_path);
+
+ end:
+  tor_free(creds_file_path);
+}
+
+/**
+ * Find the filesystem file corresponding to the permanent client auth
+ * credentials in <b>cred</b> and remove it.
+ */
+static void
+find_and_remove_client_auth_creds_file(
+                                 const hs_client_service_authorization_t *cred)
+{
+  smartlist_t *file_list = NULL;
+  const or_options_t *options = get_options();
+
+  tor_assert(cred->flags & CLIENT_AUTH_FLAG_IS_PERMANENT);
+
+  if (!options->ClientOnionAuthDir) {
+    log_warn(LD_REND, "Found permanent credential but no ClientOnionAuthDir "
+             "configured. There is no file to be removed.");
+    goto end;
+  }
+
+  file_list = tor_listdir(options->ClientOnionAuthDir);
+  if (file_list == NULL) {
+    log_warn(LD_REND, "Client authorization key directory %s can't be listed.",
+             options->ClientOnionAuthDir);
+    goto end;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(file_list, const char *, filename) {
+    hs_client_service_authorization_t *tmp_cred = NULL;
+
+    tmp_cred = get_creds_from_client_auth_filename(filename, options);
+    if (!tmp_cred) {
+      continue;
+    }
+
+    /* Find the right file for this credential */
+    if (!strcmp(tmp_cred->onion_address, cred->onion_address)) {
+      /* Found it! Remove the file! */
+      remove_client_auth_creds_file(filename);
+      /* cleanup and get out of here */
+      client_service_authorization_free(tmp_cred);
+      break;
+    }
+
+    client_service_authorization_free(tmp_cred);
+  } SMARTLIST_FOREACH_END(filename);
+
+ end:
+  if (file_list) {
+    SMARTLIST_FOREACH(file_list, char *, s, tor_free(s));
+    smartlist_free(file_list);
+  }
+}
+
+/** Remove client auth credentials for the service <b>hs_address</b>. */
+hs_client_removal_auth_status_t
+hs_client_remove_auth_credentials(const char *hsaddress)
+{
+  ed25519_public_key_t service_identity_pk;
+
+  if (!client_auths) {
+    return REMOVAL_SUCCESS_NOT_FOUND;
+  }
+
+  if (hs_parse_address(hsaddress, &service_identity_pk, NULL, NULL) < 0) {
+    return REMOVAL_BAD_ADDRESS;
+  }
+
+  hs_client_service_authorization_t *cred = NULL;
+  cred = digest256map_remove(client_auths, service_identity_pk.pubkey);
+
+  /* digestmap_remove() returns the previously stored data if there were any */
+  if (cred) {
+    if (cred->flags & CLIENT_AUTH_FLAG_IS_PERMANENT) {
+      /* These creds are stored on disk: remove the corresponding file. */
+      find_and_remove_client_auth_creds_file(cred);
+    }
+
+    /* Remove associated descriptor if any. */
+    hs_cache_remove_as_client(&service_identity_pk);
+
+    client_service_authorization_free(cred);
+    return REMOVAL_SUCCESS;
+  }
+
+  return REMOVAL_SUCCESS_NOT_FOUND;
+}
+
+/** Get the HS client auth map. */
+digest256map_t *
+get_hs_client_auths_map(void)
+{
+  return client_auths;
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
+/** Called when a circuit was just cleaned up. This is done right before the
+ * circuit is marked for close. */
+void
+hs_client_circuit_cleanup_on_close(const circuit_t *circ)
+{
+  bool has_timed_out;
+
+  tor_assert(circ);
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+
+  has_timed_out =
+    (circ->marked_for_close_orig_reason == END_CIRC_REASON_TIMEOUT);
+
+  switch (circ->purpose) {
+  case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
+  case CIRCUIT_PURPOSE_C_REND_READY:
+  case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED:
+  case CIRCUIT_PURPOSE_C_REND_JOINED:
+    /* Report extended SOCKS error code when a rendezvous circuit times out.
+     * This MUST be done on_close() because it is possible the entry
+     * connection would get closed before the circuit is freed and thus
+     * would fail to report the error code. */
+    if (has_timed_out) {
+      socks_mark_rend_circuit_timed_out(CONST_TO_ORIGIN_CIRCUIT(circ));
+    }
+    break;
+  default:
+    break;
+  }
+}
+
+/** Called when a circuit was just cleaned up. This is done right before the
+ * circuit is freed. */
+void
+hs_client_circuit_cleanup_on_free(const circuit_t *circ)
+{
+  bool has_timed_out;
+  rend_intro_point_failure_t failure = INTRO_POINT_FAILURE_GENERIC;
+  const origin_circuit_t *orig_circ = NULL;
+
+  tor_assert(circ);
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+
+  orig_circ = CONST_TO_ORIGIN_CIRCUIT(circ);
+  tor_assert(orig_circ->hs_ident);
+
+  has_timed_out =
+    (circ->marked_for_close_orig_reason == END_CIRC_REASON_TIMEOUT);
+  if (has_timed_out) {
+    failure = INTRO_POINT_FAILURE_TIMEOUT;
+  }
+
+  switch (circ->purpose) {
+  case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
+    log_info(LD_REND, "Failed v3 intro circ for service %s to intro point %s "
+                      "(awaiting ACK). Failure code: %d",
+        safe_str_client(ed25519_fmt(&orig_circ->hs_ident->identity_pk)),
+        safe_str_client(build_state_get_exit_nickname(orig_circ->build_state)),
+        failure);
+    hs_cache_client_intro_state_note(&orig_circ->hs_ident->identity_pk,
+                                     &orig_circ->hs_ident->intro_auth_pk,
+                                     failure);
+    break;
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    if (has_timed_out || !orig_circ->build_state) {
+      break;
+    }
+    failure = INTRO_POINT_FAILURE_UNREACHABLE;
+    log_info(LD_REND, "Failed v3 intro circ for service %s to intro point %s "
+                      "(while building circuit). Marking as unreachable.",
+       safe_str_client(ed25519_fmt(&orig_circ->hs_ident->identity_pk)),
+       safe_str_client(build_state_get_exit_nickname(orig_circ->build_state)));
+    hs_cache_client_intro_state_note(&orig_circ->hs_ident->identity_pk,
+                                     &orig_circ->hs_ident->intro_auth_pk,
+                                     failure);
+    break;
+  default:
+    break;
+  }
+}
+
 /** A circuit just finished connecting to a hidden service that the stream
  *  <b>conn</b> has been waiting for. Let the HS subsystem know about this. */
 void
@@ -1268,18 +1964,20 @@ hs_client_note_connection_attempt_succeeded(const edge_connection_t *conn)
   }
 }
 
-/* With the given encoded descriptor in desc_str and the service key in
+/** With the given encoded descriptor in desc_str and the service key in
  * service_identity_pk, decode the descriptor and set the desc pointer with a
  * newly allocated descriptor object.
  *
- * Return 0 on success else a negative value and desc is set to NULL. */
-int
+ * On success, HS_DESC_DECODE_OK is returned and desc is set to the decoded
+ * descriptor. On error, desc is set to NULL and a decoding error status is
+ * returned depending on what was the issue. */
+hs_desc_decode_status_t
 hs_client_decode_descriptor(const char *desc_str,
                             const ed25519_public_key_t *service_identity_pk,
                             hs_descriptor_t **desc)
 {
-  int ret;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_desc_decode_status_t ret;
+  hs_subcredential_t subcredential;
   ed25519_public_key_t blinded_pubkey;
   hs_client_service_authorization_t *client_auth = NULL;
   curve25519_secret_key_t *client_auth_sk = NULL;
@@ -1299,14 +1997,14 @@ hs_client_decode_descriptor(const char *desc_str,
     uint64_t current_time_period = hs_get_time_period_num(0);
     hs_build_blinded_pubkey(service_identity_pk, NULL, 0, current_time_period,
                             &blinded_pubkey);
-    hs_get_subcredential(service_identity_pk, &blinded_pubkey, subcredential);
+    hs_get_subcredential(service_identity_pk, &blinded_pubkey, &subcredential);
   }
 
   /* Parse descriptor */
-  ret = hs_desc_decode_descriptor(desc_str, subcredential,
+  ret = hs_desc_decode_descriptor(desc_str, &subcredential,
                                   client_auth_sk, desc);
-  memwipe(subcredential, 0, sizeof(subcredential));
-  if (ret < 0) {
+  memwipe(&subcredential, 0, sizeof(subcredential));
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
@@ -1319,15 +2017,16 @@ hs_client_decode_descriptor(const char *desc_str,
     log_warn(LD_GENERAL, "Descriptor signing key certificate signature "
              "doesn't validate with computed blinded key: %s",
              tor_cert_describe_signature_status(cert));
+    ret = HS_DESC_DECODE_GENERIC_ERROR;
     goto err;
   }
 
-  return 0;
+  return HS_DESC_DECODE_OK;
  err:
-  return -1;
+  return ret;
 }
 
-/* Return true iff there are at least one usable intro point in the service
+/** Return true iff there are at least one usable intro point in the service
  * descriptor desc. */
 int
 hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
@@ -1376,7 +2075,7 @@ hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk)
   return status;
 }
 
-/* This is called when we are trying to attach an AP connection to these
+/** This is called when we are trying to attach an AP connection to these
  * hidden service circuits from connection_ap_handshake_attach_circuit().
  * Return 0 on success, -1 for a transient error that is actions were
  * triggered to recover or -2 for a permenent error where both circuits will
@@ -1392,7 +2091,7 @@ hs_client_send_introduce1(origin_circuit_t *intro_circ,
                                                                 rend_circ);
 }
 
-/* Called when the client circuit circ has been established. It can be either
+/** Called when the client circuit circ has been established. It can be either
  * an introduction or rendezvous circuit. This function handles all hidden
  * service versions. */
 void
@@ -1422,7 +2121,7 @@ hs_client_circuit_has_opened(origin_circuit_t *circ)
   }
 }
 
-/* Called when we receive a RENDEZVOUS_ESTABLISHED cell. Change the state of
+/** Called when we receive a RENDEZVOUS_ESTABLISHED cell. Change the state of
  * the circuit to CIRCUIT_PURPOSE_C_REND_READY. Return 0 on success else a
  * negative value and the circuit marked for close. */
 int
@@ -1464,16 +2163,16 @@ hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
   return -1;
 }
 
-#define client_service_authorization_free(auth)                      \
-  FREE_AND_NULL(hs_client_service_authorization_t,                   \
-                client_service_authorization_free_, (auth))
-
-static void
+void
 client_service_authorization_free_(hs_client_service_authorization_t *auth)
 {
-  if (auth) {
-    memwipe(auth, 0, sizeof(*auth));
+  if (!auth) {
+    return;
   }
+
+  tor_free(auth->client_name);
+
+  memwipe(auth, 0, sizeof(*auth));
   tor_free(auth);
 }
 
@@ -1493,7 +2192,7 @@ client_service_authorization_free_all(void)
   digest256map_free(client_auths, client_service_authorization_free_void);
 }
 
-/* Check if the auth key file name is valid or not. Return 1 if valid,
+/** Check if the auth key file name is valid or not. Return 1 if valid,
  * otherwise return 0. */
 STATIC int
 auth_key_filename_is_valid(const char *filename)
@@ -1515,6 +2214,13 @@ auth_key_filename_is_valid(const char *filename)
   return ret;
 }
 
+/** Parse the client auth credentials off a string in <b>client_key_str</b>
+ *  based on the file format documented in the "Client side configuration"
+ *  section of rend-spec-v3.txt.
+ *
+ *  Return NULL if there was an error, otherwise return a newly allocated
+ *  hs_client_service_authorization_t structure.
+ */
 STATIC hs_client_service_authorization_t *
 parse_auth_file_content(const char *client_key_str)
 {
@@ -1545,7 +2251,7 @@ parse_auth_file_content(const char *client_key_str)
     goto err;
   }
 
-  if (strlen(seckey_b32) != BASE32_NOPAD_LEN(CURVE25519_PUBKEY_LEN)) {
+  if (strlen(seckey_b32) != BASE32_NOPAD_LEN(CURVE25519_SECKEY_LEN)) {
     log_warn(LD_REND, "Client authorization encoded base32 private key "
                       "length is invalid: %s", seckey_b32);
     goto err;
@@ -1554,11 +2260,24 @@ parse_auth_file_content(const char *client_key_str)
   auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
   if (base32_decode((char *) auth->enc_seckey.secret_key,
                     sizeof(auth->enc_seckey.secret_key),
-                    seckey_b32, strlen(seckey_b32)) < 0) {
+                    seckey_b32, strlen(seckey_b32)) !=
+      sizeof(auth->enc_seckey.secret_key)) {
+    log_warn(LD_REND, "Client authorization encoded base32 private key "
+                      "can't be decoded: %s", seckey_b32);
     goto err;
   }
+
+  if (fast_mem_is_zero((const char*)auth->enc_seckey.secret_key,
+                       sizeof(auth->enc_seckey.secret_key))) {
+    log_warn(LD_REND, "Client authorization private key can't be all-zeroes");
+    goto err;
+  }
+
   strncpy(auth->onion_address, onion_address, HS_SERVICE_ADDR_LEN_BASE32);
 
+  /* We are reading this from the disk, so set the permanent flag anyway. */
+  auth->flags |= CLIENT_AUTH_FLAG_IS_PERMANENT;
+
   /* Success. */
   goto done;
 
@@ -1575,7 +2294,7 @@ parse_auth_file_content(const char *client_key_str)
   return auth;
 }
 
-/* From a set of <b>options</b>, setup every client authorization detail
+/** From a set of <b>options</b>, setup every client authorization detail
  * found. Return 0 on success or -1 on failure. If <b>validate_only</b>
  * is set, parse, warn and return as normal, but don't actually change
  * the configuration. */
@@ -1585,10 +2304,7 @@ hs_config_client_authorization(const or_options_t *options,
 {
   int ret = -1;
   digest256map_t *auths = digest256map_new();
-  char *key_dir = NULL;
   smartlist_t *file_list = NULL;
-  char *client_key_str = NULL;
-  char *client_key_file_path = NULL;
 
   tor_assert(options);
 
@@ -1599,82 +2315,54 @@ hs_config_client_authorization(const or_options_t *options,
     goto end;
   }
 
-  key_dir = tor_strdup(options->ClientOnionAuthDir);
-
   /* Make sure the directory exists and is private enough. */
-  if (check_private_dir(key_dir, 0, options->User) < 0) {
+  if (check_private_dir(options->ClientOnionAuthDir, 0, options->User) < 0) {
     goto end;
   }
 
-  file_list = tor_listdir(key_dir);
+  file_list = tor_listdir(options->ClientOnionAuthDir);
   if (file_list == NULL) {
     log_warn(LD_REND, "Client authorization key directory %s can't be listed.",
-             key_dir);
+             options->ClientOnionAuthDir);
     goto end;
   }
 
-  SMARTLIST_FOREACH_BEGIN(file_list, char *, filename) {
-
+  SMARTLIST_FOREACH_BEGIN(file_list, const char *, filename) {
     hs_client_service_authorization_t *auth = NULL;
     ed25519_public_key_t identity_pk;
-    log_info(LD_REND, "Loading a client authorization key file %s...",
-             filename);
 
-    if (!auth_key_filename_is_valid(filename)) {
-      log_notice(LD_REND, "Client authorization unrecognized filename %s. "
-                          "File must end in .auth_private. Ignoring.",
-                 filename);
+    auth = get_creds_from_client_auth_filename(filename, options);
+    if (!auth) {
       continue;
     }
 
-    /* Create a full path for a file. */
-    client_key_file_path = hs_path_from_filename(key_dir, filename);
-    client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
-    /* Free the file path immediately after using it. */
-    tor_free(client_key_file_path);
-
-    /* If we cannot read the file, continue with the next file. */
-    if (!client_key_str) {
-      log_warn(LD_REND, "The file %s cannot be read.", filename);
+    /* Parse the onion address to get an identity public key and use it
+     * as a key of global map in the future. */
+    if (hs_parse_address(auth->onion_address, &identity_pk,
+                         NULL, NULL) < 0) {
+      log_warn(LD_REND, "The onion address \"%s\" is invalid in "
+               "file %s", filename, auth->onion_address);
+      client_service_authorization_free(auth);
       continue;
     }
 
-    auth = parse_auth_file_content(client_key_str);
-    /* Free immediately after using it. */
-    tor_free(client_key_str);
-
-    if (auth) {
-      /* Parse the onion address to get an identity public key and use it
-       * as a key of global map in the future. */
-      if (hs_parse_address(auth->onion_address, &identity_pk,
-                           NULL, NULL) < 0) {
-        log_warn(LD_REND, "The onion address \"%s\" is invalid in "
-                          "file %s", filename, auth->onion_address);
-        client_service_authorization_free(auth);
-        continue;
-      }
-
-      if (digest256map_get(auths, identity_pk.pubkey)) {
+    if (digest256map_get(auths, identity_pk.pubkey)) {
         log_warn(LD_REND, "Duplicate authorization for the same hidden "
-                          "service address %s.",
+                 "service address %s.",
                  safe_str_client_opts(options, auth->onion_address));
         client_service_authorization_free(auth);
         goto end;
-      }
-
-      digest256map_set(auths, identity_pk.pubkey, auth);
-      log_info(LD_REND, "Loaded a client authorization key file %s.",
-               filename);
     }
+
+    digest256map_set(auths, identity_pk.pubkey, auth);
+    log_info(LD_REND, "Loaded a client authorization key file %s.",
+             filename);
   } SMARTLIST_FOREACH_END(filename);
 
   /* Success. */
   ret = 0;
 
  end:
-  tor_free(key_dir);
-  tor_free(client_key_str);
-  tor_free(client_key_file_path);
   if (file_list) {
     SMARTLIST_FOREACH(file_list, char *, s, tor_free(s));
     smartlist_free(file_list);
@@ -1690,65 +2378,48 @@ hs_config_client_authorization(const or_options_t *options,
   return ret;
 }
 
-/* This is called when a descriptor has arrived following a fetch request and
- * has been stored in the client cache. Every entry connection that matches
- * the service identity key in the ident will get attached to the hidden
- * service circuit. */
+/** Called when a descriptor directory fetch is done.
+ *
+ * Act accordingly on all entry connections depending on the HTTP status code
+ * we got. In case of an error, the SOCKS error is set (if ExtendedErrors is
+ * set).
+ *
+ * The reason is a human readable string returned by the directory server
+ * which can describe the status of the request. The body is the response
+ * content, on 200 code it is the descriptor itself. Finally, the status_code
+ * is the HTTP code returned by the directory server. */
 void
-hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident)
+hs_client_dir_fetch_done(dir_connection_t *dir_conn, const char *reason,
+                         const char *body, const int status_code)
 {
-  time_t now = time(NULL);
-  smartlist_t *conns = NULL;
+  smartlist_t *entry_conns;
 
-  tor_assert(ident);
+  tor_assert(dir_conn);
+  tor_assert(body);
 
-  conns = connection_list_by_type_state(CONN_TYPE_AP,
-                                        AP_CONN_STATE_RENDDESC_WAIT);
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-    const hs_descriptor_t *desc;
-    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
-    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
-
-    /* Only consider the entry connections that matches the service for which
-     * we just fetched its descriptor. */
-    if (!edge_conn->hs_ident ||
-        !ed25519_pubkey_eq(&ident->identity_pk,
-                           &edge_conn->hs_ident->identity_pk)) {
-      continue;
-    }
-    assert_connection_ok(base_conn, now);
-
-    /* We were just called because we stored the descriptor for this service
-     * so not finding a descriptor means we have a bigger problem. */
-    desc = hs_cache_lookup_as_client(&ident->identity_pk);
-    if (BUG(desc == NULL)) {
-      goto end;
-    }
+  /* Get all related entry connections. */
+  entry_conns = find_entry_conns(&dir_conn->hs_ident->identity_pk);
 
-    if (!hs_client_any_intro_points_usable(&ident->identity_pk, desc)) {
-      log_info(LD_REND, "Hidden service descriptor is unusable. "
-                        "Closing streams.");
-      connection_mark_unattached_ap(entry_conn,
-                                    END_STREAM_REASON_RESOLVEFAILED);
-      /* We are unable to use the descriptor so remove the directory request
-       * from the cache so the next connection can try again. */
-      note_connection_attempt_succeeded(edge_conn->hs_ident);
-      continue;
-    }
-
-    log_info(LD_REND, "Descriptor has arrived. Launching circuits.");
-
-    /* Mark connection as waiting for a circuit since we do have a usable
-     * descriptor now. */
-    mark_conn_as_waiting_for_circuit(base_conn, now);
-  } SMARTLIST_FOREACH_END(base_conn);
+  switch (status_code) {
+  case 200:
+    client_dir_fetch_200(dir_conn, entry_conns, body);
+    break;
+  case 404:
+    client_dir_fetch_404(dir_conn, entry_conns);
+    break;
+  case 400:
+    client_dir_fetch_400(dir_conn, reason);
+    break;
+  default:
+    client_dir_fetch_unexpected(dir_conn, reason, status_code);
+    break;
+  }
 
- end:
   /* We don't have ownership of the objects in this list. */
-  smartlist_free(conns);
+  smartlist_free(entry_conns);
 }
 
-/* Return a newly allocated extend_info_t for a randomly chosen introduction
+/** Return a newly allocated extend_info_t for a randomly chosen introduction
  * point for the given edge connection identifier ident. Return NULL if we
  * can't pick any usable introduction points. */
 extend_info_t *
@@ -1761,7 +2432,7 @@ hs_client_get_random_intro_from_edge(const edge_connection_t *edge_conn)
     rend_client_get_random_intro(edge_conn->rend_data);
 }
 
-/* Called when get an INTRODUCE_ACK cell on the introduction circuit circ.
+/** Called when get an INTRODUCE_ACK cell on the introduction circuit circ.
  * Return 0 on success else a negative value is returned. The circuit will be
  * closed or reuse to extend again to another intro point. */
 int
@@ -1790,7 +2461,7 @@ hs_client_receive_introduce_ack(origin_circuit_t *circ,
   return ret;
 }
 
-/* Called when get a RENDEZVOUS2 cell on the rendezvous circuit circ.  Return
+/** Called when get a RENDEZVOUS2 cell on the rendezvous circuit circ.  Return
  * 0 on success else a negative value is returned. The circuit will be closed
  * on error. */
 int
@@ -1823,7 +2494,7 @@ hs_client_receive_rendezvous2(origin_circuit_t *circ,
   return ret;
 }
 
-/* Extend the introduction circuit circ to another valid introduction point
+/** Extend the introduction circuit circ to another valid introduction point
  * for the hidden service it is trying to connect to, or mark it and launch a
  * new circuit if we can't extend it.  Return 0 on success or possible
  * success. Return -1 and mark the introduction circuit for close on permanent
@@ -1873,7 +2544,7 @@ hs_client_reextend_intro_circuit(origin_circuit_t *circ)
   return ret;
 }
 
-/* Close all client introduction circuits related to the given descriptor.
+/** Close all client introduction circuits related to the given descriptor.
  * This is called with a descriptor that is about to get replaced in the
  * client cache.
  *
@@ -1905,7 +2576,7 @@ hs_client_close_intro_circuits_from_desc(const hs_descriptor_t *desc)
   }
 }
 
-/* Release all the storage held by the client subsystem. */
+/** Release all the storage held by the client subsystem. */
 void
 hs_client_free_all(void)
 {
@@ -1914,7 +2585,7 @@ hs_client_free_all(void)
   client_service_authorization_free_all();
 }
 
-/* Purge all potentially remotely-detectable state held in the hidden
+/** Purge all potentially remotely-detectable state held in the hidden
  * service client code. Called on SIGNAL NEWNYM. */
 void
 hs_client_purge_state(void)
@@ -1931,11 +2602,13 @@ hs_client_purge_state(void)
   hs_cache_purge_as_client();
   /* Purge the last hidden service request cache. */
   hs_purge_last_hid_serv_requests();
+  /* Purge ephemeral client authorization. */
+  purge_ephemeral_client_auth();
 
   log_info(LD_REND, "Hidden service client state has been purged.");
 }
 
-/* Called when our directory information has changed. */
+/** Called when our directory information has changed. */
 void
 hs_client_dir_info_changed(void)
 {
@@ -1947,10 +2620,10 @@ hs_client_dir_info_changed(void)
 
 #ifdef TOR_UNIT_TESTS
 
-STATIC digest256map_t *
-get_hs_client_auths_map(void)
+STATIC void
+set_hs_client_auths_map(digest256map_t *map)
 {
-  return client_auths;
+  client_auths = map;
 }
 
 #endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h
index dadfa024b8..88dede8126 100644
--- a/src/feature/hs/hs_client.h
+++ b/src/feature/hs/hs_client.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,41 +10,97 @@
 #define TOR_HS_CLIENT_H
 
 #include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_descriptor.h"
 #include "feature/hs/hs_ident.h"
 
-/* Status code of a descriptor fetch request. */
+/** Status code of a descriptor fetch request. */
 typedef enum {
-  /* Something internally went wrong. */
+  /** Something internally went wrong. */
   HS_CLIENT_FETCH_ERROR        = -1,
-  /* The fetch request has been launched successfully. */
+  /** The fetch request has been launched successfully. */
   HS_CLIENT_FETCH_LAUNCHED     = 0,
-  /* We already have a usable descriptor. No fetch. */
+  /** We already have a usable descriptor. No fetch. */
   HS_CLIENT_FETCH_HAVE_DESC    = 1,
-  /* No more HSDir available to query. */
+  /** No more HSDir available to query. */
   HS_CLIENT_FETCH_NO_HSDIRS    = 2,
-  /* The fetch request is not allowed. */
+  /** The fetch request is not allowed. */
   HS_CLIENT_FETCH_NOT_ALLOWED  = 3,
-  /* We are missing information to be able to launch a request. */
+  /** We are missing information to be able to launch a request. */
   HS_CLIENT_FETCH_MISSING_INFO = 4,
-  /* There is a pending fetch for the requested service. */
+  /** There is a pending fetch for the requested service. */
   HS_CLIENT_FETCH_PENDING      = 5,
 } hs_client_fetch_status_t;
 
-/** Client-side configuration of authorization for a service. */
+/* Status code of client auth credential registration */
+typedef enum {
+  /* We successfuly registered these credentials */
+  REGISTER_SUCCESS,
+  /* We successfully registered these credentials, but had to replace some
+   * existing ones. */
+  REGISTER_SUCCESS_ALREADY_EXISTS,
+  /* We successfuly registered these credentials, and also decrypted a cached
+   * descriptor. */
+  REGISTER_SUCCESS_AND_DECRYPTED,
+  /* We failed to register these credentials, because of a bad HS address. */
+  REGISTER_FAIL_BAD_ADDRESS,
+  /* We failed to store these credentials in a persistent file on disk. */
+  REGISTER_FAIL_PERMANENT_STORAGE,
+} hs_client_register_auth_status_t;
+
+/* Status code of client auth credential removal */
+typedef enum {
+  /* We successfuly removed these credentials */
+  REMOVAL_SUCCESS,
+  /* No need to remove those credentials, because they were not there. */
+  REMOVAL_SUCCESS_NOT_FOUND,
+  /* We failed to register these credentials, because of a bad HS address. */
+  REMOVAL_BAD_ADDRESS,
+} hs_client_removal_auth_status_t;
+
+/** Flag to set when a client auth is permanent (saved on disk). */
+#define CLIENT_AUTH_FLAG_IS_PERMANENT (1<<0)
+
+/** Client-side configuration of client authorization */
 typedef struct hs_client_service_authorization_t {
-  /* An curve25519 secret key used to compute decryption keys that
+  /** An curve25519 secret key used to compute decryption keys that
    * allow the client to decrypt the hidden service descriptor. */
   curve25519_secret_key_t enc_seckey;
 
-  /* An onion address that is used to connect to the onion service. */
+  /** An onion address that is used to connect to the onion service. */
   char onion_address[HS_SERVICE_ADDR_LEN_BASE32+1];
+
+  /** An client name used to connect to the onion service. */
+  char *client_name;
+
+  /* Optional flags for this client. */
+  int flags;
 } hs_client_service_authorization_t;
 
+hs_client_register_auth_status_t
+hs_client_register_auth_credentials(hs_client_service_authorization_t *creds);
+
+hs_client_removal_auth_status_t
+hs_client_remove_auth_credentials(const char *hsaddress);
+
+digest256map_t *get_hs_client_auths_map(void);
+
+#define client_service_authorization_free(auth)                      \
+  FREE_AND_NULL(hs_client_service_authorization_t,                   \
+                client_service_authorization_free_, (auth))
+
+void
+client_service_authorization_free_(hs_client_service_authorization_t *auth);
+
 void hs_client_note_connection_attempt_succeeded(
                                        const edge_connection_t *conn);
 
-int hs_client_decode_descriptor(
+void hs_client_launch_v3_desc_fetch(
+                               const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs);
+
+hs_desc_decode_status_t hs_client_decode_descriptor(
                      const char *desc_str,
                      const ed25519_public_key_t *service_identity_pk,
                      hs_descriptor_t **desc);
@@ -57,6 +113,8 @@ int hs_client_send_introduce1(origin_circuit_t *intro_circ,
                               origin_circuit_t *rend_circ);
 
 void hs_client_circuit_has_opened(origin_circuit_t *circ);
+void hs_client_circuit_cleanup_on_close(const circuit_t *circ);
+void hs_client_circuit_cleanup_on_free(const circuit_t *circ);
 
 int hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
                                        const uint8_t *payload,
@@ -68,7 +126,8 @@ int hs_client_receive_rendezvous2(origin_circuit_t *circ,
                                   const uint8_t *payload,
                                   size_t payload_len);
 
-void hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident);
+void hs_client_dir_fetch_done(dir_connection_t *dir_conn, const char *reason,
+                              const char *body, const int status_code);
 
 extend_info_t *hs_client_get_random_intro_from_edge(
                                           const edge_connection_t *edge_conn);
@@ -107,13 +166,14 @@ MOCK_DECL(STATIC hs_client_fetch_status_t,
 
 STATIC void retry_all_socks_conn_waiting_for_desc(void);
 
+STATIC void purge_ephemeral_client_auth(void);
+
 #ifdef TOR_UNIT_TESTS
 
-STATIC digest256map_t *get_hs_client_auths_map(void);
+STATIC void set_hs_client_auths_map(digest256map_t *map);
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
 #endif /* defined(HS_CLIENT_PRIVATE) */
 
 #endif /* !defined(TOR_HS_CLIENT_H) */
-
diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c
index de653037d1..86d3fcab7d 100644
--- a/src/feature/hs/hs_common.c
+++ b/src/feature/hs/hs_common.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,6 +21,8 @@
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_dos.h"
+#include "feature/hs/hs_ob.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_service.h"
 #include "feature/hs_common/shared_random_client.h"
@@ -31,6 +33,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendservice.h"
+#include "feature/relay/routermode.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
@@ -43,7 +46,7 @@
 /* Trunnel */
 #include "trunnel/ed25519_cert.h"
 
-/* Ed25519 Basepoint value. Taken from section 5 of
+/** Ed25519 Basepoint value. Taken from section 5 of
  * https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-03 */
 static const char *str_ed25519_basepoint =
   "(15112221349535400772501151409588531511"
@@ -85,7 +88,7 @@ set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
   return 0;
 }
 
-#else /* !(defined(HAVE_SYS_UN_H)) */
+#else /* !defined(HAVE_SYS_UN_H) */
 
 static int
 set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
@@ -105,7 +108,7 @@ add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
 
 #endif /* defined(HAVE_SYS_UN_H) */
 
-/* Helper function: The key is a digest that we compare to a node_t object
+/** Helper function: The key is a digest that we compare to a node_t object
  * current hsdir_index. */
 static int
 compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member)
@@ -115,7 +118,7 @@ compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member)
   return tor_memcmp(key, node->hsdir_index.fetch, DIGEST256_LEN);
 }
 
-/* Helper function: The key is a digest that we compare to a node_t object
+/** Helper function: The key is a digest that we compare to a node_t object
  * next hsdir_index. */
 static int
 compare_digest_to_store_first_hsdir_index(const void *_key,
@@ -126,7 +129,7 @@ compare_digest_to_store_first_hsdir_index(const void *_key,
   return tor_memcmp(key, node->hsdir_index.store_first, DIGEST256_LEN);
 }
 
-/* Helper function: The key is a digest that we compare to a node_t object
+/** Helper function: The key is a digest that we compare to a node_t object
  * next hsdir_index. */
 static int
 compare_digest_to_store_second_hsdir_index(const void *_key,
@@ -137,7 +140,7 @@ compare_digest_to_store_second_hsdir_index(const void *_key,
   return tor_memcmp(key, node->hsdir_index.store_second, DIGEST256_LEN);
 }
 
-/* Helper function: Compare two node_t objects current hsdir_index. */
+/** Helper function: Compare two node_t objects current hsdir_index. */
 static int
 compare_node_fetch_hsdir_index(const void **a, const void **b)
 {
@@ -148,7 +151,7 @@ compare_node_fetch_hsdir_index(const void **a, const void **b)
                     DIGEST256_LEN);
 }
 
-/* Helper function: Compare two node_t objects next hsdir_index. */
+/** Helper function: Compare two node_t objects next hsdir_index. */
 static int
 compare_node_store_first_hsdir_index(const void **a, const void **b)
 {
@@ -159,7 +162,7 @@ compare_node_store_first_hsdir_index(const void **a, const void **b)
                     DIGEST256_LEN);
 }
 
-/* Helper function: Compare two node_t objects next hsdir_index. */
+/** Helper function: Compare two node_t objects next hsdir_index. */
 static int
 compare_node_store_second_hsdir_index(const void **a, const void **b)
 {
@@ -170,7 +173,7 @@ compare_node_store_second_hsdir_index(const void **a, const void **b)
                     DIGEST256_LEN);
 }
 
-/* Allocate and return a string containing the path to filename in directory.
+/** Allocate and return a string containing the path to filename in directory.
  * This function will never return NULL. The caller must free this path. */
 char *
 hs_path_from_filename(const char *directory, const char *filename)
@@ -184,8 +187,9 @@ hs_path_from_filename(const char *directory, const char *filename)
   return file_path;
 }
 
-/* Make sure that the directory for <b>service</b> is private, using the config
- * <b>username</b>.
+/** Make sure that the directory for <b>service</b> is private, using the
+ * config <b>username</b>.
+ *
  * If <b>create</b> is true:
  *  - if the directory exists, change permissions if needed,
  *  - if the directory does not exist, create it with the correct permissions.
@@ -305,18 +309,18 @@ hs_get_next_time_period_num(time_t now)
   return hs_get_time_period_num(now) + 1;
 }
 
-/* Get the number of the _previous_ HS time period, given that the current time
- * is <b>now</b>. If <b>now</b> is not set, we try to get the time from a live
- * consensus. */
+/** Get the number of the _previous_ HS time period, given that the current
+ * time is <b>now</b>. If <b>now</b> is not set, we try to get the time from a
+ * live consensus. */
 uint64_t
 hs_get_previous_time_period_num(time_t now)
 {
   return hs_get_time_period_num(now) - 1;
 }
 
-/* Return the start time of the upcoming time period based on <b>now</b>. If
-   <b>now</b> is not set, we try to get the time ourselves from a live
-   consensus. */
+/** Return the start time of the upcoming time period based on <b>now</b>. If
+ * <b>now</b> is not set, we try to get the time ourselves from a live
+ * consensus. */
 time_t
 hs_get_start_time_of_next_time_period(time_t now)
 {
@@ -331,7 +335,7 @@ hs_get_start_time_of_next_time_period(time_t now)
   return (time_t)(start_of_next_tp_in_mins * 60 + time_period_rotation_offset);
 }
 
-/* Create a new rend_data_t for a specific given <b>version</b>.
+/** Create a new rend_data_t for a specific given <b>version</b>.
  * Return a pointer to the newly allocated data structure. */
 static rend_data_t *
 rend_data_alloc(uint32_t version)
@@ -380,7 +384,7 @@ rend_data_free_(rend_data_t *data)
   }
 }
 
-/* Allocate and return a deep copy of <b>data</b>. */
+/** Allocate and return a deep copy of <b>data</b>. */
 rend_data_t *
 rend_data_dup(const rend_data_t *data)
 {
@@ -410,7 +414,7 @@ rend_data_dup(const rend_data_t *data)
   return data_dup;
 }
 
-/* Compute the descriptor ID for each HS descriptor replica and save them. A
+/** Compute the descriptor ID for each HS descriptor replica and save them. A
  * valid onion address must be present in the <b>rend_data</b>.
  *
  * Return 0 on success else -1. */
@@ -448,7 +452,7 @@ compute_desc_id(rend_data_t *rend_data)
   return ret;
 }
 
-/* Allocate and initialize a rend_data_t object for a service using the
+/** Allocate and initialize a rend_data_t object for a service using the
  * provided arguments. All arguments are optional (can be NULL), except from
  * <b>onion_address</b> which MUST be set. The <b>pk_digest</b> is the hash of
  * the service private key. The <b>cookie</b> is the rendezvous cookie and
@@ -480,7 +484,7 @@ rend_data_service_create(const char *onion_address, const char *pk_digest,
   return rend_data;
 }
 
-/* Allocate and initialize a rend_data_t object for a client request using the
+/** Allocate and initialize a rend_data_t object for a client request using the
  * given arguments. Either an onion address or a descriptor ID is needed. Both
  * can be given but in this case only the onion address will be used to make
  * the descriptor fetch. The <b>cookie</b> is the rendezvous cookie and
@@ -521,7 +525,7 @@ rend_data_client_create(const char *onion_address, const char *desc_id,
   return NULL;
 }
 
-/* Return the onion address from the rend data. Depending on the version,
+/** Return the onion address from the rend data. Depending on the version,
  * the size of the address can vary but it's always NUL terminated. */
 const char *
 rend_data_get_address(const rend_data_t *rend_data)
@@ -537,7 +541,7 @@ rend_data_get_address(const rend_data_t *rend_data)
   }
 }
 
-/* Return the descriptor ID for a specific replica number from the rend
+/** Return the descriptor ID for a specific replica number from the rend
  * data. The returned data is a binary digest and depending on the version its
  * size can vary. The size of the descriptor ID is put in <b>len_out</b> if
  * non NULL. */
@@ -560,7 +564,7 @@ rend_data_get_desc_id(const rend_data_t *rend_data, uint8_t replica,
   }
 }
 
-/* Return the public key digest using the given <b>rend_data</b>. The size of
+/** Return the public key digest using the given <b>rend_data</b>. The size of
  * the digest is put in <b>len_out</b> (if set) which can differ depending on
  * the version. */
 const uint8_t *
@@ -583,7 +587,7 @@ rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out)
   }
 }
 
-/* Using the given time period number, compute the disaster shared random
+/** Using the given time period number, compute the disaster shared random
  * value and put it in srv_out. It MUST be at least DIGEST256_LEN bytes. */
 static void
 compute_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
@@ -671,7 +675,7 @@ get_second_cached_disaster_srv(void)
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
-/* When creating a blinded key, we need a parameter which construction is as
+/** When creating a blinded key, we need a parameter which construction is as
  * follow: H(pubkey | [secret] | ed25519-basepoint | nonce).
  *
  * The nonce has a pre-defined format which uses the time period number
@@ -725,7 +729,7 @@ build_blinded_key_param(const ed25519_public_key_t *pubkey,
   memwipe(nonce, 0, sizeof(nonce));
 }
 
-/* Using an ed25519 public key and version to build the checksum of an
+/** Using an ed25519 public key and version to build the checksum of an
  * address. Put in checksum_out. Format is:
  *    SHA3-256(".onion checksum" || PUBKEY || VERSION)
  *
@@ -752,7 +756,7 @@ build_hs_checksum(const ed25519_public_key_t *key, uint8_t version,
                    DIGEST_SHA3_256);
 }
 
-/* Using an ed25519 public key, checksum and version to build the binary
+/** Using an ed25519 public key, checksum and version to build the binary
  * representation of a service address. Put in addr_out. Format is:
  *    addr_out = PUBKEY || CHECKSUM || VERSION
  *
@@ -775,7 +779,7 @@ build_hs_address(const ed25519_public_key_t *key, const uint8_t *checksum,
   tor_assert(offset == HS_SERVICE_ADDR_LEN);
 }
 
-/* Helper for hs_parse_address(): Using a binary representation of a service
+/** Helper for hs_parse_address(): Using a binary representation of a service
  * address, parse its content into the key_out, checksum_out and version_out.
  * Any out variable can be NULL in case the caller would want only one field.
  * checksum_out MUST at least be 2 bytes long. address must be at least
@@ -807,13 +811,13 @@ hs_parse_address_impl(const char *address, ed25519_public_key_t *key_out,
   tor_assert(offset == HS_SERVICE_ADDR_LEN);
 }
 
-/* Using the given identity public key and a blinded public key, compute the
- * subcredential and put it in subcred_out (must be of size DIGEST256_LEN).
+/** Using the given identity public key and a blinded public key, compute the
+ * subcredential and put it in subcred_out.
  * This can't fail. */
 void
 hs_get_subcredential(const ed25519_public_key_t *identity_pk,
                      const ed25519_public_key_t *blinded_pk,
-                     uint8_t *subcred_out)
+                     hs_subcredential_t *subcred_out)
 {
   uint8_t credential[DIGEST256_LEN];
   crypto_digest_t *digest;
@@ -841,13 +845,14 @@ hs_get_subcredential(const ed25519_public_key_t *identity_pk,
                           sizeof(credential));
   crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
                           ED25519_PUBKEY_LEN);
-  crypto_digest_get_digest(digest, (char *) subcred_out, DIGEST256_LEN);
+  crypto_digest_get_digest(digest, (char *) subcred_out->subcred,
+                           SUBCRED_LEN);
   crypto_digest_free(digest);
 
   memwipe(credential, 0, sizeof(credential));
 }
 
-/* From the given list of hidden service ports, find the ones that match the
+/** From the given list of hidden service ports, find the ones that match the
  * given edge connection conn, pick one at random and use it to set the
  * connection address. Return 0 on success or -1 if none. */
 int
@@ -904,34 +909,40 @@ hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn)
   return (chosen_port) ? 0 : -1;
 }
 
-/* Using a base32 representation of a service address, parse its content into
+/** Using a base32 representation of a service address, parse its content into
  * the key_out, checksum_out and version_out. Any out variable can be NULL in
  * case the caller would want only one field. checksum_out MUST at least be 2
  * bytes long.
  *
- * Return 0 if parsing went well; return -1 in case of error. */
+ * Return 0 if parsing went well; return -1 in case of error and if errmsg is
+ * non NULL, a human readable string message is set. */
 int
-hs_parse_address(const char *address, ed25519_public_key_t *key_out,
-                 uint8_t *checksum_out, uint8_t *version_out)
+hs_parse_address_no_log(const char *address, ed25519_public_key_t *key_out,
+                        uint8_t *checksum_out, uint8_t *version_out,
+                        const char **errmsg)
 {
   char decoded[HS_SERVICE_ADDR_LEN];
 
   tor_assert(address);
 
+  if (errmsg) {
+    *errmsg = NULL;
+  }
+
   /* Obvious length check. */
   if (strlen(address) != HS_SERVICE_ADDR_LEN_BASE32) {
-    log_warn(LD_REND, "Service address %s has an invalid length. "
-                      "Expected %lu but got %lu.",
-             escaped_safe_str(address),
-             (unsigned long) HS_SERVICE_ADDR_LEN_BASE32,
-             (unsigned long) strlen(address));
+    if (errmsg) {
+      *errmsg = "Invalid length";
+    }
     goto invalid;
   }
 
   /* Decode address so we can extract needed fields. */
-  if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) {
-    log_warn(LD_REND, "Service address %s can't be decoded.",
-             escaped_safe_str(address));
+  if (base32_decode(decoded, sizeof(decoded), address, strlen(address))
+      != sizeof(decoded)) {
+    if (errmsg) {
+      *errmsg = "Unable to base32 decode";
+    }
     goto invalid;
   }
 
@@ -943,7 +954,23 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out,
   return -1;
 }
 
-/* Validate a given onion address. The length, the base32 decoding and
+/** Same has hs_parse_address_no_log() but emits a log warning on parsing
+ * failure. */
+int
+hs_parse_address(const char *address, ed25519_public_key_t *key_out,
+                 uint8_t *checksum_out, uint8_t *version_out)
+{
+  const char *errmsg = NULL;
+  int ret = hs_parse_address_no_log(address, key_out, checksum_out,
+                                    version_out, &errmsg);
+  if (ret < 0) {
+    log_warn(LD_REND, "Service address %s failed to be parsed: %s",
+             escaped_safe_str(address), errmsg);
+  }
+  return ret;
+}
+
+/** Validate a given onion address. The length, the base32 decoding, and
  * checksum are validated. Return 1 if valid else 0. */
 int
 hs_address_is_valid(const char *address)
@@ -958,7 +985,7 @@ hs_address_is_valid(const char *address)
     goto invalid;
   }
 
-  /* Get the checksum it's suppose to be and compare it with what we have
+  /* Get the checksum it's supposed to be and compare it with what we have
    * encoded in the address. */
   build_hs_checksum(&service_pubkey, version, target_checksum);
   if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) {
@@ -982,11 +1009,11 @@ hs_address_is_valid(const char *address)
   return 0;
 }
 
-/* Build a service address using an ed25519 public key and a given version.
+/** Build a service address using an ed25519 public key and a given version.
  * The returned address is base32 encoded and put in addr_out. The caller MUST
  * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long.
  *
- * Format is as follow:
+ * Format is as follows:
  *     base32(PUBKEY || CHECKSUM || VERSION)
  *     CHECKSUM = H(".onion checksum" || PUBKEY || VERSION)
  * */
@@ -1012,25 +1039,7 @@ hs_build_address(const ed25519_public_key_t *key, uint8_t version,
   tor_assert(hs_address_is_valid(addr_out));
 }
 
-/* Return a newly allocated copy of lspec. */
-link_specifier_t *
-hs_link_specifier_dup(const link_specifier_t *lspec)
-{
-  link_specifier_t *result = link_specifier_new();
-  memcpy(result, lspec, sizeof(*result));
-  /* The unrecognized field is a dynamic array so make sure to copy its
-   * content and not the pointer. */
-  link_specifier_setlen_un_unrecognized(
-                  result, link_specifier_getlen_un_unrecognized(lspec));
-  if (link_specifier_getlen_un_unrecognized(result)) {
-    memcpy(link_specifier_getarray_un_unrecognized(result),
-           link_specifier_getconstarray_un_unrecognized(lspec),
-           link_specifier_getlen_un_unrecognized(result));
-  }
-  return result;
-}
-
-/* From a given ed25519 public key pk and an optional secret, compute a
+/** From a given ed25519 public key pk and an optional secret, compute a
  * blinded public key and put it in blinded_pk_out. This is only useful to
  * the client side because the client only has access to the identity public
  * key of the service. */
@@ -1045,7 +1054,7 @@ hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
 
   tor_assert(pk);
   tor_assert(blinded_pk_out);
-  tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
 
   build_blinded_key_param(pk, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1054,7 +1063,7 @@ hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
   memwipe(param, 0, sizeof(param));
 }
 
-/* From a given ed25519 keypair kp and an optional secret, compute a blinded
+/** From a given ed25519 keypair kp and an optional secret, compute a blinded
  * keypair for the current time period and put it in blinded_kp_out. This is
  * only useful by the service side because the client doesn't have access to
  * the identity secret key. */
@@ -1070,8 +1079,8 @@ hs_build_blinded_keypair(const ed25519_keypair_t *kp,
   tor_assert(kp);
   tor_assert(blinded_kp_out);
   /* Extra safety. A zeroed key is bad. */
-  tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
-  tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
 
   build_blinded_key_param(&kp->pubkey, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1080,7 +1089,7 @@ hs_build_blinded_keypair(const ed25519_keypair_t *kp,
   memwipe(param, 0, sizeof(param));
 }
 
-/* Return true if we are currently in the time segment between a new time
+/** Return true if we are currently in the time segment between a new time
  * period and a new SRV (in the real network that happens between 12:00 and
  * 00:00 UTC). Here is a diagram showing exactly when this returns true:
  *
@@ -1121,7 +1130,7 @@ hs_in_period_between_tp_and_srv,(const networkstatus_t *consensus, time_t now))
   return 1;
 }
 
-/* Return 1 if any virtual port in ports needs a circuit with good uptime.
+/** Return 1 if any virtual port in ports needs a circuit with good uptime.
  * Else return 0. */
 int
 hs_service_requires_uptime_circ(const smartlist_t *ports)
@@ -1137,7 +1146,7 @@ hs_service_requires_uptime_circ(const smartlist_t *ports)
   return 0;
 }
 
-/* Build hs_index which is used to find the responsible hsdirs. This index
+/** Build hs_index which is used to find the responsible hsdirs. This index
  * value is used to select the responsible HSDir where their hsdir_index is
  * closest to this value.
  *    SHA3-256("store-at-idx" | blinded_public_key |
@@ -1179,7 +1188,7 @@ hs_build_hs_index(uint64_t replica, const ed25519_public_key_t *blinded_pk,
   crypto_digest_free(digest);
 }
 
-/* Build hsdir_index which is used to find the responsible hsdirs. This is the
+/** Build hsdir_index which is used to find the responsible hsdirs. This is the
  * index value that is compare to the hs_index when selecting an HSDir.
  *    SHA3-256("node-idx" | node_identity |
  *             shared_random_value | INT_8(period_length) | INT_8(period_num) )
@@ -1220,7 +1229,7 @@ hs_build_hsdir_index(const ed25519_public_key_t *identity_pk,
   crypto_digest_free(digest);
 }
 
-/* Return a newly allocated buffer containing the current shared random value
+/** Return a newly allocated buffer containing the current shared random value
  * or if not present, a disaster value is computed using the given time period
  * number. If a consensus is provided in <b>ns</b>, use it to get the SRV
  * value. This function can't fail. */
@@ -1239,7 +1248,7 @@ hs_get_current_srv(uint64_t time_period_num, const networkstatus_t *ns)
   return sr_value;
 }
 
-/* Return a newly allocated buffer containing the previous shared random
+/** Return a newly allocated buffer containing the previous shared random
  * value or if not present, a disaster value is computed using the given time
  * period number. This function can't fail. */
 uint8_t *
@@ -1257,7 +1266,7 @@ hs_get_previous_srv(uint64_t time_period_num, const networkstatus_t *ns)
   return sr_value;
 }
 
-/* Return the number of replicas defined by a consensus parameter or the
+/** Return the number of replicas defined by a consensus parameter or the
  * default value. */
 int32_t
 hs_get_hsdir_n_replicas(void)
@@ -1267,7 +1276,7 @@ hs_get_hsdir_n_replicas(void)
                                  HS_DEFAULT_HSDIR_N_REPLICAS, 1, 16);
 }
 
-/* Return the spread fetch value defined by a consensus parameter or the
+/** Return the spread fetch value defined by a consensus parameter or the
  * default value. */
 int32_t
 hs_get_hsdir_spread_fetch(void)
@@ -1277,7 +1286,7 @@ hs_get_hsdir_spread_fetch(void)
                                  HS_DEFAULT_HSDIR_SPREAD_FETCH, 1, 128);
 }
 
-/* Return the spread store value defined by a consensus parameter or the
+/** Return the spread store value defined by a consensus parameter or the
  * default value. */
 int32_t
 hs_get_hsdir_spread_store(void)
@@ -1304,15 +1313,15 @@ node_has_hsdir_index(const node_t *node)
 
   /* At this point, since the node has a desc, this node must also have an
    * hsdir index. If not, something went wrong, so BUG out. */
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.fetch,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_first,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_second,
                           DIGEST256_LEN))) {
     return 0;
   }
@@ -1320,7 +1329,7 @@ node_has_hsdir_index(const node_t *node)
   return 1;
 }
 
-/* For a given blinded key and time period number, get the responsible HSDir
+/** For a given blinded key and time period number, get the responsible HSDir
  * and put their routerstatus_t object in the responsible_dirs list. If
  * 'use_second_hsdir_index' is true, use the second hsdir_index of the node_t
  * is used. If 'for_fetching' is true, the spread fetch consensus parameter is
@@ -1612,20 +1621,25 @@ hs_purge_last_hid_serv_requests(void)
 /** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the
  *  one that we should use to fetch a descriptor right now. Take into account
  *  previous failed attempts at fetching this descriptor from HSDirs using the
- *  string identifier <b>req_key_str</b>.
+ *  string identifier <b>req_key_str</b>. We return whether we are rate limited
+ *  into *<b>is_rate_limited_out</b> if it is not NULL.
  *
  *  Steals ownership of <b>responsible_dirs</b>.
  *
  *  Return the routerstatus of the chosen HSDir if successful, otherwise return
  *  NULL if no HSDirs are worth trying right now. */
 routerstatus_t *
-hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
+hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str,
+              bool *is_rate_limited_out)
 {
   smartlist_t *usable_responsible_dirs = smartlist_new();
   const or_options_t *options = get_options();
   routerstatus_t *hs_dir;
   time_t now = time(NULL);
   int excluded_some;
+  bool rate_limited = false;
+  int rate_limited_count = 0;
+  int responsible_dirs_count = smartlist_len(responsible_dirs);
 
   tor_assert(req_key_str);
 
@@ -1645,6 +1659,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     if (last + hs_hsdir_requery_period(options) >= now ||
         !node || !node_has_preferred_descriptor(node, 0)) {
       SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
+      rate_limited_count++;
       continue;
     }
     if (!routerset_contains_node(options->ExcludeNodes, node)) {
@@ -1652,6 +1667,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     }
   } SMARTLIST_FOREACH_END(dir);
 
+  if (rate_limited_count > 0 || responsible_dirs_count > 0) {
+    rate_limited = rate_limited_count == responsible_dirs_count;
+  }
+
   excluded_some =
     smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
 
@@ -1663,9 +1682,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
   smartlist_free(responsible_dirs);
   smartlist_free(usable_responsible_dirs);
   if (!hs_dir) {
+    const char *warn_str = (rate_limited) ? "we are rate limited." :
+                              "we requested them all recently without success";
     log_info(LD_REND, "Could not pick one of the responsible hidden "
-                      "service directories, because we requested them all "
-                      "recently without success.");
+                      "service directories, because %s.", warn_str);
     if (options->StrictNodes && excluded_some) {
       log_warn(LD_REND, "Could not pick a hidden service directory for the "
                "requested hidden service: they are all either down or "
@@ -1677,17 +1697,23 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1);
   }
 
+  if (is_rate_limited_out != NULL) {
+    *is_rate_limited_out = rate_limited;
+  }
+
   return hs_dir;
 }
 
-/* From a list of link specifier, an onion key and if we are requesting a
- * direct connection (ex: single onion service), return a newly allocated
- * extend_info_t object. This function always returns an extend info with
- * an IPv4 address, or NULL.
+/** Given a list of link specifiers lspecs, a curve 25519 onion_key, and
+ * a direct connection boolean direct_conn (true for single onion services),
+ * return a newly allocated extend_info_t object.
+ *
+ * This function always returns an extend info with a valid IP address and
+ * ORPort, or NULL. If direct_conn is false, the IP address is always IPv4.
  *
  * It performs the following checks:
- *  if either IPv4 or legacy ID is missing, return NULL.
- *  if direct_conn, and we can't reach the IPv4 address, return NULL.
+ *  if there is no usable IP address, or legacy ID is missing, return NULL.
+ *  if direct_conn, and we can't reach any IP address, return NULL.
  */
 extend_info_t *
 hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
@@ -1696,21 +1722,40 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
 {
   int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0;
   char legacy_id[DIGEST_LEN] = {0};
-  uint16_t port_v4 = 0;
-  tor_addr_t addr_v4;
   ed25519_public_key_t ed25519_pk;
   extend_info_t *info = NULL;
+  tor_addr_port_t ap;
 
-  tor_assert(lspecs);
+  tor_addr_make_null(&ap.addr, AF_UNSPEC);
+  ap.port = 0;
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Specified link specifiers is null");
+    goto done;
+  }
+
+  if (onion_key == NULL) {
+    log_warn(LD_BUG, "Specified onion key is null");
+    goto done;
+  }
+
+  if (smartlist_len(lspecs) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND, "Empty link specifier list.");
+    /* Return NULL. */
+    goto done;
+  }
 
   SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
     switch (link_specifier_get_ls_type(ls)) {
     case LS_IPV4:
-      /* Skip if we already seen a v4. */
-      if (have_v4) continue;
-      tor_addr_from_ipv4h(&addr_v4,
+      /* Skip if we already seen a v4. If direct_conn is true, we skip this
+       * block because fascist_firewall_choose_address_ls() will set ap. If
+       * direct_conn is false, set ap to the first IPv4 address and port in
+       * the link specifiers.*/
+      if (have_v4 || direct_conn) continue;
+      tor_addr_from_ipv4h(&ap.addr,
                           link_specifier_get_un_ipv4_addr(ls));
-      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      ap.port = link_specifier_get_un_ipv4_port(ls);
       have_v4 = 1;
       break;
     case LS_LEGACY_ID:
@@ -1734,52 +1779,45 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
     }
   } SMARTLIST_FOREACH_END(ls);
 
-  /* Legacy ID is mandatory, and we require IPv4. */
-  if (!have_v4 || !have_legacy_id) {
+  /* Choose a preferred address first, but fall back to an allowed address. */
+  if (direct_conn)
+    fascist_firewall_choose_address_ls(lspecs, 0, &ap);
+
+  /* Legacy ID is mandatory, and we require an IP address. */
+  if (!tor_addr_port_is_valid_ap(&ap, 0)) {
+    /* If we're missing the IP address, log a warning and return NULL. */
+    log_info(LD_NET, "Unreachable or invalid IP address in link state");
     goto done;
   }
-
-  /* We know we have IPv4, because we just checked. */
-  if (!direct_conn) {
-    /* All clients can extend to any IPv4 via a 3-hop path. */
-    goto validate;
-  } else if (direct_conn &&
-             fascist_firewall_allows_address_addr(&addr_v4, port_v4,
-                                                  FIREWALL_OR_CONNECTION,
-                                                  0, 0)) {
-    /* Direct connection and we can reach it in IPv4 so go for it. */
-    goto validate;
-
-    /* We will add support for falling back to a 3-hop path in a later
-     * release. */
-  } else {
-    /* If we can't reach IPv4, return NULL. */
+  if (!have_legacy_id) {
+    /* If we're missing the legacy ID, log a warning and return NULL. */
+    log_warn(LD_PROTOCOL, "Missing Legacy ID in link state");
     goto done;
   }
 
-  /* We will add support for IPv6 in a later release. */
+  /* We will add support for falling back to a 3-hop path in a later
+   * release. */
 
- validate:
   /* We'll validate now that the address we've picked isn't a private one. If
-   * it is, are we allowing to extend to private address? */
-  if (!extend_info_addr_is_allowed(&addr_v4)) {
+   * it is, are we allowed to extend to private addresses? */
+  if (!extend_info_addr_is_allowed(&ap.addr)) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Requested address is private and we are not allowed to extend to "
-           "it: %s:%u", fmt_addr(&addr_v4), port_v4);
+           "it: %s:%u", fmt_addr(&ap.addr), ap.port);
     goto done;
   }
 
   /* We do have everything for which we think we can connect successfully. */
   info = extend_info_new(NULL, legacy_id,
                          (have_ed25519_id) ? &ed25519_pk : NULL, NULL,
-                         onion_key, &addr_v4, port_v4);
+                         onion_key, &ap.addr, ap.port);
  done:
   return info;
 }
 
 /***********************************************************************/
 
-/* Initialize the entire HS subsytem. This is called in tor_init() before any
+/** Initialize the entire HS subsytem. This is called in tor_init() before any
  * torrc options are loaded. Only for >= v3. */
 void
 hs_init(void)
@@ -1789,7 +1827,7 @@ hs_init(void)
   hs_cache_init();
 }
 
-/* Release and cleanup all memory of the HS subsystem (all version). This is
+/** Release and cleanup all memory of the HS subsystem (all version). This is
  * called by tor_free_all(). */
 void
 hs_free_all(void)
@@ -1798,9 +1836,10 @@ hs_free_all(void)
   hs_service_free_all();
   hs_cache_free_all();
   hs_client_free_all();
+  hs_ob_free_all();
 }
 
-/* For the given origin circuit circ, decrement the number of rendezvous
+/** For the given origin circuit circ, decrement the number of rendezvous
  * stream counter. This handles every hidden service version. */
 void
 hs_dec_rdv_stream_counter(origin_circuit_t *circ)
@@ -1817,7 +1856,7 @@ hs_dec_rdv_stream_counter(origin_circuit_t *circ)
   }
 }
 
-/* For the given origin circuit circ, increment the number of rendezvous
+/** For the given origin circuit circ, increment the number of rendezvous
  * stream counter. This handles every hidden service version. */
 void
 hs_inc_rdv_stream_counter(origin_circuit_t *circ)
@@ -1833,3 +1872,42 @@ hs_inc_rdv_stream_counter(origin_circuit_t *circ)
     tor_assert_nonfatal_unreached();
   }
 }
+
+/** Return a newly allocated link specifier object that is a copy of dst. */
+link_specifier_t *
+link_specifier_dup(const link_specifier_t *src)
+{
+  link_specifier_t *dup = NULL;
+  uint8_t *buf = NULL;
+
+  if (BUG(!src)) {
+    goto err;
+  }
+
+  ssize_t encoded_len_alloc = link_specifier_encoded_len(src);
+  if (BUG(encoded_len_alloc < 0)) {
+    goto err;
+  }
+
+  buf = tor_malloc_zero(encoded_len_alloc);
+  ssize_t encoded_len_data = link_specifier_encode(buf,
+                                                   encoded_len_alloc,
+                                                   src);
+  if (BUG(encoded_len_data < 0)) {
+    goto err;
+  }
+
+  ssize_t parsed_len = link_specifier_parse(&dup, buf, encoded_len_alloc);
+  if (BUG(parsed_len < 0)) {
+    goto err;
+  }
+
+  goto done;
+
+ err:
+  dup = NULL;
+
+ done:
+  tor_free(buf);
+  return dup;
+}
diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h
index a44505930a..997b7298a6 100644
--- a/src/feature/hs/hs_common.h
+++ b/src/feature/hs/hs_common.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,13 +19,14 @@ struct ed25519_keypair_t;
 /* Trunnel */
 #include "trunnel/ed25519_cert.h"
 
-/* Protocol version 2. Use this instead of hardcoding "2" in the code base,
+/** Protocol version 2. Use this instead of hardcoding "2" in the code base,
  * this adds a clearer semantic to the value when used. */
 #define HS_VERSION_TWO 2
-/* Version 3 of the protocol (prop224). */
+/** Version 3 of the protocol (prop224). */
 #define HS_VERSION_THREE 3
-/* Earliest and latest version we support. */
+/** Earliest version we support. */
 #define HS_VERSION_MIN HS_VERSION_TWO
+/** Latest version we support. */
 #define HS_VERSION_MAX HS_VERSION_THREE
 
 /** Try to maintain this many intro points per service by default. */
@@ -48,94 +49,95 @@ struct ed25519_keypair_t;
  * rendezvous point before giving up? */
 #define MAX_REND_TIMEOUT 30
 
-/* String prefix for the signature of ESTABLISH_INTRO */
+/** String prefix for the signature of ESTABLISH_INTRO */
 #define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1"
 
-/* The default HS time period length */
+/** The default HS time period length */
 #define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
 
-/* Prefix of the onion address checksum. */
+/** Prefix of the onion address checksum. */
 #define HS_SERVICE_ADDR_CHECKSUM_PREFIX ".onion checksum"
-/* Length of the checksum prefix minus the NUL terminated byte. */
+/** Length of the checksum prefix minus the NUL terminated byte. */
 #define HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN \
   (sizeof(HS_SERVICE_ADDR_CHECKSUM_PREFIX) - 1)
-/* Length of the resulting checksum of the address. The construction of this
+/** Length of the resulting checksum of the address. The construction of this
  * checksum looks like:
  *   CHECKSUM = ".onion checksum" || PUBKEY || VERSION
  * where VERSION is 1 byte. This is pre-hashing. */
 #define HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN \
   (HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN + ED25519_PUBKEY_LEN + sizeof(uint8_t))
-/* The amount of bytes we use from the address checksum. */
+/** The amount of bytes we use from the address checksum. */
 #define HS_SERVICE_ADDR_CHECKSUM_LEN_USED 2
-/* Length of the binary encoded service address which is of course before the
+/** Length of the binary encoded service address which is of course before the
  * base32 encoding. Construction is:
  *    PUBKEY || CHECKSUM || VERSION
  * with 1 byte VERSION and 2 bytes CHECKSUM. The following is 35 bytes. */
 #define HS_SERVICE_ADDR_LEN \
   (ED25519_PUBKEY_LEN + HS_SERVICE_ADDR_CHECKSUM_LEN_USED + sizeof(uint8_t))
-/* Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the
+/** Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the
  * length ends up to 56 bytes (not counting the terminated NUL byte.) */
 #define HS_SERVICE_ADDR_LEN_BASE32 \
   (CEIL_DIV(HS_SERVICE_ADDR_LEN * 8, 5))
 
-/* The default HS time period length */
+/** The default HS time period length */
 #define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
-/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */
+/** The time period rotation offset as seen in prop224 section
+ * [TIME-PERIODS] */
 #define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */
 
-/* Keyblinding parameter construction is as follow:
+/** Keyblinding parameter construction is as follow:
  *    "key-blind" || INT_8(period_num) || INT_8(start_period_sec) */
 #define HS_KEYBLIND_NONCE_PREFIX "key-blind"
 #define HS_KEYBLIND_NONCE_PREFIX_LEN (sizeof(HS_KEYBLIND_NONCE_PREFIX) - 1)
 #define HS_KEYBLIND_NONCE_LEN \
   (HS_KEYBLIND_NONCE_PREFIX_LEN + sizeof(uint64_t) + sizeof(uint64_t))
 
-/* Credential and subcredential prefix value. */
+/** Credential and subcredential prefix value. */
 #define HS_CREDENTIAL_PREFIX "credential"
 #define HS_CREDENTIAL_PREFIX_LEN (sizeof(HS_CREDENTIAL_PREFIX) - 1)
 #define HS_SUBCREDENTIAL_PREFIX "subcredential"
 #define HS_SUBCREDENTIAL_PREFIX_LEN (sizeof(HS_SUBCREDENTIAL_PREFIX) - 1)
 
-/* Node hidden service stored at index prefix value. */
+/** Node hidden service stored at index prefix value. */
 #define HS_INDEX_PREFIX "store-at-idx"
 #define HS_INDEX_PREFIX_LEN (sizeof(HS_INDEX_PREFIX) - 1)
 
-/* Node hidden service directory index prefix value. */
+/** Node hidden service directory index prefix value. */
 #define HSDIR_INDEX_PREFIX "node-idx"
 #define HSDIR_INDEX_PREFIX_LEN (sizeof(HSDIR_INDEX_PREFIX) - 1)
 
-/* Prefix of the shared random value disaster mode. */
+/** Prefix of the shared random value disaster mode. */
 #define HS_SRV_DISASTER_PREFIX "shared-random-disaster"
 #define HS_SRV_DISASTER_PREFIX_LEN (sizeof(HS_SRV_DISASTER_PREFIX) - 1)
 
-/* Default value of number of hsdir replicas (hsdir_n_replicas). */
+/** Default value of number of hsdir replicas (hsdir_n_replicas). */
 #define HS_DEFAULT_HSDIR_N_REPLICAS 2
-/* Default value of hsdir spread store (hsdir_spread_store). */
+/** Default value of hsdir spread store (hsdir_spread_store). */
 #define HS_DEFAULT_HSDIR_SPREAD_STORE 4
-/* Default value of hsdir spread fetch (hsdir_spread_fetch). */
+/** Default value of hsdir spread fetch (hsdir_spread_fetch). */
 #define HS_DEFAULT_HSDIR_SPREAD_FETCH 3
 
-/* The size of a legacy RENDEZVOUS1 cell which adds up to 168 bytes. It is
+/** The size of a legacy RENDEZVOUS1 cell which adds up to 168 bytes. It is
  * bigger than the 84 bytes needed for version 3 so we need to pad up to that
  * length so it is indistinguishable between versions. */
 #define HS_LEGACY_RENDEZVOUS_CELL_SIZE \
   (REND_COOKIE_LEN + DH1024_KEY_LEN + DIGEST_LEN)
 
-/* Type of authentication key used by an introduction point. */
+/** Type of authentication key used by an introduction point. */
 typedef enum {
   HS_AUTH_KEY_TYPE_LEGACY  = 1,
   HS_AUTH_KEY_TYPE_ED25519 = 2,
 } hs_auth_key_type_t;
 
-/* Return value when adding an ephemeral service through the ADD_ONION
+/** Return value when adding an ephemeral service through the ADD_ONION
  * control port command. Both v2 and v3 share these. */
 typedef enum {
   RSAE_BADAUTH     = -5, /**< Invalid auth_type/auth_clients */
@@ -146,18 +148,18 @@ typedef enum {
   RSAE_OKAY        = 0   /**< Service added as expected */
 } hs_service_add_ephemeral_status_t;
 
-/* Represents the mapping from a virtual port of a rendezvous service to a
+/** Represents the mapping from a virtual port of a rendezvous service to a
  * real port on some IP. */
 typedef struct rend_service_port_config_t {
-  /* The incoming HS virtual port we're mapping */
+  /** The incoming HS virtual port we're mapping */
   uint16_t virtual_port;
-  /* Is this an AF_UNIX port? */
+  /** Is this an AF_UNIX port? */
   unsigned int is_unix_addr:1;
-  /* The outgoing TCP port to use, if !is_unix_addr */
+  /** The outgoing TCP port to use, if !is_unix_addr */
   uint16_t real_port;
-  /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
+  /** The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
   tor_addr_t real_addr;
-  /* The socket path to connect to, if is_unix_addr */
+  /** The socket path to connect to, if is_unix_addr */
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 } rend_service_port_config_t;
 
@@ -177,6 +179,10 @@ void hs_build_address(const struct ed25519_public_key_t *key, uint8_t version,
 int hs_address_is_valid(const char *address);
 int hs_parse_address(const char *address, struct ed25519_public_key_t *key_out,
                      uint8_t *checksum_out, uint8_t *version_out);
+int hs_parse_address_no_log(const char *address,
+                            struct ed25519_public_key_t *key_out,
+                            uint8_t *checksum_out, uint8_t *version_out,
+                            const char **errmsg);
 
 void hs_build_blinded_pubkey(const struct ed25519_public_key_t *pubkey,
                              const uint8_t *secret, size_t secret_len,
@@ -208,17 +214,16 @@ const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data,
 
 routerstatus_t *pick_hsdir(const char *desc_id, const char *desc_id_base32);
 
+struct hs_subcredential_t;
 void hs_get_subcredential(const struct ed25519_public_key_t *identity_pk,
                           const struct ed25519_public_key_t *blinded_pk,
-                          uint8_t *subcred_out);
+                          struct hs_subcredential_t *subcred_out);
 
 uint64_t hs_get_previous_time_period_num(time_t now);
 uint64_t hs_get_time_period_num(time_t now);
 uint64_t hs_get_next_time_period_num(time_t now);
 time_t hs_get_start_time_of_next_time_period(time_t now);
 
-link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec);
-
 MOCK_DECL(int, hs_in_period_between_tp_and_srv,
           (const networkstatus_t *consensus, time_t now));
 
@@ -243,7 +248,8 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk,
                               int use_second_hsdir_index,
                               int for_fetching, smartlist_t *responsible_dirs);
 routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs,
-                              const char *req_key_str);
+                              const char *req_key_str,
+                              bool *is_rate_limited_out);
 
 time_t hs_hsdir_requery_period(const or_options_t *options);
 time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
@@ -262,6 +268,8 @@ extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
                           const struct curve25519_public_key_t *onion_key,
                           int direct_conn);
 
+link_specifier_t *link_specifier_dup(const link_specifier_t *src);
+
 #ifdef HS_COMMON_PRIVATE
 
 STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out);
diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c
index ee4499ef5b..0dad8dd6d8 100644
--- a/src/feature/hs/hs_config.c
+++ b/src/feature/hs/hs_config.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,18 +23,72 @@
  * every option that is common to all version (config_generic_service).
  **/
 
-#define HS_CONFIG_PRIVATE
-
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_client.h"
+#include "feature/hs/hs_ob.h"
 #include "feature/hs/hs_service.h"
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
 #include "lib/encoding/confline.h"
+#include "lib/conf/confdecl.h"
+#include "lib/confmgt/confmgt.h"
+
+#include "feature/hs/hs_opts_st.h"
 #include "app/config/or_options_st.h"
 
-/* Using the given list of services, stage them into our global state. Every
+/* Declare the table mapping hs options to hs_opts_t */
+#define CONF_CONTEXT TABLE
+#include "feature/hs/hs_options.inc"
+#undef CONF_CONTEXT
+
+/** Magic number for hs_opts_t. */
+#define HS_OPTS_MAGIC 0x6f6e796e
+
+static const config_format_t hs_opts_fmt = {
+  .size = sizeof(hs_opts_t),
+  .magic = { "hs_opts_t",
+             HS_OPTS_MAGIC,
+             offsetof(hs_opts_t, magic) },
+  .vars = hs_opts_t_vars,
+};
+
+/** Global configuration manager to handle HS sections*/
+static config_mgr_t *hs_opts_mgr = NULL;
+
+/**
+ * Return a configuration manager for the hs_opts_t configuration type.
+ **/
+static const config_mgr_t *
+get_hs_opts_mgr(void)
+{
+  if (PREDICT_UNLIKELY(hs_opts_mgr == NULL)) {
+    hs_opts_mgr = config_mgr_new(&hs_opts_fmt);
+    config_mgr_freeze(hs_opts_mgr);
+  }
+  return hs_opts_mgr;
+}
+
+/**
+ * Allocate, initialize, and return a new hs_opts_t.
+ **/
+static hs_opts_t *
+hs_opts_new(void)
+{
+  const config_mgr_t *mgr = get_hs_opts_mgr();
+  hs_opts_t *r = config_new(mgr);
+  tor_assert(r);
+  config_init(mgr, r);
+  return r;
+}
+
+/**
+ * Free an hs_opts_t.
+ **/
+#define hs_opts_free(opts) \
+  config_free(get_hs_opts_mgr(), (opts))
+
+/** Using the given list of services, stage them into our global state. Every
  * service version are handled. This function can remove entries in the given
  * service_list.
  *
@@ -70,7 +124,7 @@ stage_services(smartlist_t *service_list)
   hs_service_stage_services(service_list);
 }
 
-/* Validate the given service against all service in the given list. If the
+/** Validate the given service against all service in the given list. If the
  * service is ephemeral, this function ignores it. Services with the same
  * directory path aren't allowed and will return an error. If a duplicate is
  * found, 1 is returned else 0 if none found. */
@@ -118,33 +172,27 @@ service_is_duplicate_in_list(const smartlist_t *service_list,
   return ret;
 }
 
-/* Helper function: Given an configuration option name, its value, a minimum
- * min and a maxium max, parse the value as a uint64_t. On success, ok is set
- * to 1 and ret is the parsed value. On error, ok is set to 0 and ret must be
- * ignored. This function logs both on error and success. */
-static uint64_t
-helper_parse_uint64(const char *opt, const char *value, uint64_t min,
-                    uint64_t max, int *ok)
+/** Check whether an integer <b>i</b> is out of bounds (not between <b>low</b>
+ * and <b>high</b> incusive).  If it is, then log a warning about the option
+ * <b>name</b>, and return true. Otherwise return false. */
+static bool
+check_value_oob(int i, const char *name, int low, int high)
 {
-  uint64_t ret = 0;
-
-  tor_assert(opt);
-  tor_assert(value);
-  tor_assert(ok);
-
-  *ok = 0;
-  ret = tor_parse_uint64(value, 10, min, max, ok, NULL);
-  if (!*ok) {
-    log_warn(LD_CONFIG, "%s must be between %" PRIu64 " and %"PRIu64
-                        ", not %s.",
-             opt, min, max, value);
-    goto err;
+  if (i < low || i > high) {
+    log_warn(LD_CONFIG, "%s must be between %d and %d, not %d.",
+             name, low, high, i);
+    return true;
   }
-  log_info(LD_CONFIG, "%s was parsed to %" PRIu64, opt, ret);
- err:
-  return ret;
+  return false;
 }
 
+/**
+ * Helper: check whether the integer value called <b>name</b> in <b>opts</b>
+ * is out-of-bounds.
+ **/
+#define CHECK_OOB(opts, name, low, high)      \
+  check_value_oob((opts)->name, #name, (low), (high))
+
 /** Helper function: Given a configuration option and its value, parse the
  * value as a hs_circuit_id_protocol_t. On success, ok is set to 1 and ret is
  * the parse value. On error, ok is set to 0 and the "none"
@@ -173,7 +221,7 @@ helper_parse_circuit_id_protocol(const char *key, const char *value, int *ok)
   return ret;
 }
 
-/* Return the service version by trying to learn it from the key on disk if
+/** Return the service version by trying to learn it from the key on disk if
  * any. If nothing is found, the current service configured version is
  * returned. */
 static int
@@ -191,7 +239,13 @@ config_learn_service_version(hs_service_t *service)
   return version;
 }
 
-/* Return true iff the given options starting at line_ for a hidden service
+/**
+ * Header key indicating the start of a new hidden service configuration
+ * block.
+ **/
+static const char SECTION_HEADER[] = "HiddenServiceDir";
+
+/** Return true iff the given options starting at line_ for a hidden service
  * contains at least one invalid option. Each hidden service option don't
  * apply to all versions so this function can find out. The line_ MUST start
  * right after the HiddenServiceDir line of this service.
@@ -218,6 +272,10 @@ config_has_invalid_options(const config_line_t *line_,
 
   const char *opts_exclude_v2[] = {
     "HiddenServiceExportCircuitID",
+    "HiddenServiceEnableIntroDoSDefense",
+    "HiddenServiceEnableIntroDoSRatePerSec",
+    "HiddenServiceEnableIntroDoSBurstPerSec",
+    "HiddenServiceOnionBalanceInstance",
     NULL /* End marker. */
   };
 
@@ -241,8 +299,11 @@ config_has_invalid_options(const config_line_t *line_,
   for (int i = 0; optlist[i]; i++) {
     const char *opt = optlist[i];
     for (line = line_; line; line = line->next) {
-      if (!strcasecmp(line->key, "HiddenServiceDir")) {
-        /* We just hit the next hidden service, stop right now. */
+      if (!strcasecmp(line->key, SECTION_HEADER)) {
+        /* We just hit the next hidden service, stop right now.
+         * (This shouldn't be possible, now that we have partitioned the list
+         * into sections.) */
+        tor_assert_nonfatal_unreached();
         goto end;
       }
       if (!strcasecmp(line->key, opt)) {
@@ -250,6 +311,16 @@ config_has_invalid_options(const config_line_t *line_,
                             "version %" PRIu32 " of service in %s",
                  opt, service->config.version,
                  service->config.directory_path);
+
+        if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
+          /* Special case this v2 option so that we can offer alternatives.
+           * If more such special cases appear, it would be good to
+           * generalize the exception mechanism here. */
+          log_warn(LD_CONFIG, "For v3 onion service client authorization, "
+                   "please read the 'CLIENT AUTHORIZATION' section in the "
+                   "manual.");
+        }
+
         ret = 1;
         /* Continue the loop so we can find all possible options. */
         continue;
@@ -260,7 +331,7 @@ config_has_invalid_options(const config_line_t *line_,
   return ret;
 }
 
-/* Validate service configuration. This is used when loading the configuration
+/** Validate service configuration. This is used when loading the configuration
  * and once we've setup a service object, it's config object is passed to this
  * function for further validation. This does not validate service key
  * material. Return 0 if valid else -1 if invalid. */
@@ -276,63 +347,83 @@ config_validate_service(const hs_service_config_t *config)
     goto invalid;
   }
 
+  /* DoS validation values. */
+  if (config->has_dos_defense_enabled &&
+      (config->intro_dos_burst_per_sec < config->intro_dos_rate_per_sec)) {
+    log_warn(LD_CONFIG, "Hidden service DoS defenses burst (%" PRIu32 ") can "
+                        "not be smaller than the rate value (%" PRIu32 ").",
+             config->intro_dos_burst_per_sec, config->intro_dos_rate_per_sec);
+    goto invalid;
+  }
+
   /* Valid. */
   return 0;
  invalid:
   return -1;
 }
 
-/* Configuration funcion for a version 3 service. The line_ must be pointing
- * to the directive directly after a HiddenServiceDir. That way, when hitting
- * the next HiddenServiceDir line or reaching the end of the list of lines, we
- * know that we have to stop looking for more options. The given service
+/** Configuration funcion for a version 3 service. The given service
  * object must be already allocated and passed through
  * config_generic_service() prior to calling this function.
  *
  * Return 0 on success else a negative value. */
 static int
-config_service_v3(const config_line_t *line_,
+config_service_v3(const hs_opts_t *hs_opts,
                   hs_service_config_t *config)
 {
-  int have_num_ip = 0;
-  bool export_circuit_id = false; /* just to detect duplicate options */
-  const char *dup_opt_seen = NULL;
-  const config_line_t *line;
-
   tor_assert(config);
+  tor_assert(hs_opts);
 
-  for (line = line_; line; line = line->next) {
-    int ok = 0;
-    if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* We just hit the next hidden service, stop right now. */
-      break;
-    }
-    /* Number of introduction points. */
-    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
-      config->num_intro_points =
-        (unsigned int) helper_parse_uint64(line->key, line->value,
-                                           NUM_INTRO_POINTS_DEFAULT,
-                                           HS_CONFIG_V3_MAX_INTRO_POINTS,
-                                           &ok);
-      if (!ok || have_num_ip) {
-        if (have_num_ip)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_num_ip = 1;
-      continue;
+  /* Number of introduction points. */
+  if (CHECK_OOB(hs_opts, HiddenServiceNumIntroductionPoints,
+                NUM_INTRO_POINTS_DEFAULT,
+                HS_CONFIG_V3_MAX_INTRO_POINTS)) {
+    goto err;
+  }
+  config->num_intro_points = hs_opts->HiddenServiceNumIntroductionPoints;
+
+  /* Circuit ID export setting. */
+  if (hs_opts->HiddenServiceExportCircuitID) {
+    int ok;
+    config->circuit_id_protocol =
+      helper_parse_circuit_id_protocol("HiddenServcieExportCircuitID",
+                                       hs_opts->HiddenServiceExportCircuitID,
+                                       &ok);
+    if (!ok) {
+      goto err;
     }
-    if (!strcasecmp(line->key, "HiddenServiceExportCircuitID")) {
-      config->circuit_id_protocol =
-        helper_parse_circuit_id_protocol(line->key, line->value, &ok);
-      if (!ok || export_circuit_id) {
-        if (export_circuit_id) {
-          dup_opt_seen = line->key;
-        }
-        goto err;
-      }
-      export_circuit_id = true;
-      continue;
+  }
+
+  /* Is the DoS defense enabled? */
+  config->has_dos_defense_enabled =
+    hs_opts->HiddenServiceEnableIntroDoSDefense;
+
+  /* Rate for DoS defense */
+  if (CHECK_OOB(hs_opts, HiddenServiceEnableIntroDoSRatePerSec,
+                 HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN,
+                 HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX)) {
+    goto err;
+  }
+  config->intro_dos_rate_per_sec =
+    hs_opts->HiddenServiceEnableIntroDoSRatePerSec;
+  log_info(LD_REND, "Service INTRO2 DoS defenses rate set to: %" PRIu32,
+           config->intro_dos_rate_per_sec);
+
+  if (CHECK_OOB(hs_opts, HiddenServiceEnableIntroDoSBurstPerSec,
+                HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN,
+                HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX)) {
+    goto err;
+  }
+  config->intro_dos_burst_per_sec =
+    hs_opts->HiddenServiceEnableIntroDoSBurstPerSec;
+  log_info(LD_REND, "Service INTRO2 DoS defenses burst set to: %" PRIu32,
+           config->intro_dos_burst_per_sec);
+
+  /* Is this an onionbalance instance? */
+  if (hs_opts->HiddenServiceOnionBalanceInstance) {
+    /* Option is enabled, parse config file. */
+    if (! hs_ob_parse_config_file(config)) {
+      goto err;
     }
   }
 
@@ -347,13 +438,10 @@ config_service_v3(const config_line_t *line_,
 
   return 0;
  err:
-  if (dup_opt_seen) {
-    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
-  }
   return -1;
 }
 
-/* Configure a service using the given options in line_ and options. This is
+/** Configure a service using the given options in hs_opts and options. This is
  * called for any service regardless of its version which means that all
  * directives in this function are generic to any service version. This
  * function will also check the validity of the service directory path.
@@ -365,168 +453,98 @@ config_service_v3(const config_line_t *line_,
  *
  * Return 0 on success else -1. */
 static int
-config_generic_service(const config_line_t *line_,
+config_generic_service(const hs_opts_t *hs_opts,
                        const or_options_t *options,
                        hs_service_t *service)
 {
-  int dir_seen = 0;
-  const config_line_t *line;
   hs_service_config_t *config;
-  /* If this is set, we've seen a duplicate of this option. Keep the string
-   * so we can log the directive. */
-  const char *dup_opt_seen = NULL;
-  /* These variables will tell us if we ever have duplicate. */
-  int have_version = 0, have_allow_unknown_ports = 0;
-  int have_dir_group_read = 0, have_max_streams = 0;
-  int have_max_streams_close = 0;
-
-  tor_assert(line_);
+
+  tor_assert(hs_opts);
   tor_assert(options);
   tor_assert(service);
 
   /* Makes thing easier. */
   config = &service->config;
 
-  /* The first line starts with HiddenServiceDir so we consider what's next is
-   * the configuration of the service. */
-  for (line = line_; line ; line = line->next) {
-    int ok = 0;
-
-    /* This indicate that we have a new service to configure. */
-    if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* This function only configures one service at a time so if we've
-       * already seen one, stop right now. */
-      if (dir_seen) {
-        break;
+  /* Directory where the service's keys are stored. */
+  tor_assert(hs_opts->HiddenServiceDir);
+  config->directory_path = tor_strdup(hs_opts->HiddenServiceDir);
+  log_info(LD_CONFIG, "%s=%s. Configuring...",
+           SECTION_HEADER, escaped(config->directory_path));
+
+  /* Protocol version for the service. */
+  if (hs_opts->HiddenServiceVersion == -1) {
+    /* No value was set; stay with the default. */
+  } else if (CHECK_OOB(hs_opts, HiddenServiceVersion,
+                       HS_VERSION_MIN, HS_VERSION_MAX)) {
+    goto err;
+  } else {
+    config->hs_version_explicitly_set = 1;
+    config->version = hs_opts->HiddenServiceVersion;
+  }
+
+  /* Virtual port. */
+  for (const config_line_t *portline = hs_opts->HiddenServicePort;
+       portline; portline = portline->next) {
+    char *err_msg = NULL;
+    /* XXX: Can we rename this? */
+    rend_service_port_config_t *portcfg =
+      rend_service_parse_port_config(portline->value, " ", &err_msg);
+    if (!portcfg) {
+      if (err_msg) {
+        log_warn(LD_CONFIG, "%s", err_msg);
       }
-      /* Ok, we've seen one and we are about to configure it. */
-      dir_seen = 1;
-      config->directory_path = tor_strdup(line->value);
-      log_info(LD_CONFIG, "HiddenServiceDir=%s. Configuring...",
-               escaped(config->directory_path));
-      continue;
-    }
-    if (BUG(!dir_seen)) {
+      tor_free(err_msg);
       goto err;
     }
-    /* Version of the service. */
-    if (!strcasecmp(line->key, "HiddenServiceVersion")) {
-      service->config.version =
-        (uint32_t) helper_parse_uint64(line->key, line->value, HS_VERSION_MIN,
-                                       HS_VERSION_MAX, &ok);
-      if (!ok || have_version) {
-        if (have_version)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_version = service->config.hs_version_explicitly_set = 1;
-      continue;
-    }
-    /* Virtual port. */
-    if (!strcasecmp(line->key, "HiddenServicePort")) {
-      char *err_msg = NULL;
-      /* XXX: Can we rename this? */
-      rend_service_port_config_t *portcfg =
-        rend_service_parse_port_config(line->value, " ", &err_msg);
-      if (!portcfg) {
-        if (err_msg) {
-          log_warn(LD_CONFIG, "%s", err_msg);
-        }
-        tor_free(err_msg);
-        goto err;
-      }
-      tor_assert(!err_msg);
-      smartlist_add(config->ports, portcfg);
-      log_info(LD_CONFIG, "HiddenServicePort=%s for %s",
-               line->value, escaped(config->directory_path));
-      continue;
-    }
-    /* Do we allow unknown ports. */
-    if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) {
-      config->allow_unknown_ports =
-        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
-      if (!ok || have_allow_unknown_ports) {
-        if (have_allow_unknown_ports)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_allow_unknown_ports = 1;
-      continue;
-    }
-    /* Directory group readable. */
-    if (!strcasecmp(line->key, "HiddenServiceDirGroupReadable")) {
-      config->dir_group_readable =
-        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
-      if (!ok || have_dir_group_read) {
-        if (have_dir_group_read)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_dir_group_read = 1;
-      continue;
-    }
-    /* Maximum streams per circuit. */
-    if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
-      config->max_streams_per_rdv_circuit =
-        helper_parse_uint64(line->key, line->value, 0,
-                            HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT, &ok);
-      if (!ok || have_max_streams) {
-        if (have_max_streams)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_max_streams = 1;
-      continue;
-    }
-    /* Maximum amount of streams before we close the circuit. */
-    if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
-      config->max_streams_close_circuit =
-        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
-      if (!ok || have_max_streams_close) {
-        if (have_max_streams_close)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_max_streams_close = 1;
-      continue;
-    }
+    tor_assert(!err_msg);
+    smartlist_add(config->ports, portcfg);
+    log_info(LD_CONFIG, "HiddenServicePort=%s for %s",
+             portline->value, escaped(config->directory_path));
+  }
+
+  /* Do we allow unknown ports? */
+  config->allow_unknown_ports = hs_opts->HiddenServiceAllowUnknownPorts;
+
+  /* Directory group readable. */
+  config->dir_group_readable = hs_opts->HiddenServiceDirGroupReadable;
+
+  /* Maximum streams per circuit. */
+  if (CHECK_OOB(hs_opts, HiddenServiceMaxStreams,
+                0, HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT)) {
+    goto err;
   }
+  config->max_streams_per_rdv_circuit = hs_opts->HiddenServiceMaxStreams;
+
+  /* Maximum amount of streams before we close the circuit. */
+  config->max_streams_close_circuit =
+    hs_opts->HiddenServiceMaxStreamsCloseCircuit;
 
   /* Check if we are configured in non anonymous mode meaning every service
    * becomes a single onion service. */
   if (rend_service_non_anonymous_mode_enabled(options)) {
     config->is_single_onion = 1;
-    /* We will add support for IPv6-only v3 single onion services in a future
-     * Tor version. This won't catch "ReachableAddresses reject *4", but that
-     * option doesn't work anyway. */
-    if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) {
-      log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not "
-               "supported. Set HiddenServiceSingleHopMode 0 and "
-               "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1.");
-      goto err;
-    }
   }
 
   /* Success */
   return 0;
  err:
-  if (dup_opt_seen) {
-    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
-  }
   return -1;
 }
 
-/* Configure a service using the given line and options. This function will
+/** Configure a service using the given line and options. This function will
  * call the corresponding configuration function for a specific service
  * version and validate the service against the other ones. On success, add
  * the service to the given list and return 0. On error, nothing is added to
  * the list and a negative value is returned. */
 static int
-config_service(const config_line_t *line, const or_options_t *options,
+config_service(config_line_t *line, const or_options_t *options,
                smartlist_t *service_list)
 {
   int ret;
   hs_service_t *service = NULL;
+  hs_opts_t *hs_opts = NULL;
+  char *msg = NULL;
 
   tor_assert(line);
   tor_assert(options);
@@ -535,9 +553,25 @@ config_service(const config_line_t *line, const or_options_t *options,
   /* We have a new hidden service. */
   service = hs_service_new(options);
 
+  /* Try to validate and parse the configuration lines into 'hs_opts' */
+  hs_opts = hs_opts_new();
+  ret = config_assign(get_hs_opts_mgr(), hs_opts, line, 0, &msg);
+  if (ret < 0) {
+    log_warn(LD_REND, "Can't parse configuration for onion service: %s", msg);
+    goto err;
+  }
+  tor_assert_nonfatal(msg == NULL);
+  validation_status_t vs = config_validate(get_hs_opts_mgr(), NULL,
+                                           hs_opts, &msg);
+  if (vs < 0) {
+    log_warn(LD_REND, "Bad configuration for onion service: %s", msg);
+    goto err;
+  }
+  tor_assert_nonfatal(msg == NULL);
+
   /* We'll configure that service as a generic one and then pass it to a
    * specific function according to the configured version number. */
-  if (config_generic_service(line, options, service) < 0) {
+  if (config_generic_service(hs_opts, options, service) < 0) {
     goto err;
   }
 
@@ -572,10 +606,10 @@ config_service(const config_line_t *line, const or_options_t *options,
    * directory line, the function knows that it has to stop parsing. */
   switch (service->config.version) {
   case HS_VERSION_TWO:
-    ret = rend_config_service(line->next, options, &service->config);
+    ret = rend_config_service(hs_opts, options, &service->config);
     break;
   case HS_VERSION_THREE:
-    ret = config_service_v3(line->next, &service->config);
+    ret = config_service_v3(hs_opts, &service->config);
     break;
   default:
     /* We do validate before if we support the parsed version. */
@@ -594,22 +628,25 @@ config_service(const config_line_t *line, const or_options_t *options,
 
   /* Passes, add it to the given list. */
   smartlist_add(service_list, service);
+  hs_opts_free(hs_opts);
 
   return 0;
 
  err:
   hs_service_free(service);
+  hs_opts_free(hs_opts);
+  tor_free(msg);
   return -1;
 }
 
-/* From a set of <b>options</b>, setup every hidden service found. Return 0 on
+/** From a set of <b>options</b>, setup every hidden service found. Return 0 on
  * success or -1 on failure. If <b>validate_only</b> is set, parse, warn and
  * return as normal, but don't actually change the configured services. */
 int
 hs_config_service_all(const or_options_t *options, int validate_only)
 {
-  int dir_option_seen = 0, ret = -1;
-  const config_line_t *line;
+  int ret = -1;
+  config_line_t *remaining = NULL;
   smartlist_t *new_service_list = NULL;
 
   tor_assert(options);
@@ -618,23 +655,24 @@ hs_config_service_all(const or_options_t *options, int validate_only)
    * validation and staging for >= v3. */
   new_service_list = smartlist_new();
 
-  for (line = options->RendConfigLines; line; line = line->next) {
-    /* Ignore all directives that aren't the start of a service. */
-    if (strcasecmp(line->key, "HiddenServiceDir")) {
-      if (!dir_option_seen) {
-        log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
-                 line->key);
-        goto err;
-      }
-      continue;
-    }
-    /* Flag that we've seen a directory directive and we'll use it to make
-     * sure that the torrc options ordering is actually valid. */
-    dir_option_seen = 1;
+  /* We need to start with a HiddenServiceDir line */
+  if (options->RendConfigLines &&
+      strcasecmp(options->RendConfigLines->key, SECTION_HEADER)) {
+    log_warn(LD_CONFIG, "%s with no preceding %s directive",
+             options->RendConfigLines->key, SECTION_HEADER);
+    goto err;
+  }
+
+  remaining = config_lines_dup(options->RendConfigLines);
+  while (remaining) {
+    config_line_t *section = remaining;
+    remaining = config_lines_partition(section, SECTION_HEADER);
 
     /* Try to configure this service now. On success, it will be added to the
      * list and validated against the service in that same list. */
-    if (config_service(line, options, new_service_list) < 0) {
+    int rv = config_service(section, options, new_service_list);
+    config_free_lines(section);
+    if (rv < 0) {
       goto err;
     }
   }
@@ -670,7 +708,7 @@ hs_config_service_all(const or_options_t *options, int validate_only)
   return ret;
 }
 
-/* From a set of <b>options</b>, setup every client authorization found.
+/** From a set of <b>options</b>, setup every client authorization found.
  * Return 0 on success or -1 on failure. If <b>validate_only</b> is set,
  * parse, warn and return as normal, but don't actually change the
  * configured state. */
@@ -694,3 +732,12 @@ hs_config_client_auth_all(const or_options_t *options, int validate_only)
  done:
   return ret;
 }
+
+/**
+ * Free all resources held by the hs_config.c module.
+ **/
+void
+hs_config_free_all(void)
+{
+  config_mgr_free(hs_opts_mgr);
+}
diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h
index 040e451f13..c60b4fbb5d 100644
--- a/src/feature/hs/hs_config.h
+++ b/src/feature/hs/hs_config.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,11 +15,21 @@
 #define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535
 /* Maximum number of intro points per version 3 services. */
 #define HS_CONFIG_V3_MAX_INTRO_POINTS 20
+/* Default value for the introduction DoS defenses. The MIN/MAX are inclusive
+ * meaning they can be used as valid values. */
+#define HS_CONFIG_V3_DOS_DEFENSE_DEFAULT 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT 25
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX INT32_MAX
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT 200
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX INT32_MAX
 
 /* API */
 
 int hs_config_service_all(const or_options_t *options, int validate_only);
 int hs_config_client_auth_all(const or_options_t *options, int validate_only);
 
-#endif /* !defined(TOR_HS_CONFIG_H) */
+void hs_config_free_all(void);
 
+#endif /* !defined(TOR_HS_CONFIG_H) */
diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c
index 9970fdd123..78b0735c29 100644
--- a/src/feature/hs/hs_control.c
+++ b/src/feature/hs/hs_control.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,9 +7,10 @@
  **/
 
 #include "core/or/or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_control.h"
 #include "feature/hs/hs_descriptor.h"
@@ -19,7 +20,7 @@
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
-/* Send on the control port the "HS_DESC REQUESTED [...]" event.
+/** Send on the control port the "HS_DESC REQUESTED [...]" event.
  *
  * The onion_pk is the onion service public key, base64_blinded_pk is the
  * base64 encoded blinded key for the service and hsdir_rs is the routerstatus
@@ -56,7 +57,7 @@ hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
   memwipe(onion_address, 0, sizeof(onion_address));
 }
 
-/* Send on the control port the "HS_DESC FAILED [...]" event.
+/** Send on the control port the "HS_DESC FAILED [...]" event.
  *
  * Using a directory connection identifier, the HSDir identity digest and a
  * reason for the failure. None can be NULL. */
@@ -73,17 +74,14 @@ hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
   tor_assert(reason);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk,
                                        hsdir_id_digest, reason);
 }
 
-/* Send on the control port the "HS_DESC RECEIVED [...]" event.
+/** Send on the control port the "HS_DESC RECEIVED [...]" event.
  *
  * Using a directory connection identifier and the HSDir identity digest.
  * None can be NULL. */
@@ -98,17 +96,14 @@ hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk,
                                          hsdir_id_digest);
 }
 
-/* Send on the control port the "HS_DESC CREATED [...]" event.
+/** Send on the control port the "HS_DESC CREATED [...]" event.
  *
  * Using the onion address of the descriptor's service and the blinded public
  * key of the descriptor as a descriptor ID. None can be NULL. */
@@ -122,16 +117,14 @@ hs_control_desc_event_created(const char *onion_address,
   tor_assert(blinded_pk);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   /* Version 3 doesn't use the replica number in its descriptor ID computation
    * so we pass negative value so the control port subsystem can ignore it. */
   control_event_hs_descriptor_created(onion_address, base64_blinded_pk, -1);
 }
 
-/* Send on the control port the "HS_DESC UPLOAD [...]" event.
+/** Send on the control port the "HS_DESC UPLOAD [...]" event.
  *
  * Using the onion address of the descriptor's service, the HSDir identity
  * digest, the blinded public key of the descriptor as a descriptor ID and the
@@ -150,9 +143,7 @@ hs_control_desc_event_upload(const char *onion_address,
   tor_assert(hsdir_index);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   control_event_hs_descriptor_upload(onion_address, hsdir_id_digest,
                                      base64_blinded_pk,
@@ -160,7 +151,7 @@ hs_control_desc_event_upload(const char *onion_address,
                                              DIGEST256_LEN));
 }
 
-/* Send on the control port the "HS_DESC UPLOADED [...]" event.
+/** Send on the control port the "HS_DESC UPLOADED [...]" event.
  *
  * Using the directory connection identifier and the HSDir identity digest.
  * None can be NULL. */
@@ -178,7 +169,7 @@ hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
   control_event_hs_descriptor_uploaded(hsdir_id_digest, onion_address);
 }
 
-/* Send on the control port the "HS_DESC_CONTENT [...]" event.
+/** Send on the control port the "HS_DESC_CONTENT [...]" event.
  *
  * Using the directory connection identifier, the HSDir identity digest and
  * the body of the descriptor (as it was received from the directory). None
@@ -195,17 +186,14 @@ hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hs_descriptor_content(onion_address, base64_blinded_pk,
                                       hsdir_id_digest, body);
 }
 
-/* Handle the "HSPOST [...]" command. The body is an encoded descriptor for
+/** Handle the "HSPOST [...]" command. The body is an encoded descriptor for
  * the given onion_address. The descriptor will be uploaded to each directory
  * in hsdirs_rs. If NULL, the responsible directories for the current time
  * period will be selected.
@@ -259,3 +247,16 @@ hs_control_hspost_command(const char *body, const char *onion_address,
   smartlist_free(hsdirs);
   return ret;
 }
+
+/** With a given <b>onion_identity_pk</b>, fetch its descriptor, optionally
+ * using the list of directory servers given in <b>hsdirs</b>, or a random
+ * server if it is NULL. This function calls hs_client_launch_v3_desc_fetch().
+ */
+void
+hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                           const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  hs_client_launch_v3_desc_fetch(onion_identity_pk, hsdirs);
+}
diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h
index f7ab642652..947b0ebf1c 100644
--- a/src/feature/hs/hs_control.h
+++ b/src/feature/hs/hs_control.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,5 +48,9 @@ void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
 int hs_control_hspost_command(const char *body, const char *onion_address,
                               const smartlist_t *hsdirs_rs);
 
+/* Command "HSFETCH [...]" */
+void hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                                const smartlist_t *hsdirs);
+
 #endif /* !defined(TOR_HS_CONTROL_H) */
 
diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c
index f74bb97ee2..c1e6553398 100644
--- a/src/feature/hs/hs_descriptor.c
+++ b/src/feature/hs/hs_descriptor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -56,6 +56,7 @@
 #define HS_DESCRIPTOR_PRIVATE
 
 #include "core/or/or.h"
+#include "app/config/config.h"
 #include "trunnel/ed25519_cert.h" /* Trunnel interface. */
 #include "feature/hs/hs_descriptor.h"
 #include "core/or/circuitbuild.h"
@@ -102,7 +103,7 @@
 #define str_desc_auth_client "auth-client"
 #define str_encrypted "encrypted"
 
-/* Authentication supported types. */
+/** Authentication supported types. */
 static const struct {
   hs_desc_auth_type_t type;
   const char *identifier;
@@ -112,7 +113,7 @@ static const struct {
   { 0, NULL }
 };
 
-/* Descriptor ruleset. */
+/** Descriptor ruleset. */
 static token_rule_t hs_desc_v3_token_table[] = {
   T1_START(str_hs_desc, R_HS_DESCRIPTOR, EQ(1), NO_OBJ),
   T1(str_lifetime, R3_DESC_LIFETIME, EQ(1), NO_OBJ),
@@ -123,7 +124,7 @@ static token_rule_t hs_desc_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Descriptor ruleset for the superencrypted section. */
+/** Descriptor ruleset for the superencrypted section. */
 static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
   T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ),
   T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ),
@@ -132,7 +133,7 @@ static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Descriptor ruleset for the encrypted section. */
+/** Descriptor ruleset for the encrypted section. */
 static token_rule_t hs_desc_encrypted_v3_token_table[] = {
   T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ),
   T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, GE(1), NO_OBJ),
@@ -140,7 +141,7 @@ static token_rule_t hs_desc_encrypted_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Descriptor ruleset for the introduction points section. */
+/** Descriptor ruleset for the introduction points section. */
 static token_rule_t hs_desc_intro_point_v3_token_table[] = {
   T1_START(str_intro_point, R3_INTRODUCTION_POINT, EQ(1), NO_OBJ),
   T1N(str_ip_onion_key, R3_INTRO_ONION_KEY, GE(2), OBJ_OK),
@@ -152,7 +153,7 @@ static token_rule_t hs_desc_intro_point_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
+/** Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
  * We use SHA3-256 for the MAC computation.
  * This function can't fail. */
 static void
@@ -184,7 +185,7 @@ build_mac(const uint8_t *mac_key, size_t mac_key_len,
   crypto_digest_free(digest);
 }
 
-/* Using a secret data and a given decriptor object, build the secret
+/** Using a secret data and a given decriptor object, build the secret
  * input needed for the KDF.
  *
  * secret_input = SECRET_DATA | subcredential | INT_8(revision_counter)
@@ -211,7 +212,7 @@ build_secret_input(const hs_descriptor_t *desc,
   memcpy(secret_input, secret_data, secret_data_len);
   offset += secret_data_len;
   /* Copy subcredential. */
-  memcpy(secret_input + offset, desc->subcredential, DIGEST256_LEN);
+  memcpy(secret_input + offset, desc->subcredential.subcred, DIGEST256_LEN);
   offset += DIGEST256_LEN;
   /* Copy revision counter value. */
   set_uint64(secret_input + offset,
@@ -224,7 +225,7 @@ build_secret_input(const hs_descriptor_t *desc,
   return secret_input_len;
 }
 
-/* Do the KDF construction and put the resulting data in key_out which is of
+/** Do the KDF construction and put the resulting data in key_out which is of
  * key_out_len length. It uses SHAKE-256 as specified in the spec. */
 static void
 build_kdf_key(const hs_descriptor_t *desc,
@@ -269,7 +270,7 @@ build_kdf_key(const hs_descriptor_t *desc,
   tor_free(secret_input);
 }
 
-/* Using the given descriptor, secret data, and salt, run it through our
+/** Using the given descriptor, secret data, and salt, run it through our
  * KDF function and then extract a secret key in key_out, the IV in iv_out
  * and MAC in mac_out. This function can't fail. */
 static void
@@ -308,7 +309,7 @@ build_secret_key_iv_mac(const hs_descriptor_t *desc,
 
 /* === ENCODING === */
 
-/* Encode the given link specifier objects into a newly allocated string.
+/** Encode the given link specifier objects into a newly allocated string.
  * This can't fail so caller can always assume a valid string being
  * returned. */
 STATIC char *
@@ -324,12 +325,11 @@ encode_link_specifiers(const smartlist_t *specs)
 
   link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
 
-  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+  SMARTLIST_FOREACH_BEGIN(specs, const link_specifier_t *,
                           spec) {
-    link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec);
-    if (ls) {
-      link_specifier_list_add_spec(lslist, ls);
-    }
+    link_specifier_t *ls = link_specifier_dup(spec);
+    tor_assert(ls);
+    link_specifier_list_add_spec(lslist, ls);
   } SMARTLIST_FOREACH_END(spec);
 
   {
@@ -356,7 +356,7 @@ encode_link_specifiers(const smartlist_t *specs)
   return encoded_b64;
 }
 
-/* Encode an introduction point legacy key and certificate. Return a newly
+/** Encode an introduction point legacy key and certificate. Return a newly
  * allocated string with it. On failure, return NULL. */
 static char *
 encode_legacy_key(const hs_desc_intro_point_t *ip)
@@ -393,7 +393,7 @@ encode_legacy_key(const hs_desc_intro_point_t *ip)
   return encoded;
 }
 
-/* Encode an introduction point encryption key and certificate. Return a newly
+/** Encode an introduction point encryption key and certificate. Return a newly
  * allocated string with it. On failure, return NULL. */
 static char *
 encode_enc_key(const hs_desc_intro_point_t *ip)
@@ -404,9 +404,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "enc-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->enc_key);
   if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
     goto done;
   }
@@ -421,8 +419,8 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
   return encoded;
 }
 
-/* Encode an introduction point onion key. Return a newly allocated string
- * with it. On failure, return NULL. */
+/** Encode an introduction point onion key. Return a newly allocated string
+ * with it. Can not fail. */
 static char *
 encode_onion_key(const hs_desc_intro_point_t *ip)
 {
@@ -432,16 +430,13 @@ encode_onion_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "onion-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->onion_key);
   tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64);
 
- done:
   return encoded;
 }
 
-/* Encode an introduction point object and return a newly allocated string
+/** Encode an introduction point object and return a newly allocated string
  * with it. On failure, return NULL. */
 static char *
 encode_intro_point(const ed25519_public_key_t *sig_key,
@@ -511,7 +506,7 @@ encode_intro_point(const ed25519_public_key_t *sig_key,
   return encoded_ip;
 }
 
-/* Given a source length, return the new size including padding for the
+/** Given a source length, return the new size including padding for the
  * plaintext encryption. */
 static size_t
 compute_padded_plaintext_length(size_t plaintext_len)
@@ -531,7 +526,7 @@ compute_padded_plaintext_length(size_t plaintext_len)
   return plaintext_padded_len;
 }
 
-/* Given a buffer, pad it up to the encrypted section padding requirement. Set
+/** Given a buffer, pad it up to the encrypted section padding requirement. Set
  * the newly allocated string in padded_out and return the length of the
  * padded buffer. */
 STATIC size_t
@@ -554,7 +549,7 @@ build_plaintext_padding(const char *plaintext, size_t plaintext_len,
   return padded_len;
 }
 
-/* Using a key, IV and plaintext data of length plaintext_len, create the
+/** Using a key, IV and plaintext data of length plaintext_len, create the
  * encrypted section by encrypting it and setting encrypted_out with the
  * data. Return size of the encrypted data buffer. */
 static size_t
@@ -599,7 +594,7 @@ build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext,
   return encrypted_len;
 }
 
-/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> and
+/** Encrypt the given <b>plaintext</b> buffer using <b>desc</b> and
  * <b>secret_data</b> to get the keys. Set encrypted_out with the encrypted
  * data and return the length of it. <b>is_superencrypted_layer</b> is set
  * if this is the outer encrypted layer of the descriptor. */
@@ -669,7 +664,7 @@ encrypt_descriptor_data(const hs_descriptor_t *desc,
   return final_blob_len;
 }
 
-/* Create and return a string containing a client-auth entry. It's the
+/** Create and return a string containing a client-auth entry. It's the
  * responsibility of the caller to free the returned string. This function
  * will never fail. */
 static char *
@@ -684,7 +679,7 @@ get_auth_client_str(const hs_desc_authorized_client_t *client)
   char encrypted_cookie_b64[HS_DESC_ENCRYPED_COOKIE_LEN * 2];
 
 #define ASSERT_AND_BASE64(field) STMT_BEGIN                        \
-  tor_assert(!tor_mem_is_zero((char *) client->field,              \
+  tor_assert(!fast_mem_is_zero((char *) client->field,              \
                               sizeof(client->field)));             \
   ret = base64_encode_nopad(field##_b64, sizeof(field##_b64),      \
                             client->field, sizeof(client->field)); \
@@ -739,7 +734,7 @@ get_all_auth_client_lines(const hs_descriptor_t *desc)
   return auth_client_lines_str;
 }
 
-/* Create the inner layer of the descriptor (which includes the intro points,
+/** Create the inner layer of the descriptor (which includes the intro points,
  * etc.). Return a newly-allocated string with the layer plaintext, or NULL if
  * an error occurred. It's the responsibility of the caller to free the
  * returned string. */
@@ -795,11 +790,11 @@ get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
   return encoded_str;
 }
 
-/* Create the middle layer of the descriptor, which includes the client auth
+/** Create the middle layer of the descriptor, which includes the client auth
  * data and the encrypted inner layer (provided as a base64 string at
  * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
- * layer plaintext, or NULL if an error occurred. It's the responsibility of
- * the caller to free the returned string. */
+ * layer plaintext. It's the responsibility of the caller to free the returned
+ * string. Can not fail. */
 static char *
 get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
                                     const char *layer2_b64_ciphertext)
@@ -815,13 +810,10 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
     const curve25519_public_key_t *ephemeral_pubkey;
 
     ephemeral_pubkey = &desc->superencrypted_data.auth_ephemeral_pubkey;
-    tor_assert(!tor_mem_is_zero((char *) ephemeral_pubkey->public_key,
+    tor_assert(!fast_mem_is_zero((char *) ephemeral_pubkey->public_key,
                                 CURVE25519_PUBKEY_LEN));
 
-    if (curve25519_public_to_base64(ephemeral_key_base64,
-                                    ephemeral_pubkey) < 0) {
-      goto done;
-    }
+    curve25519_public_to_base64(ephemeral_key_base64, ephemeral_pubkey);
     smartlist_add_asprintf(lines, "%s %s\n",
                            str_desc_auth_key, ephemeral_key_base64);
 
@@ -846,7 +838,6 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
 
   layer1_str = smartlist_join_strings(lines, "", 0, NULL);
 
- done:
   /* We need to memwipe all lines because it contains the ephemeral key */
   SMARTLIST_FOREACH(lines, char *, a, memwipe(a, 0, strlen(a)));
   SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
@@ -855,7 +846,7 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
   return layer1_str;
 }
 
-/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
+/** Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
  * returning it. <b>desc</b> is provided to derive the encryption
  * keys. <b>secret_data</b> is also proved to derive the encryption keys.
  * <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the
@@ -888,7 +879,7 @@ encrypt_desc_data_and_base64(const hs_descriptor_t *desc,
   return enc_b64;
 }
 
-/* Generate the secret data which is used to encrypt/decrypt the descriptor.
+/** Generate the secret data which is used to encrypt/decrypt the descriptor.
  *
  * SECRET_DATA = blinded-public-key
  * SECRET_DATA = blinded-public-key | descriptor_cookie
@@ -935,7 +926,7 @@ build_secret_data(const ed25519_public_key_t *blinded_pubkey,
   return secret_data_len;
 }
 
-/* Generate and encode the superencrypted portion of <b>desc</b>. This also
+/** Generate and encode the superencrypted portion of <b>desc</b>. This also
  * involves generating the encrypted portion of the descriptor, and performing
  * the superencryption. A newly allocated NUL-terminated string pointer
  * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0
@@ -1009,7 +1000,7 @@ encode_superencrypted_data(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
+/** Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
  * newly allocated string of the encoded descriptor. On error, -1 is returned
  * and encoded_out is untouched. */
 static int
@@ -1028,10 +1019,6 @@ desc_encode_v3(const hs_descriptor_t *desc,
   tor_assert(encoded_out);
   tor_assert(desc->plaintext_data.version == 3);
 
-  if (BUG(desc->subcredential == NULL)) {
-    goto err;
-  }
-
   /* Build the non-encrypted values. */
   {
     char *encoded_cert;
@@ -1092,11 +1079,7 @@ desc_encode_v3(const hs_descriptor_t *desc,
       tor_free(encoded_str);
       goto err;
     }
-    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
-      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
-      tor_free(encoded_str);
-      goto err;
-    }
+    ed25519_signature_to_base64(ed_sig_b64, &sig);
     /* Create the signature line. */
     smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
   }
@@ -1125,7 +1108,7 @@ desc_encode_v3(const hs_descriptor_t *desc,
 
 /* === DECODING === */
 
-/* Given the token tok for an auth client, decode it as
+/** Given the token tok for an auth client, decode it as
  * hs_desc_authorized_client_t. tok->args MUST contain at least 3 elements
  * Return 0 on success else -1 on failure. */
 static int
@@ -1161,7 +1144,7 @@ decode_auth_client(const directory_token_t *tok,
   return ret;
 }
 
-/* Given an encoded string of the link specifiers, return a newly allocated
+/** Given an encoded string of the link specifiers, return a newly allocated
  * list of decoded link specifiers. Return NULL on error. */
 STATIC smartlist_t *
 decode_link_specifiers(const char *encoded)
@@ -1190,52 +1173,22 @@ decode_link_specifiers(const char *encoded)
   results = smartlist_new();
 
   for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
-    hs_desc_link_specifier_t *hs_spec;
     link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
-    tor_assert(ls);
-
-    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
-    hs_spec->type = link_specifier_get_ls_type(ls);
-    switch (hs_spec->type) {
-    case LS_IPV4:
-      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
-                          link_specifier_get_un_ipv4_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
-      break;
-    case LS_IPV6:
-      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
-                               link_specifier_getarray_un_ipv6_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
-      break;
-    case LS_LEGACY_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
-                 sizeof(hs_spec->u.legacy_id));
-      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
-             sizeof(hs_spec->u.legacy_id));
-      break;
-    case LS_ED25519_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_ed25519_id(ls) ==
-                 sizeof(hs_spec->u.ed25519_id));
-      memcpy(hs_spec->u.ed25519_id,
-             link_specifier_getconstarray_un_ed25519_id(ls),
-             sizeof(hs_spec->u.ed25519_id));
-      break;
-    default:
-      tor_free(hs_spec);
+    if (BUG(!ls)) {
       goto err;
     }
-
-    smartlist_add(results, hs_spec);
+    link_specifier_t *ls_dup = link_specifier_dup(ls);
+    if (BUG(!ls_dup)) {
+      goto err;
+    }
+    smartlist_add(results, ls_dup);
   }
 
   goto done;
  err:
   if (results) {
-    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    SMARTLIST_FOREACH(results, link_specifier_t *, s,
+                      link_specifier_free(s));
     smartlist_free(results);
     results = NULL;
   }
@@ -1245,7 +1198,7 @@ decode_link_specifiers(const char *encoded)
   return results;
 }
 
-/* Given a list of authentication types, decode it and put it in the encrypted
+/** Given a list of authentication types, decode it and put it in the encrypted
  * data section. Return 1 if we at least know one of the type or 0 if we know
  * none of them. */
 static int
@@ -1273,7 +1226,7 @@ decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list)
   return match;
 }
 
-/* Parse a space-delimited list of integers representing CREATE2 formats into
+/** Parse a space-delimited list of integers representing CREATE2 formats into
  * the bitfield in hs_desc_encrypted_data_t. Ignore unrecognized values. */
 static void
 decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
@@ -1307,7 +1260,7 @@ decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
   smartlist_free(tokens);
 }
 
-/* Given a certificate, validate the certificate for certain conditions which
+/** Given a certificate, validate the certificate for certain conditions which
  * are if the given type matches the cert's one, if the signing key is
  * included and if the that key was actually used to sign the certificate.
  *
@@ -1331,11 +1284,20 @@ cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
     log_warn(LD_REND, "Signing key is NOT included for %s.", log_obj_type);
     goto err;
   }
+
   /* The following will not only check if the signature matches but also the
    * expiration date and overall validity. */
   if (tor_cert_checksig(cert, &cert->signing_key, approx_time()) < 0) {
-    log_warn(LD_REND, "Invalid signature for %s: %s", log_obj_type,
-             tor_cert_describe_signature_status(cert));
+    if (cert->cert_expired) {
+      char expiration_str[ISO_TIME_LEN+1];
+      format_iso_time(expiration_str, cert->valid_until);
+      log_fn(LOG_PROTOCOL_WARN, LD_REND, "Invalid signature for %s: %s (%s)",
+             log_obj_type, tor_cert_describe_signature_status(cert),
+             expiration_str);
+    } else {
+      log_warn(LD_REND, "Invalid signature for %s: %s",
+               log_obj_type, tor_cert_describe_signature_status(cert));
+    }
     goto err;
   }
 
@@ -1344,7 +1306,7 @@ cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
   return 0;
 }
 
-/* Given some binary data, try to parse it to get a certificate object. If we
+/** Given some binary data, try to parse it to get a certificate object. If we
  * have a valid cert, validate it using the given wanted type. On error, print
  * a log using the err_msg has the certificate identifier adding semantic to
  * the log and cert_out is set to NULL. On success, 0 is returned and cert_out
@@ -1381,7 +1343,7 @@ cert_parse_and_validate(tor_cert_t **cert_out, const char *data,
   return -1;
 }
 
-/* Return true iff the given length of the encrypted data of a descriptor
+/** Return true iff the given length of the encrypted data of a descriptor
  * passes validation. */
 STATIC int
 encrypted_data_length_is_valid(size_t len)
@@ -1400,7 +1362,50 @@ encrypted_data_length_is_valid(size_t len)
   return 0;
 }
 
-/* Decrypt the descriptor cookie given the descriptor, the auth client,
+/** Build the KEYS component for the authorized client computation. The format
+ * of the construction is:
+ *
+ *    SECRET_SEED = x25519(sk, pk)
+ *    KEYS = KDF(subcredential | SECRET_SEED, 40)
+ *
+ * Set the <b>keys_out</b> argument to point to the buffer containing the KEYS,
+ * and return the buffer's length. The caller should wipe and free its content
+ * once done with it. This function can't fail. */
+static size_t
+build_descriptor_cookie_keys(const hs_subcredential_t *subcredential,
+                             const curve25519_secret_key_t *sk,
+                             const curve25519_public_key_t *pk,
+                             uint8_t **keys_out)
+{
+  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
+  uint8_t *keystream;
+  size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN;
+  crypto_xof_t *xof;
+
+  tor_assert(subcredential);
+  tor_assert(sk);
+  tor_assert(pk);
+  tor_assert(keys_out);
+
+  keystream = tor_malloc_zero(keystream_len);
+
+  /* Calculate x25519(sk, pk) to get the secret seed. */
+  curve25519_handshake(secret_seed, sk, pk);
+
+  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, subcredential->subcred, SUBCRED_LEN);
+  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
+  crypto_xof_squeeze_bytes(xof, keystream, keystream_len);
+  crypto_xof_free(xof);
+
+  memwipe(secret_seed, 0, sizeof(secret_seed));
+
+  *keys_out = keystream;
+  return keystream_len;
+}
+
+/** Decrypt the descriptor cookie given the descriptor, the auth client,
  * and the client secret key. On sucess, return 0 and a newly allocated
  * descriptor cookie descriptor_cookie_out. On error or if the client id
  * is invalid, return -1 and descriptor_cookie_out is set to
@@ -1412,33 +1417,34 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
                           uint8_t **descriptor_cookie_out)
 {
   int ret = -1;
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key = NULL;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
   uint8_t *descriptor_cookie = NULL;
+  const uint8_t *cookie_key = NULL;
   crypto_cipher_t *cipher = NULL;
-  crypto_xof_t *xof = NULL;
 
   tor_assert(desc);
   tor_assert(client);
   tor_assert(client_auth_sk);
-  tor_assert(!tor_mem_is_zero(
+  tor_assert(!fast_mem_is_zero(
         (char *) &desc->superencrypted_data.auth_ephemeral_pubkey,
         sizeof(desc->superencrypted_data.auth_ephemeral_pubkey)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_sk,
-                              sizeof(*client_auth_sk)));
-  tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN));
+  tor_assert(!fast_mem_is_zero((char *) desc->subcredential.subcred,
+                               DIGEST256_LEN));
 
-  /* Calculate x25519(client_x, hs_Y) */
-  curve25519_handshake(secret_seed, client_auth_sk,
-                       &desc->superencrypted_data.auth_ephemeral_pubkey);
+  /* Catch potential code-flow cases of an unitialized private key sneaking
+   * into this function. */
+  if (BUG(fast_mem_is_zero((char *)client_auth_sk, sizeof(*client_auth_sk)))) {
+    goto done;
+  }
 
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(&desc->subcredential,
+                             client_auth_sk,
+                             &desc->superencrypted_data.auth_ephemeral_pubkey,
+                             &keystream);
+  tor_assert(keystream_length > 0);
 
   /* If the client id of auth client is not the same as the calculcated
    * client id, it means that this auth client is invaild according to the
@@ -1464,8 +1470,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
   if (cipher) {
     crypto_cipher_free(cipher);
   }
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
   return ret;
 }
 
@@ -1481,10 +1487,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
  */
 MOCK_IMPL(STATIC size_t,
 decrypt_desc_layer,(const hs_descriptor_t *desc,
-                    const uint8_t *encrypted_blob,
-                    size_t encrypted_blob_size,
                     const uint8_t *descriptor_cookie,
-                    int is_superencrypted_layer,
+                    bool is_superencrypted_layer,
                     char **decrypted_out))
 {
   uint8_t *decrypted = NULL;
@@ -1494,6 +1498,12 @@ decrypt_desc_layer,(const hs_descriptor_t *desc,
   uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
   const uint8_t *salt, *encrypted, *desc_mac;
   size_t encrypted_len, result_len = 0;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
 
   tor_assert(decrypted_out);
   tor_assert(desc);
@@ -1592,7 +1602,7 @@ decrypt_desc_layer,(const hs_descriptor_t *desc,
   return result_len;
 }
 
-/* Decrypt the superencrypted section of the descriptor using the given
+/** Decrypt the superencrypted section of the descriptor using the given
  * descriptor object <b>desc</b>. A newly allocated NUL terminated string is
  * put in decrypted_out which contains the superencrypted layer of the
  * descriptor. Return the length of decrypted_out on success else 0 is
@@ -1607,9 +1617,8 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out)
   tor_assert(decrypted_out);
 
   superencrypted_len = decrypt_desc_layer(desc,
-                                 desc->plaintext_data.superencrypted_blob,
-                                 desc->plaintext_data.superencrypted_blob_size,
-                                 NULL, 1, &superencrypted_plaintext);
+                                          NULL,
+                                          true, &superencrypted_plaintext);
 
   if (!superencrypted_len) {
     log_warn(LD_REND, "Decrypting superencrypted desc failed.");
@@ -1625,7 +1634,7 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out)
   return superencrypted_len;
 }
 
-/* Decrypt the encrypted section of the descriptor using the given descriptor
+/** Decrypt the encrypted section of the descriptor using the given descriptor
  * object <b>desc</b>. A newly allocated NUL terminated string is put in
  * decrypted_out which contains the encrypted layer of the descriptor.
  * Return the length of decrypted_out on success else 0 is returned and
@@ -1658,9 +1667,9 @@ desc_decrypt_encrypted(const hs_descriptor_t *desc,
   }
 
   encrypted_len = decrypt_desc_layer(desc,
-                                 desc->superencrypted_data.encrypted_blob,
-                                 desc->superencrypted_data.encrypted_blob_size,
-                                 descriptor_cookie, 0, &encrypted_plaintext);
+                                     descriptor_cookie,
+                                     false, &encrypted_plaintext);
+
   if (!encrypted_len) {
     goto err;
   }
@@ -1678,7 +1687,7 @@ desc_decrypt_encrypted(const hs_descriptor_t *desc,
   return encrypted_len;
 }
 
-/* Given the token tok for an intro point legacy key, the list of tokens, the
+/** Given the token tok for an intro point legacy key, the list of tokens, the
  * introduction point ip being decoded and the descriptor desc from which it
  * comes from, decode the legacy key and set the intro point object. Return 0
  * on success else -1 on failure. */
@@ -1736,7 +1745,7 @@ decode_intro_legacy_key(const directory_token_t *tok,
   return -1;
 }
 
-/* Dig into the descriptor <b>tokens</b> to find the onion key we should use
+/** Dig into the descriptor <b>tokens</b> to find the onion key we should use
  * for this intro point, and set it into <b>onion_key_out</b>. Return 0 if it
  * was found and well-formed, otherwise return -1 in case of errors. */
 static int
@@ -1780,7 +1789,7 @@ set_intro_point_onion_key(curve25519_public_key_t *onion_key_out,
   return retval;
 }
 
-/* Given the start of a section and the end of it, decode a single
+/** Given the start of a section and the end of it, decode a single
  * introduction point from that section. Return a newly allocated introduction
  * point object containing the decoded data. Return NULL if the section can't
  * be decoded. */
@@ -1909,7 +1918,7 @@ decode_introduction_point(const hs_descriptor_t *desc, const char *start)
   return ip;
 }
 
-/* Given a descriptor string at <b>data</b>, decode all possible introduction
+/** Given a descriptor string at <b>data</b>, decode all possible introduction
  * points that we can find. Add the introduction point object to desc_enc as we
  * find them. This function can't fail and it is possible that zero
  * introduction points can be decoded. */
@@ -1972,7 +1981,8 @@ decode_intro_points(const hs_descriptor_t *desc,
   SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
   smartlist_free(intro_points);
 }
-/* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
+
+/** Return 1 iff the given base64 encoded signature in b64_sig from the encoded
  * descriptor in encoded_desc validates the descriptor content. */
 STATIC int
 desc_sig_is_valid(const char *b64_sig,
@@ -2031,14 +2041,14 @@ desc_sig_is_valid(const char *b64_sig,
   return ret;
 }
 
-/* Decode descriptor plaintext data for version 3. Given a list of tokens, an
+/** Decode descriptor plaintext data for version 3. Given a list of tokens, an
  * allocated plaintext object that will be populated and the encoded
  * descriptor with its length. The last one is needed for signature
  * verification. Unknown tokens are simply ignored so this won't error on
  * unknowns but requires that all v3 token be present and valid.
  *
  * Return 0 on success else a negative value. */
-static int
+static hs_desc_decode_status_t
 desc_decode_plaintext_v3(smartlist_t *tokens,
                          hs_desc_plaintext_data_t *desc,
                          const char *encoded_desc, size_t encoded_len)
@@ -2128,21 +2138,19 @@ desc_decode_plaintext_v3(smartlist_t *tokens,
     goto err;
   }
 
-  return 0;
-
+  return HS_DESC_DECODE_OK;
  err:
-  return -1;
+  return HS_DESC_DECODE_PLAINTEXT_ERROR;
 }
 
-/* Decode the version 3 superencrypted section of the given descriptor desc.
- * The desc_superencrypted_out will be populated with the decoded data.
- * Return 0 on success else -1. */
-static int
+/** Decode the version 3 superencrypted section of the given descriptor desc.
+ * The desc_superencrypted_out will be populated with the decoded data. */
+static hs_desc_decode_status_t
 desc_decode_superencrypted_v3(const hs_descriptor_t *desc,
                               hs_desc_superencrypted_data_t *
                               desc_superencrypted_out)
 {
-  int ret = -1;
+  int ret = HS_DESC_DECODE_SUPERENC_ERROR;
   char *message = NULL;
   size_t message_len;
   memarea_t *area = NULL;
@@ -2228,11 +2236,11 @@ desc_decode_superencrypted_v3(const hs_descriptor_t *desc,
                                               tok->object_size);
   superencrypted->encrypted_blob_size = tok->object_size;
 
-  ret = 0;
+  ret = HS_DESC_DECODE_OK;
   goto done;
 
  err:
-  tor_assert(ret < 0);
+  tor_assert(ret < HS_DESC_DECODE_OK);
   hs_desc_superencrypted_data_free_contents(desc_superencrypted_out);
 
  done:
@@ -2249,15 +2257,14 @@ desc_decode_superencrypted_v3(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Decode the version 3 encrypted section of the given descriptor desc. The
- * desc_encrypted_out will be populated with the decoded data. Return 0 on
- * success else -1. */
-static int
+/** Decode the version 3 encrypted section of the given descriptor desc. The
+ * desc_encrypted_out will be populated with the decoded data. */
+static hs_desc_decode_status_t
 desc_decode_encrypted_v3(const hs_descriptor_t *desc,
                          const curve25519_secret_key_t *client_auth_sk,
                          hs_desc_encrypted_data_t *desc_encrypted_out)
 {
-  int ret = -1;
+  int ret = HS_DESC_DECODE_ENCRYPTED_ERROR;
   char *message = NULL;
   size_t message_len;
   memarea_t *area = NULL;
@@ -2280,12 +2287,14 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc,
        * authorization is failing. */
       log_warn(LD_REND, "Client authorization for requested onion address "
                         "is invalid. Can't decrypt the descriptor.");
+      ret = HS_DESC_DECODE_BAD_CLIENT_AUTH;
     } else {
       /* Inform at notice level that the onion address requested can't be
        * reached without client authorization most likely. */
       log_notice(LD_REND, "Fail to decrypt descriptor for requested onion "
                         "address. It is likely requiring client "
                         "authorization.");
+      ret = HS_DESC_DECODE_NEED_CLIENT_AUTH;
     }
     goto err;
   }
@@ -2344,11 +2353,11 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc,
   /* NOTE: Unknown fields are allowed because this function could be used to
    * decode other descriptor version. */
 
-  ret = 0;
+  ret = HS_DESC_DECODE_OK;
   goto done;
 
  err:
-  tor_assert(ret < 0);
+  tor_assert(ret < HS_DESC_DECODE_OK);
   hs_desc_encrypted_data_free_contents(desc_encrypted_out);
 
  done:
@@ -2365,9 +2374,9 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Table of encrypted decode function version specific. The function are
+/** Table of encrypted decode function version specific. The function are
  * indexed by the version number so v3 callback is at index 3 in the array. */
-static int
+static hs_desc_decode_status_t
   (*decode_encrypted_handlers[])(
       const hs_descriptor_t *desc,
       const curve25519_secret_key_t *client_auth_sk,
@@ -2377,15 +2386,15 @@ static int
   desc_decode_encrypted_v3,
 };
 
-/* Decode the encrypted data section of the given descriptor and store the
+/** Decode the encrypted data section of the given descriptor and store the
  * data in the given encrypted data object. Return 0 on success else a
  * negative value on error. */
-int
+hs_desc_decode_status_t
 hs_desc_decode_encrypted(const hs_descriptor_t *desc,
                          const curve25519_secret_key_t *client_auth_sk,
                          hs_desc_encrypted_data_t *desc_encrypted)
 {
-  int ret;
+  int ret = HS_DESC_DECODE_ENCRYPTED_ERROR;
   uint32_t version;
 
   tor_assert(desc);
@@ -2399,7 +2408,6 @@ hs_desc_decode_encrypted(const hs_descriptor_t *desc,
   /* Let's make sure we have a supported version as well. By correctly parsing
    * the plaintext, this should not fail. */
   if (BUG(!hs_desc_is_supported_version(version))) {
-    ret = -1;
     goto err;
   }
   /* Extra precaution. Having no handler for the supported version should
@@ -2418,9 +2426,9 @@ hs_desc_decode_encrypted(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Table of superencrypted decode function version specific. The function are
+/** Table of superencrypted decode function version specific. The function are
  * indexed by the version number so v3 callback is at index 3 in the array. */
-static int
+static hs_desc_decode_status_t
   (*decode_superencrypted_handlers[])(
       const hs_descriptor_t *desc,
       hs_desc_superencrypted_data_t *desc_superencrypted) =
@@ -2429,15 +2437,14 @@ static int
   desc_decode_superencrypted_v3,
 };
 
-/* Decode the superencrypted data section of the given descriptor and store the
- * data in the given superencrypted data object. Return 0 on success else a
- * negative value on error. */
-int
+/** Decode the superencrypted data section of the given descriptor and store
+ * the data in the given superencrypted data object. */
+hs_desc_decode_status_t
 hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
                               hs_desc_superencrypted_data_t *
                               desc_superencrypted)
 {
-  int ret;
+  int ret = HS_DESC_DECODE_SUPERENC_ERROR;
   uint32_t version;
 
   tor_assert(desc);
@@ -2451,7 +2458,6 @@ hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
   /* Let's make sure we have a supported version as well. By correctly parsing
    * the plaintext, this should not fail. */
   if (BUG(!hs_desc_is_supported_version(version))) {
-    ret = -1;
     goto err;
   }
   /* Extra precaution. Having no handler for the supported version should
@@ -2469,9 +2475,9 @@ hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Table of plaintext decode function version specific. The function are
+/** Table of plaintext decode function version specific. The function are
  * indexed by the version number so v3 callback is at index 3 in the array. */
-static int
+static hs_desc_decode_status_t
   (*decode_plaintext_handlers[])(
       smartlist_t *tokens,
       hs_desc_plaintext_data_t *desc,
@@ -2482,13 +2488,13 @@ static int
   desc_decode_plaintext_v3,
 };
 
-/* Fully decode the given descriptor plaintext and store the data in the
- * plaintext data object. Returns 0 on success else a negative value. */
-int
+/** Fully decode the given descriptor plaintext and store the data in the
+ * plaintext data object. */
+hs_desc_decode_status_t
 hs_desc_decode_plaintext(const char *encoded,
                          hs_desc_plaintext_data_t *plaintext)
 {
-  int ok = 0, ret = -1;
+  int ok = 0, ret = HS_DESC_DECODE_PLAINTEXT_ERROR;
   memarea_t *area = NULL;
   smartlist_t *tokens = NULL;
   size_t encoded_len;
@@ -2538,11 +2544,11 @@ hs_desc_decode_plaintext(const char *encoded,
   /* Run the version specific plaintext decoder. */
   ret = decode_plaintext_handlers[plaintext->version](tokens, plaintext,
                                                       encoded, encoded_len);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
   /* Success. Descriptor has been populated with the data. */
-  ret = 0;
+  ret = HS_DESC_DECODE_OK;
 
  err:
   if (tokens) {
@@ -2555,19 +2561,19 @@ hs_desc_decode_plaintext(const char *encoded,
   return ret;
 }
 
-/* Fully decode an encoded descriptor and set a newly allocated descriptor
+/** Fully decode an encoded descriptor and set a newly allocated descriptor
  * object in desc_out.  Client secret key is used to decrypt the "encrypted"
  * section if not NULL else it's ignored.
  *
  * Return 0 on success. A negative value is returned on error and desc_out is
  * set to NULL. */
-int
+hs_desc_decode_status_t
 hs_desc_decode_descriptor(const char *encoded,
-                          const uint8_t *subcredential,
+                          const hs_subcredential_t *subcredential,
                           const curve25519_secret_key_t *client_auth_sk,
                           hs_descriptor_t **desc_out)
 {
-  int ret = -1;
+  hs_desc_decode_status_t ret = HS_DESC_DECODE_GENERIC_ERROR;
   hs_descriptor_t *desc;
 
   tor_assert(encoded);
@@ -2576,25 +2582,25 @@ hs_desc_decode_descriptor(const char *encoded,
 
   /* Subcredentials are not optional. */
   if (BUG(!subcredential ||
-          tor_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
+          fast_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
     log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!");
     goto err;
   }
 
-  memcpy(desc->subcredential, subcredential, sizeof(desc->subcredential));
+  memcpy(&desc->subcredential, subcredential, sizeof(desc->subcredential));
 
   ret = hs_desc_decode_plaintext(encoded, &desc->plaintext_data);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
   ret = hs_desc_decode_superencrypted(desc, &desc->superencrypted_data);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
   ret = hs_desc_decode_encrypted(desc, client_auth_sk, &desc->encrypted_data);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
@@ -2615,7 +2621,7 @@ hs_desc_decode_descriptor(const char *encoded,
   return ret;
 }
 
-/* Table of encode function version specific. The functions are indexed by the
+/** Table of encode function version specific. The functions are indexed by the
  * version number so v3 callback is at index 3 in the array. */
 static int
   (*encode_handlers[])(
@@ -2628,7 +2634,7 @@ static int
   desc_encode_v3,
 };
 
-/* Encode the given descriptor desc including signing with the given key pair
+/** Encode the given descriptor desc including signing with the given key pair
  * signing_kp and encrypting with the given descriptor cookie.
  *
  * If the client authorization is enabled, descriptor_cookie must be the same
@@ -2671,9 +2677,10 @@ hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
    * symmetric only if the client auth is disabled. That is, the descriptor
    * cookie will be NULL. */
   if (!descriptor_cookie) {
-    ret = hs_desc_decode_descriptor(*encoded_out, desc->subcredential,
+    ret = hs_desc_decode_descriptor(*encoded_out, &desc->subcredential,
                                     NULL, NULL);
-    if (BUG(ret < 0)) {
+    if (BUG(ret != HS_DESC_DECODE_OK)) {
+      ret = -1;
       goto err;
     }
   }
@@ -2685,7 +2692,7 @@ hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Free the content of the plaintext section of a descriptor. */
+/** Free the content of the plaintext section of a descriptor. */
 void
 hs_desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
 {
@@ -2701,7 +2708,7 @@ hs_desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
   memwipe(desc, 0, sizeof(*desc));
 }
 
-/* Free the content of the superencrypted section of a descriptor. */
+/** Free the content of the superencrypted section of a descriptor. */
 void
 hs_desc_superencrypted_data_free_contents(hs_desc_superencrypted_data_t *desc)
 {
@@ -2721,7 +2728,7 @@ hs_desc_superencrypted_data_free_contents(hs_desc_superencrypted_data_t *desc)
   memwipe(desc, 0, sizeof(*desc));
 }
 
-/* Free the content of the encrypted section of a descriptor. */
+/** Free the content of the encrypted section of a descriptor. */
 void
 hs_desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
 {
@@ -2741,7 +2748,7 @@ hs_desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
   memwipe(desc, 0, sizeof(*desc));
 }
 
-/* Free the descriptor plaintext data object. */
+/** Free the descriptor plaintext data object. */
 void
 hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc)
 {
@@ -2749,7 +2756,7 @@ hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc)
   tor_free(desc);
 }
 
-/* Free the descriptor plaintext data object. */
+/** Free the descriptor plaintext data object. */
 void
 hs_desc_superencrypted_data_free_(hs_desc_superencrypted_data_t *desc)
 {
@@ -2757,7 +2764,7 @@ hs_desc_superencrypted_data_free_(hs_desc_superencrypted_data_t *desc)
   tor_free(desc);
 }
 
-/* Free the descriptor encrypted data object. */
+/** Free the descriptor encrypted data object. */
 void
 hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc)
 {
@@ -2765,7 +2772,7 @@ hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc)
   tor_free(desc);
 }
 
-/* Free the given descriptor object. */
+/** Free the given descriptor object. */
 void
 hs_descriptor_free_(hs_descriptor_t *desc)
 {
@@ -2779,7 +2786,7 @@ hs_descriptor_free_(hs_descriptor_t *desc)
   tor_free(desc);
 }
 
-/* Return the size in bytes of the given plaintext data object. A sizeof() is
+/** Return the size in bytes of the given plaintext data object. A sizeof() is
  * not enough because the object contains pointers and the encrypted blob.
  * This is particularly useful for our OOM subsystem that tracks the HSDir
  * cache size for instance. */
@@ -2791,7 +2798,7 @@ hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data)
           data->superencrypted_blob_size);
 }
 
-/* Return the size in bytes of the given encrypted data object. Used by OOM
+/** Return the size in bytes of the given encrypted data object. Used by OOM
  * subsystem. */
 static size_t
 hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data)
@@ -2811,18 +2818,20 @@ hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data)
   return sizeof(*data) + intro_size;
 }
 
-/* Return the size in bytes of the given descriptor object. Used by OOM
+/** Return the size in bytes of the given descriptor object. Used by OOM
  * subsystem. */
   size_t
 hs_desc_obj_size(const hs_descriptor_t *data)
 {
-  tor_assert(data);
+  if (data == NULL) {
+    return 0;
+  }
   return (hs_desc_plaintext_obj_size(&data->plaintext_data) +
           hs_desc_encrypted_obj_size(&data->encrypted_data) +
           sizeof(data->subcredential));
 }
 
-/* Return a newly allocated descriptor intro point. */
+/** Return a newly allocated descriptor intro point. */
 hs_desc_intro_point_t *
 hs_desc_intro_point_new(void)
 {
@@ -2831,7 +2840,7 @@ hs_desc_intro_point_new(void)
   return ip;
 }
 
-/* Free a descriptor intro point object. */
+/** Free a descriptor intro point object. */
 void
 hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
 {
@@ -2839,8 +2848,8 @@ hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
     return;
   }
   if (ip->link_specifiers) {
-    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
-                      ls, hs_desc_link_specifier_free(ls));
+    SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *,
+                      ls, link_specifier_free(ls));
     smartlist_free(ip->link_specifiers);
   }
   tor_cert_free(ip->auth_key_cert);
@@ -2850,7 +2859,7 @@ hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
   tor_free(ip);
 }
 
-/* Allocate and build a new fake client info for the descriptor. Return a
+/** Allocate and build a new fake client info for the descriptor. Return a
  * newly allocated object. This can't fail. */
 hs_desc_authorized_client_t *
 hs_desc_build_fake_authorized_client(void)
@@ -2868,49 +2877,44 @@ hs_desc_build_fake_authorized_client(void)
   return client_auth;
 }
 
-/* Using the service's subcredential, client public key, auth ephemeral secret
+/** Using the service's subcredential, client public key, auth ephemeral secret
  * key, and descriptor cookie, build the auth client so we can then encode the
  * descriptor for publication. client_out must be already allocated. */
 void
-hs_desc_build_authorized_client(const uint8_t *subcredential,
+hs_desc_build_authorized_client(const hs_subcredential_t *subcredential,
                                 const curve25519_public_key_t *client_auth_pk,
                                 const curve25519_secret_key_t *
                                 auth_ephemeral_sk,
                                 const uint8_t *descriptor_cookie,
                                 hs_desc_authorized_client_t *client_out)
 {
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
+  const uint8_t *cookie_key;
   crypto_cipher_t *cipher;
-  crypto_xof_t *xof;
 
   tor_assert(client_auth_pk);
   tor_assert(auth_ephemeral_sk);
   tor_assert(descriptor_cookie);
   tor_assert(client_out);
   tor_assert(subcredential);
-  tor_assert(!tor_mem_is_zero((char *) auth_ephemeral_sk,
+  tor_assert(!fast_mem_is_zero((char *) auth_ephemeral_sk,
                               sizeof(*auth_ephemeral_sk)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_pk,
+  tor_assert(!fast_mem_is_zero((char *) client_auth_pk,
                               sizeof(*client_auth_pk)));
-  tor_assert(!tor_mem_is_zero((char *) descriptor_cookie,
+  tor_assert(!fast_mem_is_zero((char *) descriptor_cookie,
                               HS_DESC_DESCRIPTOR_COOKIE_LEN));
-  tor_assert(!tor_mem_is_zero((char *) subcredential,
+  tor_assert(!fast_mem_is_zero((char *) subcredential,
                               DIGEST256_LEN));
 
-  /* Calculate x25519(hs_y, client_X) */
-  curve25519_handshake(secret_seed,
-                       auth_ephemeral_sk,
-                       client_auth_pk);
-
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(subcredential,
+                                 auth_ephemeral_sk, client_auth_pk,
+                                 &keystream);
+  tor_assert(keystream_length > 0);
 
+  /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */
   memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN);
   cookie_key = keystream + HS_DESC_CLIENT_ID_LEN;
 
@@ -2925,83 +2929,20 @@ hs_desc_build_authorized_client(const uint8_t *subcredential,
                         (const char *) descriptor_cookie,
                         HS_DESC_DESCRIPTOR_COOKIE_LEN);
 
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
 
   crypto_cipher_free(cipher);
 }
 
-/* Free an authoriezd client object. */
+/** Free an authoriezd client object. */
 void
 hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client)
 {
   tor_free(client);
 }
 
-/* Free the given descriptor link specifier. */
-void
-hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls)
-{
-  if (ls == NULL) {
-    return;
-  }
-  tor_free(ls);
-}
-
-/* Return a newly allocated descriptor link specifier using the given extend
- * info and requested type. Return NULL on error. */
-hs_desc_link_specifier_t *
-hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type)
-{
-  hs_desc_link_specifier_t *ls = NULL;
-
-  tor_assert(info);
-
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = type;
-  switch (ls->type) {
-  case LS_IPV4:
-    if (info->addr.family != AF_INET) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_IPV6:
-    if (info->addr.family != AF_INET6) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_LEGACY_ID:
-    /* Bug out if the identity digest is not set */
-    if (BUG(tor_mem_is_zero(info->identity_digest,
-                            sizeof(info->identity_digest)))) {
-      goto err;
-    }
-    memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id));
-    break;
-  case LS_ED25519_ID:
-    /* ed25519 keys are optional for intro points */
-    if (ed25519_public_key_is_zero(&info->ed_identity)) {
-      goto err;
-    }
-    memcpy(ls->u.ed25519_id, info->ed_identity.pubkey,
-           sizeof(ls->u.ed25519_id));
-    break;
-  default:
-    /* Unknown type is code flow error. */
-    tor_assert(0);
-  }
-
-  return ls;
- err:
-  tor_free(ls);
-  return NULL;
-}
-
-/* From the given descriptor, remove and free every introduction point. */
+/** From the given descriptor, remove and free every introduction point. */
 void
 hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
 {
@@ -3016,59 +2957,3 @@ hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
     smartlist_clear(ips);
   }
 }
-
-/* From a descriptor link specifier object spec, returned a newly allocated
- * link specifier object that is the encoded representation of spec. Return
- * NULL on error. */
-link_specifier_t *
-hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec)
-{
-  tor_assert(spec);
-
-  link_specifier_t *ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, spec->type);
-
-  switch (spec->type) {
-  case LS_IPV4:
-    link_specifier_set_un_ipv4_addr(ls,
-                                    tor_addr_to_ipv4h(&spec->u.ap.addr));
-    link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
-    /* Four bytes IPv4 and two bytes port. */
-    link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
-                                  sizeof(spec->u.ap.port));
-    break;
-  case LS_IPV6:
-  {
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
-    break;
-  }
-  case LS_LEGACY_ID:
-  {
-    size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
-    uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
-    memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
-    link_specifier_set_ls_len(ls, legacy_id_len);
-    break;
-  }
-  case LS_ED25519_ID:
-  {
-    size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls);
-    uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls);
-    memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len);
-    link_specifier_set_ls_len(ls, ed25519_id_len);
-    break;
-  }
-  default:
-    tor_assert_nonfatal_unreached();
-    link_specifier_free(ls);
-    ls = NULL;
-  }
-
-  return ls;
-}
diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h
index 04a8e16d63..08daa904b6 100644
--- a/src/feature/hs/hs_descriptor.h
+++ b/src/feature/hs/hs_descriptor.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,110 +14,118 @@
 #include "core/or/or.h"
 #include "trunnel/ed25519_cert.h" /* needed for trunnel */
 #include "feature/nodelist/torcert.h"
+#include "core/crypto/hs_ntor.h" /* for hs_subcredential_t */
 
 /* Trunnel */
 struct link_specifier_t;
 
-/* The earliest descriptor format version we support. */
+/** The earliest descriptor format version we support. */
 #define HS_DESC_SUPPORTED_FORMAT_VERSION_MIN 3
-/* The latest descriptor format version we support. */
+/** The latest descriptor format version we support. */
 #define HS_DESC_SUPPORTED_FORMAT_VERSION_MAX 3
 
-/* Default lifetime of a descriptor in seconds. The valus is set at 3 hours
+/** Default lifetime of a descriptor in seconds. The valus is set at 3 hours
  * which is 180 minutes or 10800 seconds. */
 #define HS_DESC_DEFAULT_LIFETIME (3 * 60 * 60)
-/* Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
+/** Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
  * which is 720 minutes or 43200 seconds. */
 #define HS_DESC_MAX_LIFETIME (12 * 60 * 60)
-/* Lifetime of certificate in the descriptor. This defines the lifetime of the
+/** Lifetime of certificate in the descriptor. This defines the lifetime of the
  * descriptor signing key and the cross certification cert of that key. It is
  * set to 54 hours because a descriptor can be around for 48 hours and because
  * consensuses are used after the hour, add an extra 6 hours to give some time
  * for the service to stop using it. */
 #define HS_DESC_CERT_LIFETIME (54 * 60 * 60)
-/* Length of the salt needed for the encrypted section of a descriptor. */
+/** Length of the salt needed for the encrypted section of a descriptor. */
 #define HS_DESC_ENCRYPTED_SALT_LEN 16
-/* Length of the KDF output value which is the length of the secret key,
+/** Length of the KDF output value which is the length of the secret key,
  * the secret IV and MAC key length which is the length of H() output. */
 #define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \
   CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN
-/* Pad plaintext of superencrypted data section before encryption so that its
+/** Pad plaintext of superencrypted data section before encryption so that its
  * length is a multiple of this value. */
 #define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000
-/* Maximum length in bytes of a full hidden service descriptor. */
+/** Maximum length in bytes of a full hidden service descriptor. */
 #define HS_DESC_MAX_LEN 50000 /* 50kb max size */
 
-/* Key length for the descriptor symmetric encryption. As specified in the
+/** Key length for the descriptor symmetric encryption. As specified in the
  * protocol, we use AES-256 for the encrypted section of the descriptor. The
  * following is the length in bytes and the bit size. */
 #define HS_DESC_ENCRYPTED_KEY_LEN CIPHER256_KEY_LEN
 #define HS_DESC_ENCRYPTED_BIT_SIZE (HS_DESC_ENCRYPTED_KEY_LEN * 8)
 
-/* Length of each components in the auth client section in the descriptor. */
+/** Length of each components in the auth client section in the descriptor. */
 #define HS_DESC_CLIENT_ID_LEN 8
 #define HS_DESC_DESCRIPTOR_COOKIE_LEN 16
 #define HS_DESC_COOKIE_KEY_LEN 32
 #define HS_DESC_COOKIE_KEY_BIT_SIZE (HS_DESC_COOKIE_KEY_LEN * 8)
 #define HS_DESC_ENCRYPED_COOKIE_LEN HS_DESC_DESCRIPTOR_COOKIE_LEN
 
-/* The number of auth client entries in the descriptor must be the multiple
+/** The number of auth client entries in the descriptor must be the multiple
  * of this constant. */
 #define HS_DESC_AUTH_CLIENT_MULTIPLE 16
 
-/* Type of authentication in the descriptor. */
+/** Type of authentication in the descriptor. */
 typedef enum {
   HS_DESC_AUTH_ED25519 = 1
 } hs_desc_auth_type_t;
 
-/* Link specifier object that contains information on how to extend to the
- * relay that is the address, port and handshake type. */
-typedef struct hs_desc_link_specifier_t {
-  /* Indicate the type of link specifier. See trunnel ed25519_cert
-   * specification. */
-  uint8_t type;
-
-  /* It must be one of these types, can't be more than one. */
-  union {
-    /* IP address and port of the relay use to extend. */
-    tor_addr_port_t ap;
-    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
-    uint8_t legacy_id[DIGEST_LEN];
-    /* ed25519 identity. A 32-byte key. */
-    uint8_t ed25519_id[ED25519_PUBKEY_LEN];
-  } u;
-} hs_desc_link_specifier_t;
-
-/* Introduction point information located in a descriptor. */
+/** Error code when decoding a descriptor. */
+typedef enum {
+  /* The configured client authorization for the requested .onion address
+   * failed to decode the descriptor. */
+  HS_DESC_DECODE_BAD_CLIENT_AUTH  = -6,
+
+  /* The requested .onion address requires a client authorization. */
+  HS_DESC_DECODE_NEED_CLIENT_AUTH = -5,
+
+  /* Error during decryption of the encrypted layer. */
+  HS_DESC_DECODE_ENCRYPTED_ERROR  = -4,
+
+  /* Error during decryption of the super encrypted layer. */
+  HS_DESC_DECODE_SUPERENC_ERROR   = -3,
+
+  /* Error while decoding the plaintext section. */
+  HS_DESC_DECODE_PLAINTEXT_ERROR  = -2,
+
+  /* Generic error. */
+  HS_DESC_DECODE_GENERIC_ERROR    = -1,
+
+  /* Decoding a descriptor was successful. */
+  HS_DESC_DECODE_OK               =  0,
+} hs_desc_decode_status_t;
+
+/** Introduction point information located in a descriptor. */
 typedef struct hs_desc_intro_point_t {
-  /* Link specifier(s) which details how to extend to the relay. This list
-   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+  /** Link specifier(s) which details how to extend to the relay. This list
+   * contains link_specifier_t objects. It MUST have at least one. */
   smartlist_t *link_specifiers;
 
-  /* Onion key of the introduction point used to extend to it for the ntor
+  /** Onion key of the introduction point used to extend to it for the ntor
    * handshake. */
   curve25519_public_key_t onion_key;
 
-  /* Authentication key used to establish the introduction point circuit and
+  /** Authentication key used to establish the introduction point circuit and
    * cross-certifies the blinded public key for the replica thus signed by
    * the blinded key and in turn signs it. */
   tor_cert_t *auth_key_cert;
 
-  /* Encryption key for the "ntor" type. */
+  /** Encryption key for the "ntor" type. */
   curve25519_public_key_t enc_key;
 
-  /* Certificate cross certifying the descriptor signing key by the encryption
+  /** Certificate cross certifying the descriptor signing key by the encryption
    * curve25519 key. This certificate contains the signing key and is of type
    * CERT_TYPE_CROSS_HS_IP_KEYS [0B]. */
   tor_cert_t *enc_key_cert;
 
-  /* (Optional): If this introduction point is a legacy one that is version <=
+  /** (Optional): If this introduction point is a legacy one that is version <=
    * 0.2.9.x (HSIntro=3), we use this extra key for the intro point to be able
    * to relay the cells to the service correctly. */
   struct {
-    /* RSA public key. */
+    /** RSA public key. */
     crypto_pk_t *key;
 
-    /* Cross certified cert with the descriptor signing key (RSA->Ed). Because
+    /** Cross certified cert with the descriptor signing key (RSA->Ed). Because
      * of the cross certification API, we need to keep the certificate binary
      * blob and its length in order to properly encode it after. */
     struct {
@@ -126,115 +134,115 @@ typedef struct hs_desc_intro_point_t {
     } cert;
   } legacy;
 
-  /* True iff the introduction point has passed the cross certification. Upon
+  /** True iff the introduction point has passed the cross certification. Upon
    * decoding an intro point, this must be true. */
   unsigned int cross_certified : 1;
 } hs_desc_intro_point_t;
 
-/* Authorized client information located in a descriptor. */
+/** Authorized client information located in a descriptor. */
 typedef struct hs_desc_authorized_client_t {
-  /* An identifier that the client will use to identify which auth client
+  /** An identifier that the client will use to identify which auth client
    * entry it needs to use. */
   uint8_t client_id[HS_DESC_CLIENT_ID_LEN];
 
-  /* An IV that is used to decrypt the encrypted descriptor cookie. */
+  /** An IV that is used to decrypt the encrypted descriptor cookie. */
   uint8_t iv[CIPHER_IV_LEN];
 
-  /* An encrypted descriptor cookie that the client needs to decrypt to use
+  /** An encrypted descriptor cookie that the client needs to decrypt to use
    * it to decrypt the descriptor. */
   uint8_t encrypted_cookie[HS_DESC_ENCRYPED_COOKIE_LEN];
 } hs_desc_authorized_client_t;
 
-/* The encrypted data section of a descriptor. Obviously the data in this is
+/** The encrypted data section of a descriptor. Obviously the data in this is
  * in plaintext but encrypted once encoded. */
 typedef struct hs_desc_encrypted_data_t {
-  /* Bitfield of CREATE2 cell supported formats. The only currently supported
+  /** Bitfield of CREATE2 cell supported formats. The only currently supported
    * format is ntor. */
   unsigned int create2_ntor : 1;
 
-  /* A list of authentication types that a client must at least support one
+  /** A list of authentication types that a client must at least support one
    * in order to contact the service. Contains NULL terminated strings. */
   smartlist_t *intro_auth_types;
 
-  /* Is this descriptor a single onion service? */
+  /** Is this descriptor a single onion service? */
   unsigned int single_onion_service : 1;
 
-  /* A list of intro points. Contains hs_desc_intro_point_t objects. */
+  /** A list of intro points. Contains hs_desc_intro_point_t objects. */
   smartlist_t *intro_points;
 } hs_desc_encrypted_data_t;
 
-/* The superencrypted data section of a descriptor. Obviously the data in
+/** The superencrypted data section of a descriptor. Obviously the data in
  * this is in plaintext but encrypted once encoded. */
 typedef struct hs_desc_superencrypted_data_t {
-  /* This field contains ephemeral x25519 public key which is used by
+  /** This field contains ephemeral x25519 public key which is used by
    * the encryption scheme in the client authorization. */
   curve25519_public_key_t auth_ephemeral_pubkey;
 
-  /* A list of authorized clients. Contains hs_desc_authorized_client_t
+  /** A list of authorized clients. Contains hs_desc_authorized_client_t
    * objects. */
   smartlist_t *clients;
 
-  /* Decoding only: The b64-decoded encrypted blob from the descriptor */
+  /** Decoding only: The b64-decoded encrypted blob from the descriptor */
   uint8_t *encrypted_blob;
 
-  /* Decoding only: Size of the encrypted_blob */
+  /** Decoding only: Size of the encrypted_blob */
   size_t encrypted_blob_size;
 } hs_desc_superencrypted_data_t;
 
-/* Plaintext data that is unencrypted information of the descriptor. */
+/** Plaintext data that is unencrypted information of the descriptor. */
 typedef struct hs_desc_plaintext_data_t {
-  /* Version of the descriptor format. Spec specifies this field as a
+  /** Version of the descriptor format. Spec specifies this field as a
    * positive integer. */
   uint32_t version;
 
-  /* The lifetime of the descriptor in seconds. */
+  /** The lifetime of the descriptor in seconds. */
   uint32_t lifetime_sec;
 
-  /* Certificate with the short-term ed22519 descriptor signing key for the
+  /** Certificate with the short-term ed22519 descriptor signing key for the
    * replica which is signed by the blinded public key for that replica. */
   tor_cert_t *signing_key_cert;
 
-  /* Signing public key which is used to sign the descriptor. Same public key
+  /** Signing public key which is used to sign the descriptor. Same public key
    * as in the signing key certificate. */
   ed25519_public_key_t signing_pubkey;
 
-  /* Blinded public key used for this descriptor derived from the master
+  /** Blinded public key used for this descriptor derived from the master
    * identity key and generated for a specific replica number. */
   ed25519_public_key_t blinded_pubkey;
 
-  /* Revision counter is incremented at each upload, regardless of whether
+  /** Revision counter is incremented at each upload, regardless of whether
    * the descriptor has changed. This avoids leaking whether the descriptor
    * has changed. Spec specifies this as a 8 bytes positive integer. */
   uint64_t revision_counter;
 
-  /* Decoding only: The b64-decoded superencrypted blob from the descriptor */
+  /** Decoding only: The b64-decoded superencrypted blob from the descriptor */
   uint8_t *superencrypted_blob;
 
-  /* Decoding only: Size of the superencrypted_blob */
+  /** Decoding only: Size of the superencrypted_blob */
   size_t superencrypted_blob_size;
 } hs_desc_plaintext_data_t;
 
-/* Service descriptor in its decoded form. */
+/** Service descriptor in its decoded form. */
 typedef struct hs_descriptor_t {
-  /* Contains the plaintext part of the descriptor. */
+  /** Contains the plaintext part of the descriptor. */
   hs_desc_plaintext_data_t plaintext_data;
 
-  /* The following contains what's in the superencrypted part of the
+  /** The following contains what's in the superencrypted part of the
    * descriptor. It's only encrypted in the encoded version of the descriptor
    * thus the data contained in that object is in plaintext. */
   hs_desc_superencrypted_data_t superencrypted_data;
 
-  /* The following contains what's in the encrypted part of the descriptor.
+  /** The following contains what's in the encrypted part of the descriptor.
    * It's only encrypted in the encoded version of the descriptor thus the
    * data contained in that object is in plaintext. */
   hs_desc_encrypted_data_t encrypted_data;
 
-  /* Subcredentials of a service, used by the client and service to decrypt
+  /** Subcredentials of a service, used by the client and service to decrypt
    * the encrypted data. */
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 } hs_descriptor_t;
 
-/* Return true iff the given descriptor format version is supported. */
+/** Return true iff the given descriptor format version is supported. */
 static inline int
 hs_desc_is_supported_version(uint32_t version)
 {
@@ -261,12 +269,6 @@ void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc);
 #define hs_desc_encrypted_data_free(desc) \
   FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc))
 
-void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls);
-#define hs_desc_link_specifier_free(ls) \
-  FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls))
-
-hs_desc_link_specifier_t *hs_desc_link_specifier_new(
-                                  const extend_info_t *info, uint8_t type);
 void hs_descriptor_clear_intro_points(hs_descriptor_t *desc);
 
 MOCK_DECL(int,
@@ -276,7 +278,7 @@ MOCK_DECL(int,
                                      char **encoded_out));
 
 int hs_desc_decode_descriptor(const char *encoded,
-                              const uint8_t *subcredential,
+                              const hs_subcredential_t *subcredential,
                               const curve25519_secret_key_t *client_auth_sk,
                               hs_descriptor_t **desc_out);
 int hs_desc_decode_plaintext(const char *encoded,
@@ -299,11 +301,9 @@ void hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client);
   FREE_AND_NULL(hs_desc_authorized_client_t, \
                 hs_desc_authorized_client_free_, (client))
 
-link_specifier_t *hs_desc_lspec_to_trunnel(
-                                   const hs_desc_link_specifier_t *spec);
-
 hs_desc_authorized_client_t *hs_desc_build_fake_authorized_client(void);
-void hs_desc_build_authorized_client(const uint8_t *subcredential,
+
+void hs_desc_build_authorized_client(const hs_subcredential_t *subcredential,
                                      const curve25519_public_key_t *
                                      client_auth_pk,
                                      const curve25519_secret_key_t *
@@ -335,10 +335,8 @@ STATIC int desc_sig_is_valid(const char *b64_sig,
                              const char *encoded_desc, size_t encoded_len);
 
 MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc,
-                                             const uint8_t *encrypted_blob,
-                                             size_t encrypted_blob_size,
                                              const uint8_t *descriptor_cookie,
-                                             int is_superencrypted_layer,
+                                             bool is_superencrypted_layer,
                                              char **decrypted_out));
 
 #endif /* defined(HS_DESCRIPTOR_PRIVATE) */
diff --git a/src/feature/hs/hs_dos.c b/src/feature/hs/hs_dos.c
new file mode 100644
index 0000000000..04c2bfbb89
--- /dev/null
+++ b/src/feature/hs/hs_dos.c
@@ -0,0 +1,228 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.c
+ * \brief Implement denial of service mitigation for the onion service
+ *        subsystem.
+ *
+ * This module defenses:
+ *
+ * - Introduction Rate Limiting: If enabled by the consensus, an introduction
+ *   point will rate limit client introduction towards the service (INTRODUCE2
+ *   cells). It uses a token bucket model with a rate and burst per second.
+ *
+ *   Proposal 305 will expand this module by allowing an operator to define
+ *   these values into the ESTABLISH_INTRO cell. Not yet implemented.
+ **/
+
+#define HS_DOS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/routermode.h"
+
+#include "lib/evloop/token_bucket.h"
+
+#include "feature/hs/hs_dos.h"
+
+/** Default value of the allowed INTRODUCE2 cell rate per second. Above that
+ * value per second, the introduction is denied. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC 25
+
+/** Default value of the allowed INTRODUCE2 cell burst per second. This is the
+ * maximum value a token bucket has per second. We thus allow up to this value
+ * of INTRODUCE2 cell per second but the bucket is refilled by the rate value
+ * but never goes above that burst value. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC 200
+
+/** Default value of the consensus parameter enabling or disabling the
+ * introduction DoS defense. Disabled by default. */
+#define HS_DOS_INTRODUCE_ENABLED_DEFAULT 0
+
+/** INTRODUCE2 rejected request counter. */
+static uint64_t intro2_rejected_count = 0;
+
+/* Consensus parameters. The ESTABLISH_INTRO DoS cell extension have higher
+ * priority than these values. If no extension is sent, these are used only by
+ * the introduction point. */
+static uint32_t consensus_param_introduce_rate_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC;
+static uint32_t consensus_param_introduce_burst_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC;
+static uint32_t consensus_param_introduce_defense_enabled =
+  HS_DOS_INTRODUCE_ENABLED_DEFAULT;
+
+STATIC uint32_t
+get_intro2_enable_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSDefense",
+                                 HS_DOS_INTRODUCE_ENABLED_DEFAULT, 0, 1);
+}
+
+/** Return the parameter for the introduction rate per sec. */
+STATIC uint32_t
+get_intro2_rate_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSRatePerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/** Return the parameter for the introduction burst per sec. */
+STATIC uint32_t
+get_intro2_burst_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSBurstPerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/** Go over all introduction circuit relay side and adjust their rate/burst
+ * values using the global parameters. This is called right after the
+ * consensus parameters might have changed. */
+static void
+update_intro_circuits(void)
+{
+  /* Returns all HS version intro circuits. */
+  smartlist_t *intro_circs = hs_circuitmap_get_all_intro_circ_relay_side();
+
+  SMARTLIST_FOREACH_BEGIN(intro_circs, circuit_t *, circ) {
+    /* Ignore circuit if the defenses were set explicitly through the
+     * ESTABLISH_INTRO cell DoS extension. */
+    if (TO_OR_CIRCUIT(circ)->introduce2_dos_defense_explicit) {
+      continue;
+    }
+    /* Defenses might have been enabled or disabled. */
+    TO_OR_CIRCUIT(circ)->introduce2_dos_defense_enabled =
+      consensus_param_introduce_defense_enabled;
+    /* Adjust the rate/burst value that might have changed. */
+    token_bucket_ctr_adjust(&TO_OR_CIRCUIT(circ)->introduce2_bucket,
+                            consensus_param_introduce_rate_per_sec,
+                            consensus_param_introduce_burst_per_sec);
+  } SMARTLIST_FOREACH_END(circ);
+
+  smartlist_free(intro_circs);
+}
+
+/** Set consensus parameters. */
+static void
+set_consensus_parameters(const networkstatus_t *ns)
+{
+  consensus_param_introduce_rate_per_sec =
+    get_intro2_rate_consensus_param(ns);
+  consensus_param_introduce_burst_per_sec =
+    get_intro2_burst_consensus_param(ns);
+  consensus_param_introduce_defense_enabled =
+    get_intro2_enable_consensus_param(ns);
+
+  /* The above might have changed which means we need to go through all
+   * introduction circuits (relay side) and update the token buckets. */
+  update_intro_circuits();
+}
+
+/*
+ * Public API.
+ */
+
+/** Initialize the INTRODUCE2 token bucket for the DoS defenses using the
+ * consensus/default values. We might get a cell extension that changes those
+ * later but if we don't, the default or consensus parameters are used. */
+void
+hs_dos_setup_default_intro2_defenses(or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  circ->introduce2_dos_defense_enabled =
+    consensus_param_introduce_defense_enabled;
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        consensus_param_introduce_rate_per_sec,
+                        consensus_param_introduce_burst_per_sec,
+                        (uint32_t) approx_time());
+}
+
+/** Called when the consensus has changed. We might have new consensus
+ * parameters to look at. */
+void
+hs_dos_consensus_has_changed(const networkstatus_t *ns)
+{
+  /* No point on updating these values if we are not a public relay that can
+   * be picked to be an introduction point. */
+  if (!public_server_mode(get_options())) {
+    return;
+  }
+
+  set_consensus_parameters(ns);
+}
+
+/** Return true iff an INTRODUCE2 cell can be sent on the given service
+ * introduction circuit. */
+bool
+hs_dos_can_send_intro2(or_circuit_t *s_intro_circ)
+{
+  tor_assert(s_intro_circ);
+
+  /* Allow to send the cell if the DoS defenses are disabled on the circuit.
+   * This can be set by the consensus, the ESTABLISH_INTRO cell extension or
+   * the hardcoded values in tor code. */
+  if (!s_intro_circ->introduce2_dos_defense_enabled) {
+    goto allow;
+  }
+
+  /* Should not happen but if so, scream loudly. */
+  if (BUG(TO_CIRCUIT(s_intro_circ)->purpose != CIRCUIT_PURPOSE_INTRO_POINT)) {
+    goto disallow;
+  }
+
+  /* This is called just after we got a valid and parsed INTRODUCE1 cell. The
+   * service has been found and we have its introduction circuit.
+   *
+   * First, the INTRODUCE2 bucket will be refilled (if any). Then, decremented
+   * because we are about to send or not the cell we just got. Finally,
+   * evaluate if we can send it based on our token bucket state. */
+
+  /* Refill INTRODUCE2 bucket. */
+  token_bucket_ctr_refill(&s_intro_circ->introduce2_bucket,
+                          (uint32_t) approx_time());
+
+  /* Decrement the bucket for this valid INTRODUCE1 cell we just got. Don't
+   * underflow else we end up with a too big of a bucket. */
+  if (token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0) {
+    token_bucket_ctr_dec(&s_intro_circ->introduce2_bucket, 1);
+  }
+
+  /* Finally, we can send a new INTRODUCE2 if there are still tokens. */
+  if (token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0) {
+    goto allow;
+  }
+
+  /* If we reach this point, then it means the bucket has reached zero, and
+     we're going to disallow. */
+
+ disallow:
+  /* Increment stats counter, we are rejecting the INTRO2 cell. */
+  intro2_rejected_count++;
+  return false;
+
+ allow:
+  return true;
+}
+
+/** Return rolling count of rejected INTRO2. */
+uint64_t
+hs_dos_get_intro2_rejected_count(void)
+{
+  return intro2_rejected_count;
+}
+
+/** Initialize the onion service Denial of Service subsystem. */
+void
+hs_dos_init(void)
+{
+  set_consensus_parameters(NULL);
+}
diff --git a/src/feature/hs/hs_dos.h b/src/feature/hs/hs_dos.h
new file mode 100644
index 0000000000..8e36ece204
--- /dev/null
+++ b/src/feature/hs/hs_dos.h
@@ -0,0 +1,42 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.h
+ * \brief Header file containing denial of service defenses for the HS
+ *        subsystem for all versions.
+ **/
+
+#ifndef TOR_HS_DOS_H
+#define TOR_HS_DOS_H
+
+#include "core/or/or_circuit_st.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+/* Init */
+void hs_dos_init(void);
+
+/* Consensus. */
+void hs_dos_consensus_has_changed(const networkstatus_t *ns);
+
+/* Introduction Point. */
+bool hs_dos_can_send_intro2(or_circuit_t *s_intro_circ);
+void hs_dos_setup_default_intro2_defenses(or_circuit_t *circ);
+
+/* Statistics. */
+uint64_t hs_dos_get_intro2_rejected_count(void);
+
+#ifdef HS_DOS_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC uint32_t get_intro2_enable_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_rate_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_burst_consensus_param(const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_DOS_PRIVATE) */
+
+#endif /* !defined(TOR_HS_DOS_H) */
diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c
index 8fd0013941..1d93ff9610 100644
--- a/src/feature/hs/hs_ident.c
+++ b/src/feature/hs/hs_ident.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,21 +10,17 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/hs/hs_ident.h"
 
-/* Return a newly allocated circuit identifier. The given public key is copied
+/** Return a newly allocated circuit identifier. The given public key is copied
  * identity_pk into the identifier. */
 hs_ident_circuit_t *
-hs_ident_circuit_new(const ed25519_public_key_t *identity_pk,
-                     hs_ident_circuit_type_t circuit_type)
+hs_ident_circuit_new(const ed25519_public_key_t *identity_pk)
 {
-  tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO ||
-             circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS);
   hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
   ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
-  ident->circuit_type = circuit_type;
   return ident;
 }
 
-/* Free the given circuit identifier. */
+/** Free the given circuit identifier. */
 void
 hs_ident_circuit_free_(hs_ident_circuit_t *ident)
 {
@@ -35,7 +31,7 @@ hs_ident_circuit_free_(hs_ident_circuit_t *ident)
   tor_free(ident);
 }
 
-/* For a given circuit identifier src, return a newly allocated copy of it.
+/** For a given circuit identifier src, return a newly allocated copy of it.
  * This can't fail. */
 hs_ident_circuit_t *
 hs_ident_circuit_dup(const hs_ident_circuit_t *src)
@@ -45,7 +41,7 @@ hs_ident_circuit_dup(const hs_ident_circuit_t *src)
   return ident;
 }
 
-/* For a given directory connection identifier src, return a newly allocated
+/** For a given directory connection identifier src, return a newly allocated
  * copy of it. This can't fail. */
 hs_ident_dir_conn_t *
 hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src)
@@ -55,7 +51,7 @@ hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src)
   return ident;
 }
 
-/* Free the given directory connection identifier. */
+/** Free the given directory connection identifier. */
 void
 hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident)
 {
@@ -66,7 +62,7 @@ hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident)
   tor_free(ident);
 }
 
-/* Initialized the allocated ident object with identity_pk and blinded_pk.
+/** Initialized the allocated ident object with identity_pk and blinded_pk.
  * None of them can be NULL since a valid directory connection identifier must
  * have all fields set. */
 void
@@ -82,7 +78,7 @@ hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
   ed25519_pubkey_copy(&ident->blinded_pk, blinded_pk);
 }
 
-/* Return a newly allocated edge connection identifier. The given public key
+/** Return a newly allocated edge connection identifier. The given public key
  * identity_pk is copied into the identifier. */
 hs_ident_edge_conn_t *
 hs_ident_edge_conn_new(const ed25519_public_key_t *identity_pk)
@@ -92,7 +88,7 @@ hs_ident_edge_conn_new(const ed25519_public_key_t *identity_pk)
   return ident;
 }
 
-/* Free the given edge connection identifier. */
+/** Free the given edge connection identifier. */
 void
 hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident)
 {
@@ -103,7 +99,7 @@ hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident)
   tor_free(ident);
 }
 
-/* Return true if the given ident is valid for an introduction circuit. */
+/** Return true if the given ident is valid for an introduction circuit. */
 int
 hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident)
 {
@@ -124,4 +120,3 @@ hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident)
  invalid:
   return 0;
 }
-
diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h
index 8c46936a1e..f4b9b2432d 100644
--- a/src/feature/hs/hs_ident.h
+++ b/src/feature/hs/hs_ident.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,77 +25,71 @@
 
 #include "feature/hs/hs_common.h"
 
-/* Length of the rendezvous cookie that is used to connect circuits at the
+/** Length of the rendezvous cookie that is used to connect circuits at the
  * rendezvous point. */
 #define HS_REND_COOKIE_LEN DIGEST_LEN
 
-/* Type of circuit an hs_ident_t object is associated with. */
+/** Type of circuit an hs_ident_t object is associated with. */
 typedef enum {
   HS_IDENT_CIRCUIT_INTRO      = 1,
   HS_IDENT_CIRCUIT_RENDEZVOUS = 2,
 } hs_ident_circuit_type_t;
 
-/* Client and service side circuit identifier that is used for hidden service
+/** Client and service side circuit identifier that is used for hidden service
  * circuit establishment. Not all fields contain data, it depends on the
  * circuit purpose. This is attached to an origin_circuit_t. All fields are
  * used by both client and service. */
 typedef struct hs_ident_circuit_t {
-  /* (All circuit) The public key used to uniquely identify the service. It is
+  /** (All circuit) The public key used to uniquely identify the service. It is
    * the one found in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* (All circuit) The type of circuit this identifier is attached to.
-   * Accessors of the fields in this object assert non fatal on this circuit
-   * type. In other words, if a rendezvous field is being accessed, the
-   * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is
-   * set when an object is initialized in its constructor. */
-  hs_ident_circuit_type_t circuit_type;
-
-  /* (All circuit) Introduction point authentication key. It's also needed on
+  /** (All circuit) Introduction point authentication key. It's also needed on
    * the rendezvous circuit for the ntor handshake. It's used as the unique key
    * of the introduction point so it should not be shared between multiple
    * intro points. */
   ed25519_public_key_t intro_auth_pk;
 
-  /* (Only client rendezvous circuit) Introduction point encryption public
+  /** (Only client rendezvous circuit) Introduction point encryption public
    * key. We keep it in the rendezvous identifier for the ntor handshake. */
   curve25519_public_key_t intro_enc_pk;
 
-  /* (Only rendezvous circuit) Rendezvous cookie sent from the client to the
+  /** (Only rendezvous circuit) Rendezvous cookie sent from the client to the
    * service with an INTRODUCE1 cell and used by the service in an
    * RENDEZVOUS1 cell. */
   uint8_t rendezvous_cookie[HS_REND_COOKIE_LEN];
 
-  /* (Only service rendezvous circuit) The HANDSHAKE_INFO needed in the
+  /** (Only service rendezvous circuit) The HANDSHAKE_INFO needed in the
    * RENDEZVOUS1 cell of the service. The construction is as follows:
-   *    SERVER_PK   [32 bytes]
-   *    AUTH_MAC    [32 bytes]
+   *
+   *      SERVER_PK   [32 bytes]
+   *      AUTH_MAC    [32 bytes]
    */
   uint8_t rendezvous_handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
 
-  /* (Only client rendezvous circuit) Client ephemeral keypair needed for the
+  /** (Only client rendezvous circuit) Client ephemeral keypair needed for the
    * e2e encryption with the service. */
   curve25519_keypair_t rendezvous_client_kp;
 
-  /* (Only rendezvous circuit) The NTOR_KEY_SEED needed for key derivation for
+  /** (Only rendezvous circuit) The NTOR_KEY_SEED needed for key derivation for
    * the e2e encryption with the client on the circuit. */
   uint8_t rendezvous_ntor_key_seed[DIGEST256_LEN];
 
-  /* (Only rendezvous circuit) Number of streams associated with this
+  /** (Only rendezvous circuit) Number of streams associated with this
    * rendezvous circuit. We track this because there is a check on a maximum
    * value. */
   uint64_t num_rdv_streams;
 } hs_ident_circuit_t;
 
-/* Client and service side directory connection identifier used for a
+/** Client and service side directory connection identifier used for a
  * directory connection to identify which service is being queried. This is
  * attached to a dir_connection_t. */
 typedef struct hs_ident_dir_conn_t {
-  /* The public key used to uniquely identify the service. It is the one found
+  /** The public key used to uniquely identify the service. It is the one found
    * in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* The blinded public key used to uniquely identify the descriptor that this
+  /** The blinded public key used to uniquely identify the descriptor that this
    * directory connection identifier is for. Only used by the service-side code
    * to fine control descriptor uploads. */
   ed25519_public_key_t blinded_pk;
@@ -103,15 +97,15 @@ typedef struct hs_ident_dir_conn_t {
   /* XXX: Client authorization. */
 } hs_ident_dir_conn_t;
 
-/* Client and service side edge connection identifier used for an edge
+/** Client and service side edge connection identifier used for an edge
  * connection to identify which service is being queried. This is attached to
  * a edge_connection_t. */
 typedef struct hs_ident_edge_conn_t {
-  /* The public key used to uniquely identify the service. It is the one found
+  /** The public key used to uniquely identify the service. It is the one found
    * in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* The original virtual port that was used by the client to access the onion
+  /** The original virtual port that was used by the client to access the onion
    * service, regardless of the internal port forwarding that might have
    * happened on the service-side. */
   uint16_t orig_virtual_port;
@@ -120,8 +114,7 @@ typedef struct hs_ident_edge_conn_t {
 
 /* Circuit identifier API. */
 hs_ident_circuit_t *hs_ident_circuit_new(
-                             const ed25519_public_key_t *identity_pk,
-                             hs_ident_circuit_type_t circuit_type);
+                             const ed25519_public_key_t *identity_pk);
 void hs_ident_circuit_free_(hs_ident_circuit_t *ident);
 #define hs_ident_circuit_free(id) \
   FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id))
@@ -147,4 +140,3 @@ void hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident);
 int hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident);
 
 #endif /* !defined(TOR_HS_IDENT_H) */
-
diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c
index 7717ed53d4..69d60f21c3 100644
--- a/src/feature/hs/hs_intropoint.c
+++ b/src/feature/hs/hs_intropoint.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,6 +10,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/relay.h"
@@ -24,9 +25,11 @@
 #include "trunnel/hs/cell_introduce1.h"
 
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
 #include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
-#include "feature/hs/hs_common.h"
 
 #include "core/or/or_circuit_st.h"
 
@@ -144,7 +147,7 @@ verify_establish_intro_cell(const trn_cell_establish_intro_t *cell,
   return 0;
 }
 
-/* Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
+/** Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
 MOCK_IMPL(int,
 hs_intro_send_intro_established_cell,(or_circuit_t *circ))
 {
@@ -179,6 +182,190 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ))
   return ret;
 }
 
+/** Validate the cell DoS extension parameters. Return true iff they've been
+ * bound check and can be used. Else return false. See proposal 305 for
+ * details and reasons about this validation. */
+STATIC bool
+cell_dos_extension_parameters_are_valid(uint64_t intro2_rate_per_sec,
+                                        uint64_t intro2_burst_per_sec)
+{
+  bool ret = false;
+
+  /* Check that received value is not below the minimum. Don't check if minimum
+     is set to 0, since the param is a positive value and gcc will complain. */
+#if HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0
+  if (intro2_rate_per_sec < HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too small. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_rate_per_sec > HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too big. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+
+  /* Check that received value is not below the minimum */
+#if HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0
+  if (intro2_burst_per_sec < HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too small. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_burst_per_sec > HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too big. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* In a rate limiting scenario, burst can never be smaller than the rate. At
+   * best it can be equal. */
+  if (intro2_burst_per_sec < intro2_rate_per_sec) {
+    log_info(LD_REND, "Intro point DoS defenses burst is smaller than rate. "
+                      "Rate: %" PRIu64 " vs Burst: %" PRIu64,
+             intro2_rate_per_sec, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* Passing validation. */
+  ret = true;
+
+ end:
+  return ret;
+}
+
+/** Parse the cell DoS extension and apply defenses on the given circuit if
+ * validation passes. If the cell extension is malformed or contains unusable
+ * values, the DoS defenses is disabled on the circuit. */
+static void
+handle_establish_intro_cell_dos_extension(
+                                const trn_cell_extension_field_t *field,
+                                or_circuit_t *circ)
+{
+  ssize_t ret;
+  uint64_t intro2_rate_per_sec = 0, intro2_burst_per_sec = 0;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  tor_assert(field);
+  tor_assert(circ);
+
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  if (ret < 0) {
+    goto end;
+  }
+
+  for (size_t i = 0; i < trn_cell_extension_dos_get_n_params(dos); i++) {
+    const trn_cell_extension_dos_param_t *param =
+      trn_cell_extension_dos_getconst_params(dos, i);
+    if (BUG(param == NULL)) {
+      goto end;
+    }
+
+    switch (trn_cell_extension_dos_param_get_type(param)) {
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC:
+      intro2_rate_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC:
+      intro2_burst_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    default:
+      goto end;
+    }
+  }
+
+  /* At this point, the extension is valid so any values out of it implies
+   * that it was set explicitly and thus flag the circuit that it should not
+   * look at the consensus for that reason for the defenses' values. */
+  circ->introduce2_dos_defense_explicit = 1;
+
+  /* A value of 0 is valid in the sense that we accept it but we still disable
+   * the defenses so return false. */
+  if (intro2_rate_per_sec == 0 || intro2_burst_per_sec == 0) {
+    log_info(LD_REND, "Intro point DoS defenses parameter set to 0. "
+                      "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    circ->introduce2_dos_defense_enabled = 0;
+    goto end;
+  }
+
+  /* If invalid, we disable the defense on the circuit. */
+  if (!cell_dos_extension_parameters_are_valid(intro2_rate_per_sec,
+                                               intro2_burst_per_sec)) {
+    circ->introduce2_dos_defense_enabled = 0;
+    log_info(LD_REND, "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    goto end;
+  }
+
+  /* We passed validation, enable defenses and apply rate/burst. */
+  circ->introduce2_dos_defense_enabled = 1;
+
+  /* Initialize the INTRODUCE2 token bucket for the rate limiting. */
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        (uint32_t) intro2_rate_per_sec,
+                        (uint32_t) intro2_burst_per_sec,
+                        (uint32_t) approx_time());
+  log_info(LD_REND, "Intro point DoS defenses enabled. Rate is %" PRIu64
+                    " and Burst is %" PRIu64,
+           intro2_rate_per_sec, intro2_burst_per_sec);
+
+ end:
+  trn_cell_extension_dos_free(dos);
+  return;
+}
+
+/** Parse every cell extension in the given ESTABLISH_INTRO cell. */
+static void
+handle_establish_intro_cell_extensions(
+                            const trn_cell_establish_intro_t *parsed_cell,
+                            or_circuit_t *circ)
+{
+  const trn_cell_extension_t *extensions;
+
+  tor_assert(parsed_cell);
+  tor_assert(circ);
+
+  extensions = trn_cell_establish_intro_getconst_extensions(parsed_cell);
+  if (extensions == NULL) {
+    goto end;
+  }
+
+  /* Go over all extensions. */
+  for (size_t idx = 0; idx < trn_cell_extension_get_num(extensions); idx++) {
+    const trn_cell_extension_field_t *field =
+      trn_cell_extension_getconst_fields(extensions, idx);
+    if (BUG(field == NULL)) {
+      /* The number of extensions should match the number of fields. */
+      break;
+    }
+
+    switch (trn_cell_extension_field_get_field_type(field)) {
+    case TRUNNEL_CELL_EXTENSION_TYPE_DOS:
+      /* After this, the circuit should be set for DoS defenses. */
+      handle_establish_intro_cell_dos_extension(field, circ);
+      break;
+    default:
+      /* Unknown extension. Skip over. */
+      break;
+    }
+  }
+
+ end:
+  return;
+}
+
 /** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
  *  well-formed and passed our verifications. Perform appropriate actions to
  *  establish an intro point. */
@@ -191,6 +378,13 @@ handle_verified_establish_intro_cell(or_circuit_t *circ,
   get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
                          parsed_cell);
 
+  /* Setup INTRODUCE2 defenses on the circuit. Must be done before parsing the
+   * cell extension that can possibly change the defenses' values. */
+  hs_dos_setup_default_intro2_defenses(circ);
+
+  /* Handle cell extension if any. */
+  handle_establish_intro_cell_extensions(parsed_cell, circ);
+
   /* Then notify the hidden service that the intro point is established by
      sending an INTRO_ESTABLISHED cell */
   if (hs_intro_send_intro_established_cell(circ)) {
@@ -268,7 +462,7 @@ handle_establish_intro(or_circuit_t *circ, const uint8_t *request,
   return retval;
 }
 
-/* Return True if circuit is suitable for being an intro circuit. */
+/** Return True if circuit is suitable for being an intro circuit. */
 static int
 circuit_is_suitable_intro_point(const or_circuit_t *circ,
                                 const char *log_cell_type_str)
@@ -293,14 +487,14 @@ circuit_is_suitable_intro_point(const or_circuit_t *circ,
   return 1;
 }
 
-/* Return True if circuit is suitable for being service-side intro circuit. */
+/** Return True if circuit is suitable for being service-side intro circuit. */
 int
 hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ)
 {
   return circuit_is_suitable_intro_point(circ, "ESTABLISH_INTRO");
 }
 
-/* We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
+/** We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
  * a legacy or a next gen cell, and pass it to the appropriate handler. */
 int
 hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
@@ -334,7 +528,7 @@ hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
   return -1;
 }
 
-/* Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
+/** Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
  * value in <b>status</b>. Depending on the status, it can be ACK or a NACK.
  * Return 0 on success else a negative value on error which will close the
  * circuit. */
@@ -378,7 +572,7 @@ send_introduce_ack_cell(or_circuit_t *circ, uint16_t status)
   return ret;
 }
 
-/* Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
+/** Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
  * negative value for an invalid cell that should be NACKed. */
 STATIC int
 validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
@@ -392,7 +586,7 @@ validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
    * safety net here. The legacy ID must be zeroes in this case. */
   legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
   legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
-  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+  if (BUG(!fast_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
     goto invalid;
   }
 
@@ -424,7 +618,7 @@ validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
   return -1;
 }
 
-/* We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
+/** We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
  * the payload in <b>request</b> of size <b>request_len</b>. Return 0 if
  * everything went well, or -1 if an error occurred. This function is in charge
  * of sending back an INTRODUCE_ACK cell and will close client_circ on error.
@@ -480,6 +674,20 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
     }
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(service_circ)) {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(5 * 60);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v3 cell due to DoS "
+                            "limitations. Sending NACK to client.");
+      tor_free(msg);
+    }
+    status = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+    goto send_ack;
+  }
+
   /* Relay the cell to the service on its intro circuit with an INTRODUCE2
    * cell which is the same exact payload. */
   if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
@@ -509,7 +717,7 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
   return ret;
 }
 
-/* Identify if the encoded cell we just received is a legacy one or not. The
+/** Identify if the encoded cell we just received is a legacy one or not. The
  * <b>request</b> should be at least DIGEST_LEN bytes long. */
 STATIC int
 introduce1_cell_is_legacy(const uint8_t *request)
@@ -518,7 +726,7 @@ introduce1_cell_is_legacy(const uint8_t *request)
 
   /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
    * indicates a legacy cell (v2). */
-  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+  if (!fast_mem_is_zero((const char *) request, DIGEST_LEN)) {
     /* Legacy cell. */
     return 1;
   }
@@ -526,7 +734,7 @@ introduce1_cell_is_legacy(const uint8_t *request)
   return 0;
 }
 
-/* Return true iff the circuit <b>circ</b> is suitable for receiving an
+/** Return true iff the circuit <b>circ</b> is suitable for receiving an
  * INTRODUCE1 cell. */
 STATIC int
 circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
@@ -546,10 +754,18 @@ circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
     return 0;
   }
 
+  /* Disallow single hop client circuit. */
+  if (circ->p_chan && channel_is_client(circ->p_chan)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Single hop client was rejected while trying to introduce. "
+           "Closing circuit.");
+    return 0;
+  }
+
   return 1;
 }
 
-/* We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
+/** We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
  * it is and pass it to the appropriate handler. Return 0 on success else a
  * negative value and the circuit is closed. */
 int
@@ -593,8 +809,8 @@ hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
   return -1;
 }
 
-/* Clear memory allocated by the given intropoint object ip (but don't free the
- * object itself). */
+/** Clear memory allocated by the given intropoint object ip (but don't free
+ * the object itself). */
 void
 hs_intropoint_clear(hs_intropoint_t *ip)
 {
@@ -602,8 +818,8 @@ hs_intropoint_clear(hs_intropoint_t *ip)
     return;
   }
   tor_cert_free(ip->auth_key_cert);
-  SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls,
-                    hs_desc_link_specifier_free(ls));
+  SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *, ls,
+                    link_specifier_free(ls));
   smartlist_free(ip->link_specifiers);
   memset(ip, 0, sizeof(hs_intropoint_t));
 }
diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h
index e82575f052..8b2b9892b3 100644
--- a/src/feature/hs/hs_intropoint.h
+++ b/src/feature/hs/hs_intropoint.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,15 +12,15 @@
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "feature/nodelist/torcert.h"
 
-/* Object containing introduction point common data between the service and
+/** Object containing introduction point common data between the service and
  * the client side. */
 typedef struct hs_intropoint_t {
-  /* Does this intro point only supports legacy ID ?. */
+  /** Does this intro point only supports legacy ID ?. */
   unsigned int is_only_legacy : 1;
 
-  /* Authentication key certificate from the descriptor. */
+  /** Authentication key certificate from the descriptor. */
   tor_cert_t *auth_key_cert;
-  /* A list of link specifier. */
+  /** A list of link specifier. */
   smartlist_t *link_specifiers;
 } hs_intropoint_t;
 
@@ -57,8 +57,10 @@ STATIC int handle_introduce1(or_circuit_t *client_circ,
                              const uint8_t *request, size_t request_len);
 STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
 STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+STATIC bool cell_dos_extension_parameters_are_valid(
+                                          uint64_t intro2_rate_per_sec,
+                                          uint64_t intro2_burst_per_sec);
 
 #endif /* defined(HS_INTROPOINT_PRIVATE) */
 
 #endif /* !defined(TOR_HS_INTRO_H) */
-
diff --git a/src/feature/hs/hs_ob.c b/src/feature/hs/hs_ob.c
new file mode 100644
index 0000000000..9499c28d20
--- /dev/null
+++ b/src/feature/hs/hs_ob.c
@@ -0,0 +1,408 @@
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ob.c
+ * \brief Implement Onion Balance specific code.
+ **/
+
+#define HS_OB_PRIVATE
+
+#include "feature/hs/hs_service.h"
+
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "lib/confmgt/confmgt.h"
+#include "lib/encoding/confline.h"
+
+#include "feature/hs/hs_ob.h"
+
+/* Options config magic number. */
+#define OB_OPTIONS_MAGIC 0x631DE7EA
+
+/* Helper macros. */
+#define VAR(varname, conftype, member, initvalue)                          \
+  CONFIG_VAR_ETYPE(ob_options_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)        \
+  VAR(#member, conftype, member, initvalue)
+
+/* Dummy instance of ob_options_t, used for type-checking its members with
+ * CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(ob_options_t);
+
+/* Array of variables for the config file options. */
+static const config_var_t config_vars[] = {
+  V(MasterOnionAddress, LINELIST, NULL),
+
+  END_OF_CONFIG_VARS
+};
+
+/* "Extra" variable in the state that receives lines we can't parse. This
+ * lets us preserve options from versions of Tor newer than us. */
+static const struct_member_t config_extra_vars = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(ob_options_t, ExtraLines),
+};
+
+/* Configuration format of ob_options_t. */
+static const config_format_t config_format = {
+  .size = sizeof(ob_options_t),
+  .magic = {
+     "ob_options_t",
+     OB_OPTIONS_MAGIC,
+     offsetof(ob_options_t, magic_),
+  },
+  .vars = config_vars,
+  .extra = &config_extra_vars,
+};
+
+/* Global configuration manager for the config file. */
+static config_mgr_t *config_options_mgr = NULL;
+
+/* Return the configuration manager for the config file. */
+static const config_mgr_t *
+get_config_options_mgr(void)
+{
+  if (PREDICT_UNLIKELY(config_options_mgr == NULL)) {
+    config_options_mgr = config_mgr_new(&config_format);
+    config_mgr_freeze(config_options_mgr);
+  }
+  return config_options_mgr;
+}
+
+#define ob_option_free(val) \
+  FREE_AND_NULL(ob_options_t, ob_option_free_, (val))
+
+/** Helper: Free a config options object. */
+static void
+ob_option_free_(ob_options_t *opts)
+{
+  if (opts == NULL) {
+    return;
+  }
+  config_free(get_config_options_mgr(), opts);
+}
+
+/** Return an allocated config options object. */
+static ob_options_t *
+ob_option_new(void)
+{
+  ob_options_t *opts = config_new(get_config_options_mgr());
+  config_init(get_config_options_mgr(), opts);
+  return opts;
+}
+
+/** Helper function: From the configuration line value which is an onion
+ * address with the ".onion" extension, find the public key and put it in
+ * pkey_out.
+ *
+ * On success, true is returned. Else, false and pkey is untouched. */
+static bool
+get_onion_public_key(const char *value, ed25519_public_key_t *pkey_out)
+{
+  char address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(value);
+  tor_assert(pkey_out);
+
+  if (strcmpend(value, ".onion")) {
+    /* Not a .onion extension, bad format. */
+    return false;
+  }
+
+  /* Length validation. The -1 is because sizeof() counts the NUL byte. */
+  if (strlen(value) >
+      (HS_SERVICE_ADDR_LEN_BASE32 + sizeof(".onion") - 1)) {
+    /* Too long, bad format. */
+    return false;
+  }
+
+  /* We don't want the .onion so we add 2 because size - 1 is copied with
+   * strlcpy() in order to accomodate the NUL byte and sizeof() counts the NUL
+   * byte so we need to remove them from the equation. */
+  strlcpy(address, value, strlen(value) - sizeof(".onion") + 2);
+
+  if (hs_parse_address_no_log(address, pkey_out, NULL, NULL, NULL) < 0) {
+    return false;
+  }
+
+  /* Success. */
+  return true;
+}
+
+/** Parse the given ob options in opts and set the service config object
+ * accordingly.
+ *
+ * Return 1 on success else 0. */
+static int
+ob_option_parse(hs_service_config_t *config, const ob_options_t *opts)
+{
+  int ret = 0;
+  config_line_t *line;
+
+  tor_assert(config);
+  tor_assert(opts);
+
+  for (line = opts->MasterOnionAddress; line; line = line->next) {
+    /* Allocate config list if need be. */
+    if (!config->ob_master_pubkeys) {
+      config->ob_master_pubkeys = smartlist_new();
+    }
+    ed25519_public_key_t *pubkey = tor_malloc_zero(sizeof(*pubkey));
+
+    if (!get_onion_public_key(line->value, pubkey)) {
+      log_warn(LD_REND, "OnionBalance: MasterOnionAddress %s is invalid",
+               line->value);
+      tor_free(pubkey);
+      goto end;
+    }
+    smartlist_add(config->ob_master_pubkeys, pubkey);
+    log_notice(LD_REND, "OnionBalance: MasterOnionAddress %s registered",
+               line->value);
+  }
+  /* Success. */
+  ret = 1;
+
+ end:
+  /* No keys added, we free the list since no list means no onion balance
+   * support for this tor instance. */
+  if (smartlist_len(config->ob_master_pubkeys) == 0) {
+    smartlist_free(config->ob_master_pubkeys);
+  }
+  return ret;
+}
+
+/** For the given master public key and time period, compute the subcredential
+ * and put them into subcredential. The subcredential parameter needs to be at
+ * least DIGEST256_LEN in size. */
+static void
+build_subcredential(const ed25519_public_key_t *pkey, uint64_t tp,
+                    hs_subcredential_t *subcredential)
+{
+  ed25519_public_key_t blinded_pubkey;
+
+  tor_assert(pkey);
+  tor_assert(subcredential);
+
+  hs_build_blinded_pubkey(pkey, NULL, 0, tp, &blinded_pubkey);
+  hs_get_subcredential(pkey, &blinded_pubkey, subcredential);
+}
+
+/*
+ * Public API.
+ */
+
+/** Return true iff the given service is configured as an onion balance
+ * instance. To satisfy that condition, there must at least be one master
+ * ed25519 public key configured. */
+bool
+hs_ob_service_is_instance(const hs_service_t *service)
+{
+  if (BUG(service == NULL)) {
+    return false;
+  }
+
+  /* No list, we are not an instance. */
+  if (!service->config.ob_master_pubkeys) {
+    return false;
+  }
+
+  return smartlist_len(service->config.ob_master_pubkeys) > 0;
+}
+
+/** Read and parse the config file at fname on disk. The service config object
+ * is populated with the options if any.
+ *
+ * Return 1 on success else 0. This is to follow the "ok" convention in
+ * hs_config.c. */
+int
+hs_ob_parse_config_file(hs_service_config_t *config)
+{
+  static const char *fname = "ob_config";
+  int ret = 0;
+  char *content = NULL, *errmsg = NULL, *config_file_path = NULL;
+  ob_options_t *options = NULL;
+  config_line_t *lines = NULL;
+
+  tor_assert(config);
+
+  /* Read file from disk. */
+  config_file_path = hs_path_from_filename(config->directory_path, fname);
+  content = read_file_to_str(config_file_path, 0, NULL);
+  if (!content) {
+    log_warn(LD_FS, "OnionBalance: Unable to read config file %s",
+             escaped(config_file_path));
+    goto end;
+  }
+
+  /* Parse lines. */
+  if (config_get_lines(content, &lines, 0) < 0) {
+    goto end;
+  }
+
+  options = ob_option_new();
+  config_assign(get_config_options_mgr(), options, lines, 0, &errmsg);
+  if (errmsg) {
+    log_warn(LD_REND, "OnionBalance: Unable to parse config file: %s",
+             errmsg);
+    tor_free(errmsg);
+    goto end;
+  }
+
+  /* Parse the options and set the service config object with the details. */
+  ret = ob_option_parse(config, options);
+
+ end:
+  config_free_lines(lines);
+  ob_option_free(options);
+  tor_free(content);
+  tor_free(config_file_path);
+  return ret;
+}
+
+/** Compute all possible subcredentials for every onion master key in the given
+ * service config object. subcredentials_out is allocated and set as an
+ * continous array containing all possible values.
+ *
+ * On success, return the number of subcredential put in the array which will
+ * correspond to an arry of size: n * DIGEST256_LEN where DIGEST256_LEN is the
+ * length of a single subcredential.
+ *
+ * If the given configuration object has no OB master keys configured, 0 is
+ * returned and subcredentials_out is set to NULL.
+ *
+ * Otherwise, this can't fail. */
+STATIC size_t
+compute_subcredentials(const hs_service_t *service,
+                       hs_subcredential_t **subcredentials_out)
+{
+  unsigned int num_pkeys, idx = 0;
+  hs_subcredential_t *subcreds = NULL;
+  const int steps[3] = {0, -1, 1};
+  const unsigned int num_steps = ARRAY_LENGTH(steps);
+  const uint64_t tp = hs_get_time_period_num(0);
+
+  tor_assert(service);
+  tor_assert(subcredentials_out);
+  /* Our caller has checked these too */
+  tor_assert(service->desc_current);
+  tor_assert(service->desc_next);
+
+  /* Make sure we are an OB instance, or bail out. */
+  num_pkeys = smartlist_len(service->config.ob_master_pubkeys);
+  if (!num_pkeys) {
+    *subcredentials_out = NULL;
+    return 0;
+  }
+
+  /* Time to build all the subcredentials for each time period: two for each
+   * instance descriptor plus three for the onionbalance frontend service: the
+   * previous one (-1), the current one (0) and the next one (1) for each
+   * configured key in order to accomodate client and service consensus skew.
+   *
+   * If the client consensus after_time is at 23:00 but the service one is at
+   * 01:00, the client will be using the previous time period where the
+   * service will think it is the client next time period. Thus why we have
+   * to try them all.
+   *
+   * The normal use case works because the service gets the descriptor object
+   * that corresponds to the intro point's request, and because each
+   * descriptor corresponds to a specific subcredential, we get the right
+   * subcredential out of it, and use that to do the decryption.
+   *
+   * As a slight optimization, statistically, the current time period (0) will
+   * be the one to work first so we'll put them first in the array to maximize
+   * our chance of success. */
+
+  /* We use a flat array, not a smartlist_t, in order to minimize memory
+   * allocation.
+   *
+   * Size of array is: length of a single subcredential multiplied by the
+   * number of time period we need to compute and finally multiplied by the
+   * total number of keys we are about to process. In other words, for each
+   * key, we allocate 3 subcredential slots. Then in the end we also add two
+   * subcredentials for this instance's active descriptors. */
+  subcreds =
+    tor_calloc((num_steps * num_pkeys) + 2, sizeof(hs_subcredential_t));
+
+  /* For each master pubkey we add 3 subcredentials: */
+  for (unsigned int i = 0; i < num_steps; i++) {
+    SMARTLIST_FOREACH_BEGIN(service->config.ob_master_pubkeys,
+                            const ed25519_public_key_t *, pkey) {
+      build_subcredential(pkey, tp + steps[i], &subcreds[idx]);
+      idx++;
+    } SMARTLIST_FOREACH_END(pkey);
+  }
+
+  /* And then in the end we add the two subcredentials of the current active
+   * instance descriptors */
+  memcpy(&subcreds[idx++], &service->desc_current->desc->subcredential,
+         sizeof(hs_subcredential_t));
+  memcpy(&subcreds[idx++], &service->desc_next->desc->subcredential,
+         sizeof(hs_subcredential_t));
+
+  log_info(LD_REND, "Refreshing %u onionbalance keys (TP #%d).",
+             idx, (int)tp);
+
+  *subcredentials_out = subcreds;
+  return idx;
+}
+
+/**
+ *  If we are an Onionbalance instance, refresh our keys.
+ *
+ *  If we are not an Onionbalance instance or we are not ready to do so, this
+ *  is a NOP.
+ *
+ *  This function is called everytime we build a new descriptor. That's because
+ *  we want our Onionbalance keys to always use up-to-date subcredentials both
+ *  for the instance (ourselves) and for the onionbalance frontend.
+ */
+void
+hs_ob_refresh_keys(hs_service_t *service)
+{
+  hs_subcredential_t *ob_subcreds = NULL;
+  size_t num_subcreds;
+
+  tor_assert(service);
+
+  /* Don't do any of this if we are not configured as an OB instance */
+  if (!hs_ob_service_is_instance(service)) {
+    return;
+  }
+
+  /* We need both service descriptors created to make onionbalance keys.
+   *
+   * That's because we fetch our own (the instance's) subcredentials from our
+   * own descriptors which should always include the latest subcredentials that
+   * clients would use.
+   *
+   * This function is called with each descriptor build, so we will be
+   * eventually be called when both descriptors are created. */
+  if (!service->desc_current || !service->desc_next) {
+    return;
+  }
+
+  /* Get a new set of subcreds */
+  num_subcreds = compute_subcredentials(service, &ob_subcreds);
+  if (BUG(!num_subcreds)) {
+    return;
+  }
+
+  /* Delete old subcredentials if any */
+  if (service->state.ob_subcreds) {
+    tor_free(service->state.ob_subcreds);
+  }
+
+  service->state.ob_subcreds = ob_subcreds;
+  service->state.n_ob_subcreds = num_subcreds;
+}
+
+/** Free any memory allocated by the onionblance subsystem. */
+void
+hs_ob_free_all(void)
+{
+  config_mgr_free(config_options_mgr);
+}
diff --git a/src/feature/hs/hs_ob.h b/src/feature/hs/hs_ob.h
new file mode 100644
index 0000000000..d6e6e73a84
--- /dev/null
+++ b/src/feature/hs/hs_ob.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ob.h
+ * \brief Header file for the specific code for onion balance.
+ **/
+
+#ifndef TOR_HS_OB_H
+#define TOR_HS_OB_H
+
+#include "feature/hs/hs_service.h"
+
+bool hs_ob_service_is_instance(const hs_service_t *service);
+
+int hs_ob_parse_config_file(hs_service_config_t *config);
+
+struct hs_subcredential_t;
+
+void hs_ob_free_all(void);
+
+void hs_ob_refresh_keys(hs_service_t *service);
+
+#ifdef HS_OB_PRIVATE
+
+STATIC size_t compute_subcredentials(const hs_service_t *service,
+                                   struct hs_subcredential_t **subcredentials);
+
+typedef struct ob_options_t {
+  /** Magic number to identify the structure in memory. */
+  uint32_t magic_;
+  /** Master Onion Address(es). */
+  struct config_line_t *MasterOnionAddress;
+  /** Extra Lines for configuration we might not know. */
+  struct config_line_t *ExtraLines;
+} ob_options_t;
+
+#endif /* defined(HS_OB_PRIVATE) */
+
+#endif /* !defined(TOR_HS_OB_H) */
diff --git a/src/feature/hs/hs_options.inc b/src/feature/hs/hs_options.inc
new file mode 100644
index 0000000000..1a1444fd05
--- /dev/null
+++ b/src/feature/hs/hs_options.inc
@@ -0,0 +1,36 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options.inc
+ * @brief Declare configuration options for a single hidden service.
+ *
+ * Note that this options file behaves differently from most, since it
+ * is not used directly by the options manager.  Instead, it is applied to
+ * a group of hidden service options starting with a HiddenServiceDir and
+ * extending up to the next HiddenServiceDir.
+ **/
+
+/** Holds configuration for a single hidden service. */
+BEGIN_CONF_STRUCT(hs_opts_t)
+
+CONF_VAR(HiddenServiceDir, FILENAME, 0, NULL)
+CONF_VAR(HiddenServiceDirGroupReadable, BOOL, 0, "0")
+CONF_VAR(HiddenServicePort, LINELIST, 0, NULL)
+// "-1" means "auto" here.
+CONF_VAR(HiddenServiceVersion, INT, 0, "-1")
+CONF_VAR(HiddenServiceAuthorizeClient, STRING, 0, NULL)
+CONF_VAR(HiddenServiceAllowUnknownPorts, BOOL, 0, "0")
+CONF_VAR(HiddenServiceMaxStreams, POSINT, 0, "0")
+CONF_VAR(HiddenServiceMaxStreamsCloseCircuit, BOOL, 0, "0")
+CONF_VAR(HiddenServiceNumIntroductionPoints, POSINT, 0, "3")
+CONF_VAR(HiddenServiceExportCircuitID, STRING, 0, NULL)
+CONF_VAR(HiddenServiceEnableIntroDoSDefense, BOOL, 0, "0")
+CONF_VAR(HiddenServiceEnableIntroDoSRatePerSec, POSINT, 0, "25")
+CONF_VAR(HiddenServiceEnableIntroDoSBurstPerSec, POSINT, 0, "200")
+CONF_VAR(HiddenServiceOnionBalanceInstance, BOOL, 0, "0")
+
+END_CONF_STRUCT(hs_opts_t)
diff --git a/src/feature/hs/hs_opts_st.h b/src/feature/hs/hs_opts_st.h
new file mode 100644
index 0000000000..279f0d6da6
--- /dev/null
+++ b/src/feature/hs/hs_opts_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options_st.h
+ * @brief Structure hs_opts_t to hold options for a single hidden service.
+ **/
+
+#ifndef TOR_FEATURE_HS_HS_OPTS_ST_H
+#define TOR_FEATURE_HS_HS_OPTS_ST_H
+
+#include "lib/conf/confdecl.h"
+#define CONF_CONTEXT STRUCT
+#include "feature/hs/hs_options.inc"
+#undef CONF_CONTEXT
+
+/**
+ * An hs_opts_t holds the parsed options for a single HS configuration
+ * section.
+ *
+ * This name ends with 'opts' instead of 'options' to signal that it is not
+ * handled directly by the or_options_t configuration manager, but that
+ * first we partition the "HiddenService*" options by section.
+ **/
+typedef struct hs_opts_t hs_opts_t;
+
+#endif /* !defined(TOR_FEATURE_HS_HS_OPTS_ST_H) */
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
index e820ce9d0b..c29f39c6b4 100644
--- a/src/feature/hs/hs_service.c
+++ b/src/feature/hs/hs_service.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,7 +31,6 @@
 #include "feature/rend/rendservice.h"
 #include "lib/crypt_ops/crypto_ope.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
 #include "feature/hs/hs_circuit.h"
@@ -43,6 +42,7 @@
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 #include "feature/hs/hs_stats.h"
+#include "feature/hs/hs_ob.h"
 
 #include "feature/dircommon/dir_connection_st.h"
 #include "core/or/edge_connection_st.h"
@@ -68,7 +68,8 @@
 #include <unistd.h>
 #endif
 
-/* Helper macro. Iterate over every service in the global map. The var is the
+#ifndef COCCI
+/** Helper macro. Iterate over every service in the global map. The var is the
  * name of the service pointer. */
 #define FOR_EACH_SERVICE_BEGIN(var)                          \
     STMT_BEGIN                                               \
@@ -77,7 +78,7 @@
       var = *var##_iter;
 #define FOR_EACH_SERVICE_END } STMT_END ;
 
-/* Helper macro. Iterate over both current and previous descriptor of a
+/** Helper macro. Iterate over both current and previous descriptor of a
  * service. The var is the name of the descriptor pointer. This macro skips
  * any descriptor object of the service that is NULL. */
 #define FOR_EACH_DESCRIPTOR_BEGIN(service, var)                  \
@@ -89,6 +90,7 @@
                                 (var = service->desc_next);      \
       if (var == NULL) continue;
 #define FOR_EACH_DESCRIPTOR_END } STMT_END ;
+#endif /* !defined(COCCI) */
 
 /* Onion service directory file names. */
 static const char fname_keyfile_prefix[] = "hs_ed25519";
@@ -96,7 +98,7 @@ static const char dname_client_pubkeys[] = "authorized_clients";
 static const char fname_hostname[] = "hostname";
 static const char address_tld[] = "onion";
 
-/* Staging list of service object. When configuring service, we add them to
+/** Staging list of service object. When configuring service, we add them to
  * this list considered a staging area and they will get added to our global
  * map once the keys have been loaded. These two steps are separated because
  * loading keys requires that we are an actual running tor process. */
@@ -119,7 +121,7 @@ static int service_encode_descriptor(const hs_service_t *service,
                                      const ed25519_keypair_t *signing_kp,
                                      char **encoded_out);
 
-/* Helper: Function to compare two objects in the service map. Return 1 if the
+/** Helper: Function to compare two objects in the service map. Return 1 if the
  * two service have the same master public identity key. */
 static inline int
 hs_service_ht_eq(const hs_service_t *first, const hs_service_t *second)
@@ -131,7 +133,7 @@ hs_service_ht_eq(const hs_service_t *first, const hs_service_t *second)
                            &second->keys.identity_pk);
 }
 
-/* Helper: Function for the service hash table code below. The key used is the
+/** Helper: Function for the service hash table code below. The key used is the
  * master public identity key which is ultimately the onion address. */
 static inline unsigned int
 hs_service_ht_hash(const hs_service_t *service)
@@ -141,7 +143,7 @@ hs_service_ht_hash(const hs_service_t *service)
                                    sizeof(service->keys.identity_pk.pubkey));
 }
 
-/* This is _the_ global hash map of hidden services which indexed the service
+/** This is _the_ global hash map of hidden services which indexed the service
  * contained in it by master public identity key which is roughly the onion
  * address of the service. */
 static struct hs_service_ht *hs_service_map;
@@ -151,13 +153,13 @@ HT_PROTOTYPE(hs_service_ht,      /* Name of hashtable. */
              hs_service_t,       /* Object contained in the map. */
              hs_service_node,    /* The name of the HT_ENTRY member. */
              hs_service_ht_hash, /* Hashing function. */
-             hs_service_ht_eq)   /* Compare function for objects. */
+             hs_service_ht_eq);  /* Compare function for objects. */
 
 HT_GENERATE2(hs_service_ht, hs_service_t, hs_service_node,
              hs_service_ht_hash, hs_service_ht_eq,
-             0.6, tor_reallocarray, tor_free_)
+             0.6, tor_reallocarray, tor_free_);
 
-/* Query the given service map with a public key and return a service object
+/** Query the given service map with a public key and return a service object
  * if found else NULL. It is also possible to set a directory path in the
  * search query. If pk is NULL, then it will be set to zero indicating the
  * hash table to compare the directory path instead. */
@@ -172,7 +174,7 @@ find_service(hs_service_ht *map, const ed25519_public_key_t *pk)
   return HT_FIND(hs_service_ht, map, &dummy_service);
 }
 
-/* Register the given service in the given map. If the service already exists
+/** Register the given service in the given map. If the service already exists
  * in the map, -1 is returned. On success, 0 is returned and the service
  * ownership has been transferred to the global map. */
 STATIC int
@@ -197,7 +199,7 @@ register_service(hs_service_ht *map, hs_service_t *service)
   return 0;
 }
 
-/* Remove a given service from the given map. If service is NULL or the
+/** Remove a given service from the given map. If service is NULL or the
  * service key is unset, return gracefully. */
 STATIC void
 remove_service(hs_service_ht *map, hs_service_t *service)
@@ -227,7 +229,7 @@ remove_service(hs_service_ht *map, hs_service_t *service)
   }
 }
 
-/* Set the default values for a service configuration object <b>c</b>. */
+/** Set the default values for a service configuration object <b>c</b>. */
 static void
 set_service_default_config(hs_service_config_t *c,
                            const or_options_t *options)
@@ -243,9 +245,12 @@ set_service_default_config(hs_service_config_t *c,
   c->is_single_onion = 0;
   c->dir_group_readable = 0;
   c->is_ephemeral = 0;
+  c->has_dos_defense_enabled = HS_CONFIG_V3_DOS_DEFENSE_DEFAULT;
+  c->intro_dos_rate_per_sec = HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT;
+  c->intro_dos_burst_per_sec = HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT;
 }
 
-/* From a service configuration object config, clear everything from it
+/** From a service configuration object config, clear everything from it
  * meaning free allocated pointers and reset the values. */
 STATIC void
 service_clear_config(hs_service_config_t *config)
@@ -264,10 +269,15 @@ service_clear_config(hs_service_config_t *config)
                       service_authorized_client_free(p));
     smartlist_free(config->clients);
   }
+  if (config->ob_master_pubkeys) {
+    SMARTLIST_FOREACH(config->ob_master_pubkeys, ed25519_public_key_t *, k,
+                      tor_free(k));
+    smartlist_free(config->ob_master_pubkeys);
+  }
   memset(config, 0, sizeof(*config));
 }
 
-/* Helper function to return a human readable description of the given intro
+/** Helper function to return a human readable description of the given intro
  * point object.
  *
  * This function is not thread-safe. Each call to this invalidates the
@@ -281,9 +291,10 @@ describe_intro_point(const hs_service_intro_point_t *ip)
   const char *legacy_id = NULL;
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, lspec) {
-    if (lspec->type == LS_LEGACY_ID) {
-      legacy_id = (const char *) lspec->u.legacy_id;
+                          const link_specifier_t *, lspec) {
+    if (link_specifier_get_ls_type(lspec) == LS_LEGACY_ID) {
+      legacy_id = (const char *)
+        link_specifier_getconstarray_un_legacy_id(lspec);
       break;
     }
   } SMARTLIST_FOREACH_END(lspec);
@@ -298,7 +309,7 @@ describe_intro_point(const hs_service_intro_point_t *ip)
   return buf;
 }
 
-/* Return the lower bound of maximum INTRODUCE2 cells per circuit before we
+/** Return the lower bound of maximum INTRODUCE2 cells per circuit before we
  * rotate intro point (defined by a consensus parameter or the default
  * value). */
 static int32_t
@@ -311,7 +322,7 @@ get_intro_point_min_introduce2(void)
                                  0, INT32_MAX);
 }
 
-/* Return the upper bound of maximum INTRODUCE2 cells per circuit before we
+/** Return the upper bound of maximum INTRODUCE2 cells per circuit before we
  * rotate intro point (defined by a consensus parameter or the default
  * value). */
 static int32_t
@@ -324,8 +335,8 @@ get_intro_point_max_introduce2(void)
                                  0, INT32_MAX);
 }
 
-/* Return the minimum lifetime in seconds of an introduction point defined by a
- * consensus parameter or the default value. */
+/** Return the minimum lifetime in seconds of an introduction point defined by
+ * a consensus parameter or the default value. */
 static int32_t
 get_intro_point_min_lifetime(void)
 {
@@ -341,8 +352,8 @@ get_intro_point_min_lifetime(void)
                                  0, INT32_MAX);
 }
 
-/* Return the maximum lifetime in seconds of an introduction point defined by a
- * consensus parameter or the default value. */
+/** Return the maximum lifetime in seconds of an introduction point defined by
+ * a consensus parameter or the default value. */
 static int32_t
 get_intro_point_max_lifetime(void)
 {
@@ -358,7 +369,7 @@ get_intro_point_max_lifetime(void)
                                  0, INT32_MAX);
 }
 
-/* Return the number of extra introduction point defined by a consensus
+/** Return the number of extra introduction point defined by a consensus
  * parameter or the default value. */
 static int32_t
 get_intro_point_num_extra(void)
@@ -369,7 +380,7 @@ get_intro_point_num_extra(void)
                                  NUM_INTRO_POINTS_EXTRA, 0, 128);
 }
 
-/* Helper: Function that needs to return 1 for the HT for each loop which
+/** Helper: Function that needs to return 1 for the HT for each loop which
  * frees every service in an hash map. */
 static int
 ht_free_service_(struct hs_service_t *service, void *data)
@@ -381,7 +392,7 @@ ht_free_service_(struct hs_service_t *service, void *data)
   return 1;
 }
 
-/* Free every service that can be found in the global map. Once done, clear
+/** Free every service that can be found in the global map. Once done, clear
  * and free the global map. */
 static void
 service_free_all(void)
@@ -403,7 +414,7 @@ service_free_all(void)
   }
 }
 
-/* Free a given service intro point object. */
+/** Free a given service intro point object. */
 STATIC void
 service_intro_point_free_(hs_service_intro_point_t *ip)
 {
@@ -418,7 +429,7 @@ service_intro_point_free_(hs_service_intro_point_t *ip)
   tor_free(ip);
 }
 
-/* Helper: free an hs_service_intro_point_t object. This function is used by
+/** Helper: free an hs_service_intro_point_t object. This function is used by
  * digest256map_free() which requires a void * pointer. */
 static void
 service_intro_point_free_void(void *obj)
@@ -426,24 +437,17 @@ service_intro_point_free_void(void *obj)
   service_intro_point_free_(obj);
 }
 
-/* Return a newly allocated service intro point and fully initialized from the
- * given extend_info_t ei if non NULL.
- * If is_legacy is true, we also generate the legacy key.
- * If supports_ed25519_link_handshake_any is true, we add the relay's ed25519
- * key to the link specifiers.
+/** Return a newly allocated service intro point and fully initialized from the
+ * given node_t node, if non NULL.
  *
- * If ei is NULL, returns a hs_service_intro_point_t with an empty link
+ * If node is NULL, returns a hs_service_intro_point_t with an empty link
  * specifier list and no onion key. (This is used for testing.)
  * On any other error, NULL is returned.
  *
- * ei must be an extend_info_t containing an IPv4 address. (We will add supoort
- * for IPv6 in a later release.) When calling extend_info_from_node(), pass
- * 0 in for_direct_connection to make sure ei always has an IPv4 address. */
+ * node must be an node_t with an IPv4 address. */
 STATIC hs_service_intro_point_t *
-service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
-                        unsigned int supports_ed25519_link_handshake_any)
+service_intro_point_new(const node_t *node)
 {
-  hs_desc_link_specifier_t *ls;
   hs_service_intro_point_t *ip;
 
   ip = tor_malloc_zero(sizeof(*ip));
@@ -473,12 +477,17 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
   ip->replay_cache = replaycache_new(0, 0);
 
   /* Initialize the base object. We don't need the certificate object. */
-  ip->base.link_specifiers = smartlist_new();
+  ip->base.link_specifiers = node_get_link_specifier_smartlist(node, 0);
+
+  if (node == NULL) {
+    goto done;
+  }
 
   /* Generate the encryption key for this intro point. */
   curve25519_keypair_generate(&ip->enc_key_kp, 0);
-  /* Figure out if this chosen node supports v3 or is legacy only. */
-  if (is_legacy) {
+  /* Figure out if this chosen node supports v3 or is legacy only.
+   * NULL nodes are used in the unit tests. */
+  if (!node_supports_ed25519_hs_intro(node)) {
     ip->base.is_only_legacy = 1;
     /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */
     ip->legacy_key = crypto_pk_new();
@@ -491,40 +500,13 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
     }
   }
 
-  if (ei == NULL) {
-    goto done;
-  }
-
-  /* We'll try to add all link specifiers. Legacy is mandatory.
-   * IPv4 or IPv6 is required, and we always send IPv4. */
-  ls = hs_desc_link_specifier_new(ei, LS_IPV4);
-  /* It is impossible to have an extend info object without a v4. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID);
-  /* It is impossible to have an extend info object without an identity
-   * digest. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  /* ed25519 identity key is optional for intro points. If the node supports
-   * ed25519 link authentication, we include it. */
-  if (supports_ed25519_link_handshake_any) {
-    ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID);
-    if (ls) {
-      smartlist_add(ip->base.link_specifiers, ls);
-    }
-  }
-
-  /* IPv6 is not supported in this release. */
+  /* Flag if this intro point supports the INTRO2 dos defenses. */
+  ip->support_intro2_dos_defense =
+    node_supports_establish_intro_dos_extension(node);
 
-  /* Finally, copy onion key from the extend_info_t object. */
-  memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key));
+  /* Finally, copy onion key from the node. */
+  memcpy(&ip->onion_key, node_get_curve25519_onion_key(node),
+         sizeof(ip->onion_key));
 
  done:
   return ip;
@@ -533,7 +515,7 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
   return NULL;
 }
 
-/* Add the given intro point object to the given intro point map. The intro
+/** Add the given intro point object to the given intro point map. The intro
  * point MUST have its RSA encryption key set if this is a legacy type or the
  * authentication key set otherwise. */
 STATIC void
@@ -549,7 +531,7 @@ service_intro_point_add(digest256map_t *map, hs_service_intro_point_t *ip)
   tor_assert_nonfatal(!old_ip_entry);
 }
 
-/* For a given service, remove the intro point from that service's descriptors
+/** For a given service, remove the intro point from that service's descriptors
  * (check both current and next descriptor) */
 STATIC void
 service_intro_point_remove(const hs_service_t *service,
@@ -567,7 +549,7 @@ service_intro_point_remove(const hs_service_t *service,
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* For a given service and authentication key, return the intro point or NULL
+/** For a given service and authentication key, return the intro point or NULL
  * if not found. This will check both descriptors in the service. */
 STATIC hs_service_intro_point_t *
 service_intro_point_find(const hs_service_t *service,
@@ -598,7 +580,7 @@ service_intro_point_find(const hs_service_t *service,
   return ip;
 }
 
-/* For a given service and intro point, return the descriptor for which the
+/** For a given service and intro point, return the descriptor for which the
  * intro point is assigned to. NULL is returned if not found. */
 STATIC hs_service_descriptor_t *
 service_desc_find_by_intro(const hs_service_t *service,
@@ -620,7 +602,7 @@ service_desc_find_by_intro(const hs_service_t *service,
   return descp;
 }
 
-/* From a circuit identifier, get all the possible objects associated with the
+/** From a circuit identifier, get all the possible objects associated with the
  * ident. If not NULL, service, ip or desc are set if the object can be found.
  * They are untouched if they can't be found.
  *
@@ -653,20 +635,20 @@ get_objects_from_ident(const hs_ident_circuit_t *ident,
   }
 }
 
-/* From a given intro point, return the first link specifier of type
+/** From a given intro point, return the first link specifier of type
  * encountered in the link specifier list. Return NULL if it can't be found.
  *
  * The caller does NOT have ownership of the object, the intro point does. */
-static hs_desc_link_specifier_t *
+static link_specifier_t *
 get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
 {
-  hs_desc_link_specifier_t *lnk_spec = NULL;
+  link_specifier_t *lnk_spec = NULL;
 
   tor_assert(ip);
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          hs_desc_link_specifier_t *, ls) {
-    if (ls->type == type) {
+                          link_specifier_t *, ls) {
+    if (link_specifier_get_ls_type(ls) == type) {
       lnk_spec = ls;
       goto end;
     }
@@ -676,13 +658,13 @@ get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
   return lnk_spec;
 }
 
-/* Given a service intro point, return the node_t associated to it. This can
+/** Given a service intro point, return the node_t associated to it. This can
  * return NULL if the given intro point has no legacy ID or if the node can't
  * be found in the consensus. */
 STATIC const node_t *
 get_node_from_intro_point(const hs_service_intro_point_t *ip)
 {
-  const hs_desc_link_specifier_t *ls;
+  const link_specifier_t *ls;
 
   tor_assert(ip);
 
@@ -691,10 +673,11 @@ get_node_from_intro_point(const hs_service_intro_point_t *ip)
     return NULL;
   }
   /* XXX In the future, we want to only use the ed25519 ID (#22173). */
-  return node_get_by_id((const char *) ls->u.legacy_id);
+  return node_get_by_id(
+    (const char *) link_specifier_getconstarray_un_legacy_id(ls));
 }
 
-/* Given a service intro point, return the extend_info_t for it. This can
+/** Given a service intro point, return the extend_info_t for it. This can
  * return NULL if the node can't be found for the intro point or the extend
  * info can't be created for the found node. If direct_conn is set, the extend
  * info is validated on if we can connect directly. */
@@ -723,10 +706,10 @@ get_extend_info_from_intro_point(const hs_service_intro_point_t *ip,
   return info;
 }
 
-/* Return the number of introduction points that are established for the
+/** Return the number of introduction points that are established for the
  * given descriptor. */
-static unsigned int
-count_desc_circuit_established(const hs_service_descriptor_t *desc)
+MOCK_IMPL(STATIC unsigned int,
+count_desc_circuit_established, (const hs_service_descriptor_t *desc))
 {
   unsigned int count = 0;
 
@@ -734,13 +717,13 @@ count_desc_circuit_established(const hs_service_descriptor_t *desc)
 
   DIGEST256MAP_FOREACH(desc->intro_points.map, key,
                        const hs_service_intro_point_t *, ip) {
-    count += ip->circuit_established;
+    count += !!hs_circ_service_get_established_intro_circ(ip);
   } DIGEST256MAP_FOREACH_END;
 
   return count;
 }
 
-/* For a given service and descriptor of that service, close all active
+/** For a given service and descriptor of that service, close all active
  * directory connections. */
 static void
 close_directory_connections(const hs_service_t *service,
@@ -775,7 +758,7 @@ close_directory_connections(const hs_service_t *service,
   smartlist_free(dir_conns);
 }
 
-/* Close all rendezvous circuits for the given service. */
+/** Close all rendezvous circuits for the given service. */
 static void
 close_service_rp_circuits(hs_service_t *service)
 {
@@ -805,7 +788,7 @@ close_service_rp_circuits(hs_service_t *service)
   }
 }
 
-/* Close the circuit(s) for the given map of introduction points. */
+/** Close the circuit(s) for the given map of introduction points. */
 static void
 close_intro_circuits(hs_service_intropoints_t *intro_points)
 {
@@ -823,7 +806,7 @@ close_intro_circuits(hs_service_intropoints_t *intro_points)
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Close all introduction circuits for the given service. */
+/** Close all introduction circuits for the given service. */
 static void
 close_service_intro_circuits(hs_service_t *service)
 {
@@ -834,7 +817,7 @@ close_service_intro_circuits(hs_service_t *service)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Close any circuits related to the given service. */
+/** Close any circuits related to the given service. */
 static void
 close_service_circuits(hs_service_t *service)
 {
@@ -850,7 +833,7 @@ close_service_circuits(hs_service_t *service)
   close_service_rp_circuits(service);
 }
 
-/* Move every ephemeral services from the src service map to the dst service
+/** Move every ephemeral services from the src service map to the dst service
  * map. It is possible that a service can't be register to the dst map which
  * won't stop the process of moving them all but will trigger a log warn. */
 static void
@@ -880,7 +863,7 @@ move_ephemeral_services(hs_service_ht *src, hs_service_ht *dst)
   }
 }
 
-/* Return a const string of the directory path escaped. If this is an
+/** Return a const string of the directory path escaped. If this is an
  * ephemeral service, it returns "[EPHEMERAL]". This can only be called from
  * the main thread because escaped() uses a static variable. */
 static const char *
@@ -908,13 +891,21 @@ move_hs_state(hs_service_t *src_service, hs_service_t *dst_service)
   if (dst->replay_cache_rend_cookie != NULL) {
     replaycache_free(dst->replay_cache_rend_cookie);
   }
+
   dst->replay_cache_rend_cookie = src->replay_cache_rend_cookie;
+  src->replay_cache_rend_cookie = NULL; /* steal pointer reference */
+
   dst->next_rotation_time = src->next_rotation_time;
 
-  src->replay_cache_rend_cookie = NULL; /* steal pointer reference */
+  if (src->ob_subcreds) {
+    dst->ob_subcreds = src->ob_subcreds;
+    dst->n_ob_subcreds =  src->n_ob_subcreds;
+
+    src->ob_subcreds = NULL; /* steal pointer reference */
+  }
 }
 
-/* Register services that are in the staging list. Once this function returns,
+/** Register services that are in the staging list. Once this function returns,
  * the global service map will be set with the right content and all non
  * surviving services will be cleaned up. */
 static void
@@ -982,7 +973,7 @@ register_all_services(void)
   hs_service_map_has_changed();
 }
 
-/* Write the onion address of a given service to the given filename fname_ in
+/** Write the onion address of a given service to the given filename fname_ in
  * the service directory. Return 0 on success else -1 on error. */
 STATIC int
 write_address_to_file(const hs_service_t *service, const char *fname_)
@@ -1023,7 +1014,7 @@ write_address_to_file(const hs_service_t *service, const char *fname_)
   return ret;
 }
 
-/* Load and/or generate private keys for the given service. On success, the
+/** Load and/or generate private keys for the given service. On success, the
  * hostname file will be written to disk along with the master private key iff
  * the service is not configured for offline keys. Return 0 on success else -1
  * on failure. */
@@ -1099,7 +1090,7 @@ load_service_keys(hs_service_t *service)
   return ret;
 }
 
-/* Check if the client file name is valid or not. Return 1 if valid,
+/** Check if the client file name is valid or not. Return 1 if valid,
  * otherwise return 0. */
 STATIC int
 client_filename_is_valid(const char *filename)
@@ -1121,7 +1112,7 @@ client_filename_is_valid(const char *filename)
   return ret;
 }
 
-/* Parse an authorized client from a string. The format of a client string
+/** Parse an authorized client from a string. The format of a client string
  * looks like (see rend-spec-v3.txt):
  *
  *  <auth-type>:<key-type>:<base32-encoded-public-key>
@@ -1180,7 +1171,8 @@ parse_authorized_client(const char *client_key_str)
   client = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
   if (base32_decode((char *) client->client_pk.public_key,
                     sizeof(client->client_pk.public_key),
-                    pubkey_b32, strlen(pubkey_b32)) < 0) {
+                    pubkey_b32, strlen(pubkey_b32)) !=
+      sizeof(client->client_pk.public_key)) {
     log_warn(LD_REND, "Client authorization public key cannot be decoded: %s",
              pubkey_b32);
     goto err;
@@ -1202,7 +1194,7 @@ parse_authorized_client(const char *client_key_str)
   return client;
 }
 
-/* Load all the client public keys for the given service. Return 0 on
+/** Load all the client public keys for the given service. Return 0 on
  * success else -1 on failure. */
 static int
 load_client_keys(hs_service_t *service)
@@ -1262,7 +1254,7 @@ load_client_keys(hs_service_t *service)
     client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
 
     /* If we cannot read the file, continue with the next file. */
-    if (!client_key_str)  {
+    if (!client_key_str) {
       log_warn(LD_REND, "Client authorization file %s can't be read. "
                         "Corrupted or verify permission? Ignoring.",
                client_key_file_path);
@@ -1305,6 +1297,7 @@ load_client_keys(hs_service_t *service)
   return ret;
 }
 
+/** Release all storage held in <b>client</b>. */
 STATIC void
 service_authorized_client_free_(hs_service_authorized_client_t *client)
 {
@@ -1315,7 +1308,7 @@ service_authorized_client_free_(hs_service_authorized_client_t *client)
   tor_free(client);
 }
 
-/* Free a given service descriptor object and all key material is wiped. */
+/** Free a given service descriptor object and all key material is wiped. */
 STATIC void
 service_descriptor_free_(hs_service_descriptor_t *desc)
 {
@@ -1336,7 +1329,7 @@ service_descriptor_free_(hs_service_descriptor_t *desc)
   tor_free(desc);
 }
 
-/* Return a newly allocated service descriptor object. */
+/** Return a newly allocated service descriptor object. */
 STATIC hs_service_descriptor_t *
 service_descriptor_new(void)
 {
@@ -1349,7 +1342,7 @@ service_descriptor_new(void)
   return sdesc;
 }
 
-/* Allocate and return a deep copy of client. */
+/** Allocate and return a deep copy of client. */
 static hs_service_authorized_client_t *
 service_authorized_client_dup(const hs_service_authorized_client_t *client)
 {
@@ -1367,7 +1360,7 @@ service_authorized_client_dup(const hs_service_authorized_client_t *client)
   return client_dup;
 }
 
-/* If two authorized clients are equal, return 0. If the first one should come
+/** If two authorized clients are equal, return 0. If the first one should come
  * before the second, return less than zero. If the first should come after
  * the second, return greater than zero. */
 static int
@@ -1384,7 +1377,7 @@ service_authorized_client_cmp(const hs_service_authorized_client_t *client1,
                     CURVE25519_PUBKEY_LEN);
 }
 
-/* Helper for sorting authorized clients. */
+/** Helper for sorting authorized clients. */
 static int
 compare_service_authorzized_client_(const void **_a, const void **_b)
 {
@@ -1392,7 +1385,7 @@ compare_service_authorzized_client_(const void **_a, const void **_b)
   return service_authorized_client_cmp(a, b);
 }
 
-/* If the list of hs_service_authorized_client_t's is different between
+/** If the list of hs_service_authorized_client_t's is different between
  * src and dst, return 1. Otherwise, return 0. */
 STATIC int
 service_authorized_client_config_equal(const hs_service_config_t *config1,
@@ -1453,7 +1446,7 @@ service_authorized_client_config_equal(const hs_service_config_t *config1,
   return ret;
 }
 
-/* Move descriptor(s) from the src service to the dst service and modify their
+/** Move descriptor(s) from the src service to the dst service and modify their
  * content if necessary. We do this during SIGHUP when we re-create our
  * hidden services. */
 static void
@@ -1512,7 +1505,7 @@ move_descriptors(hs_service_t *src, hs_service_t *dst)
   service_descriptor_free(dst->desc_next);
 }
 
-/* From the given service, remove all expired failing intro points for each
+/** From the given service, remove all expired failing intro points for each
  * descriptor. */
 static void
 remove_expired_failing_intro(hs_service_t *service, time_t now)
@@ -1531,7 +1524,7 @@ remove_expired_failing_intro(hs_service_t *service, time_t now)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* For the given descriptor desc, put all node_t object found from its failing
+/** For the given descriptor desc, put all node_t object found from its failing
  * intro point list and put them in the given node_list. */
 static void
 setup_intro_point_exclude_list(const hs_service_descriptor_t *desc,
@@ -1549,7 +1542,7 @@ setup_intro_point_exclude_list(const hs_service_descriptor_t *desc,
   } DIGESTMAP_FOREACH_END;
 }
 
-/* For the given failing intro point ip, we add its time of failure to the
+/** For the given failing intro point ip, we add its time of failure to the
  * failed map and index it by identity digest (legacy ID) in the descriptor
  * desc failed id map. */
 static void
@@ -1557,7 +1550,7 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
                              hs_service_descriptor_t *desc, time_t now)
 {
   time_t *time_of_failure, *prev_ptr;
-  const hs_desc_link_specifier_t *legacy_ls;
+  const link_specifier_t *legacy_ls;
 
   tor_assert(ip);
   tor_assert(desc);
@@ -1566,23 +1559,14 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
   *time_of_failure = now;
   legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
   tor_assert(legacy_ls);
-  prev_ptr = digestmap_set(desc->intro_points.failed_id,
-                           (const char *) legacy_ls->u.legacy_id,
-                           time_of_failure);
+  prev_ptr = digestmap_set(
+    desc->intro_points.failed_id,
+    (const char *) link_specifier_getconstarray_un_legacy_id(legacy_ls),
+    time_of_failure);
   tor_free(prev_ptr);
 }
 
-/* Copy the descriptor link specifier object from src to dst. */
-static void
-link_specifier_copy(hs_desc_link_specifier_t *dst,
-                    const hs_desc_link_specifier_t *src)
-{
-  tor_assert(dst);
-  tor_assert(src);
-  memcpy(dst, src, sizeof(hs_desc_link_specifier_t));
-}
-
-/* Using a given descriptor signing keypair signing_kp, a service intro point
+/** Using a given descriptor signing keypair signing_kp, a service intro point
  * object ip and the time now, setup the content of an already allocated
  * descriptor intro desc_ip.
  *
@@ -1616,9 +1600,14 @@ setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
 
   /* Copy link specifier(s). */
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, ls) {
-    hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy));
-    link_specifier_copy(copy, ls);
+                          const link_specifier_t *, ls) {
+    if (BUG(!ls)) {
+      goto done;
+    }
+    link_specifier_t *copy = link_specifier_dup(ls);
+    if (BUG(!copy)) {
+      goto done;
+    }
     smartlist_add(desc_ip->link_specifiers, copy);
   } SMARTLIST_FOREACH_END(ls);
 
@@ -1666,7 +1655,7 @@ setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
   return ret;
 }
 
-/* Using the given descriptor from the given service, build the descriptor
+/** Using the given descriptor from the given service, build the descriptor
  * intro point list so we can then encode the descriptor for publication. This
  * function does not pick intro points, they have to be in the descriptor
  * current map. Cryptographic material (keys) must be initialized in the
@@ -1687,7 +1676,7 @@ build_desc_intro_points(const hs_service_t *service,
 
   DIGEST256MAP_FOREACH(desc->intro_points.map, key,
                        const hs_service_intro_point_t *, ip) {
-    if (!ip->circuit_established) {
+    if (!hs_circ_service_get_established_intro_circ(ip)) {
       /* Ignore un-established intro points. They can linger in that list
        * because their circuit has not opened and they haven't been removed
        * yet even though we have enough intro circuits.
@@ -1706,7 +1695,7 @@ build_desc_intro_points(const hs_service_t *service,
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Build the descriptor signing key certificate. */
+/** Build the descriptor signing key certificate. */
 static void
 build_desc_signing_key_cert(hs_service_descriptor_t *desc, time_t now)
 {
@@ -1732,7 +1721,7 @@ build_desc_signing_key_cert(hs_service_descriptor_t *desc, time_t now)
   tor_assert_nonfatal(plaintext->signing_key_cert);
 }
 
-/* Populate the descriptor encrypted section from the given service object.
+/** Populate the descriptor encrypted section from the given service object.
  * This will generate a valid list of introduction points that can be used
  * after for circuit creation. Return 0 on success else -1 on error. */
 static int
@@ -1762,7 +1751,7 @@ build_service_desc_encrypted(const hs_service_t *service,
   return 0;
 }
 
-/* Populate the descriptor superencrypted section from the given service
+/** Populate the descriptor superencrypted section from the given service
  * object. This will generate a valid list of hs_desc_authorized_client_t
  * of clients that are authorized to use the service. Return 0 on success
  * else -1 on error. */
@@ -1790,7 +1779,8 @@ build_service_desc_superencrypted(const hs_service_t *service,
          sizeof(curve25519_public_key_t));
 
   /* Test that subcred is not zero because we might use it below */
-  if (BUG(tor_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) {
+  if (BUG(fast_mem_is_zero((char*)desc->desc->subcredential.subcred,
+                           DIGEST256_LEN))) {
     return -1;
   }
 
@@ -1807,7 +1797,7 @@ build_service_desc_superencrypted(const hs_service_t *service,
 
       /* Prepare the client for descriptor and then add to the list in the
        * superencrypted part of the descriptor */
-      hs_desc_build_authorized_client(desc->desc->subcredential,
+      hs_desc_build_authorized_client(&desc->desc->subcredential,
                                       &client->client_pk,
                                       &desc->auth_ephemeral_kp.seckey,
                                       desc->descriptor_cookie, desc_client);
@@ -1845,7 +1835,7 @@ build_service_desc_superencrypted(const hs_service_t *service,
   return 0;
 }
 
-/* Populate the descriptor plaintext section from the given service object.
+/** Populate the descriptor plaintext section from the given service object.
  * The caller must make sure that the keys in the descriptors are valid that
  * is are non-zero. This can't fail. */
 static void
@@ -1856,14 +1846,14 @@ build_service_desc_plaintext(const hs_service_t *service,
 
   tor_assert(service);
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->blinded_kp,
                               sizeof(desc->blinded_kp)));
-  tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->signing_kp,
                               sizeof(desc->signing_kp)));
 
   /* Set the subcredential. */
   hs_get_subcredential(&service->keys.identity_pk, &desc->blinded_kp.pubkey,
-                       desc->desc->subcredential);
+                       &desc->desc->subcredential);
 
   plaintext = &desc->desc->plaintext_data;
 
@@ -1896,7 +1886,7 @@ generate_ope_cipher_for_desc(const hs_service_descriptor_t *hs_desc)
   return crypto_ope_new(key);
 }
 
-/* For the given service and descriptor object, create the key material which
+/** For the given service and descriptor object, create the key material which
  * is the blinded keypair, the descriptor signing keypair, the ephemeral
  * keypair, and the descriptor cookie. Return 0 on success else -1 on error
  * where the generated keys MUST be ignored. */
@@ -1908,7 +1898,7 @@ build_service_desc_keys(const hs_service_t *service,
   ed25519_keypair_t kp;
 
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk,
+  tor_assert(!fast_mem_is_zero((char *) &service->keys.identity_pk,
              ED25519_PUBKEY_LEN));
 
   /* XXX: Support offline key feature (#18098). */
@@ -1958,7 +1948,7 @@ build_service_desc_keys(const hs_service_t *service,
   return ret;
 }
 
-/* Given a service and the current time, build a descriptor for the service.
+/** Given a service and the current time, build a descriptor for the service.
  * This function does not pick introduction point, this needs to be done by
  * the update function. On success, desc_out will point to the newly allocated
  * descriptor object.
@@ -2006,16 +1996,22 @@ build_service_descriptor(hs_service_t *service, uint64_t time_period_num,
 
   /* Assign newly built descriptor to the next slot. */
   *desc_out = desc;
+
   /* Fire a CREATED control port event. */
   hs_control_desc_event_created(service->onion_address,
                                 &desc->blinded_kp.pubkey);
+
+  /* If we are an onionbalance instance, we refresh our keys when we rotate
+   * descriptors. */
+  hs_ob_refresh_keys(service);
+
   return;
 
  err:
   service_descriptor_free(desc);
 }
 
-/* Build both descriptors for the given service that has just booted up.
+/** Build both descriptors for the given service that has just booted up.
  * Because it's a special case, it deserves its special function ;). */
 static void
 build_descriptors_for_new_service(hs_service_t *service, time_t now)
@@ -2065,7 +2061,7 @@ build_descriptors_for_new_service(hs_service_t *service, time_t now)
            safe_str_client(service->onion_address));
 }
 
-/* Build descriptors for each service if needed. There are conditions to build
+/** Build descriptors for each service if needed. There are conditions to build
  * a descriptor which are details in the function. */
 STATIC void
 build_all_descriptors(time_t now)
@@ -2098,7 +2094,7 @@ build_all_descriptors(time_t now)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Randomly pick a node to become an introduction point but not present in the
+/** Randomly pick a node to become an introduction point but not present in the
  * given exclude_nodes list. The chosen node is put in the exclude list
  * regardless of success or not because in case of failure, the node is simply
  * unsusable from that point on.
@@ -2117,7 +2113,6 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
 {
   const or_options_t *options = get_options();
   const node_t *node;
-  extend_info_t *info = NULL;
   hs_service_intro_point_t *ip = NULL;
   /* Normal 3-hop introduction point flags. */
   router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC;
@@ -2146,47 +2141,21 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
    * we don't want to use that node anymore. */
   smartlist_add(exclude_nodes, (void *) node);
 
-  /* We do this to ease our life but also this call makes appropriate checks
-   * of the node object such as validating ntor support for instance.
-   *
-   * We must provide an extend_info for clients to connect over a 3-hop path,
-   * so we don't pass direct_conn here. */
-  info = extend_info_from_node(node, 0);
-  if (BUG(info == NULL)) {
-    goto err;
-  }
-
-  /* Let's do a basic sanity check here so that we don't end up advertising the
-   * ed25519 identity key of relays that don't actually support the link
-   * protocol */
-  if (!node_supports_ed25519_link_authentication(node, 0)) {
-    tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity));
-  } else {
-    /* Make sure we *do* have an ed key if we support the link authentication.
-     * Sending an empty key would result in a failure to extend. */
-    tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity));
-  }
+  /* Create our objects and populate them with the node information. */
+  ip = service_intro_point_new(node);
 
-  /* Create our objects and populate them with the node information.
-   * We don't care if the intro's link auth is compatible with us, because
-   * we are sending the ed25519 key to a remote client via the descriptor. */
-  ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node),
-                               node_supports_ed25519_link_authentication(node,
-                                                                         0));
   if (ip == NULL) {
     goto err;
   }
 
-  log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info));
-  extend_info_free(info);
+  log_info(LD_REND, "Picked intro point: %s", node_describe(node));
   return ip;
  err:
   service_intro_point_free(ip);
-  extend_info_free(info);
   return NULL;
 }
 
-/* For a given descriptor from the given service, pick any needed intro points
+/** For a given descriptor from the given service, pick any needed intro points
  * and update the current map with those newly picked intro points. Return the
  * number node that might have been added to the descriptor current map. */
 static unsigned int
@@ -2310,7 +2279,7 @@ service_desc_schedule_upload(hs_service_descriptor_t *desc,
   }
 }
 
-/* Pick missing intro points for this descriptor if needed. */
+/** Pick missing intro points for this descriptor if needed. */
 static void
 update_service_descriptor_intro_points(hs_service_t *service,
                           hs_service_descriptor_t *desc, time_t now)
@@ -2351,7 +2320,7 @@ update_service_descriptor_intro_points(hs_service_t *service,
   }
 }
 
-/* Update descriptor intro points for each service if needed. We do this as
+/** Update descriptor intro points for each service if needed. We do this as
  * part of the periodic event because we need to establish intro point circuits
  * before we publish descriptors. */
 STATIC void
@@ -2366,7 +2335,7 @@ update_all_descriptors_intro_points(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Return true iff the given intro point has expired that is it has been used
+/** Return true iff the given intro point has expired that is it has been used
  * for too long or we've reached our max seen INTRODUCE2 cell. */
 STATIC int
 intro_point_should_expire(const hs_service_intro_point_t *ip,
@@ -2388,15 +2357,66 @@ intro_point_should_expire(const hs_service_intro_point_t *ip,
   return 1;
 }
 
-/* Go over the given set of intro points for each service and remove any
- * invalid ones. The conditions for removal are:
+/** Return true iff we should remove the intro point ip from its service.
+ *
+ * We remove an intro point from the service descriptor list if one of
+ * these criteria is met:
+ *    - It has expired (either in INTRO2 count or in time).
+ *    - No node was found (fell off the consensus).
+ *    - We are over the maximum amount of retries.
  *
- *    - The node doesn't exists anymore (not in consensus)
- *                          OR
- *    - The intro point maximum circuit retry count has been reached and no
- *      circuit can be found associated with it.
- *                          OR
- *    - The intro point has expired and we should pick a new one.
+ * If an established or pending circuit is found for the given ip object, this
+ * return false indicating it should not be removed. */
+static bool
+should_remove_intro_point(hs_service_intro_point_t *ip, time_t now)
+{
+  bool ret = false;
+
+  tor_assert(ip);
+
+  /* Any one of the following needs to be True to furfill the criteria to
+   * remove an intro point. */
+  bool has_no_retries = (ip->circuit_retries >
+                         MAX_INTRO_POINT_CIRCUIT_RETRIES);
+  bool has_no_node = (get_node_from_intro_point(ip) == NULL);
+  bool has_expired = intro_point_should_expire(ip, now);
+
+  /* If the node fell off the consensus or the IP has expired, we have to
+   * remove it now. */
+  if (has_no_node || has_expired) {
+    ret = true;
+    goto end;
+  }
+
+  /* Pass this point, even though we might be over the retry limit, we check
+   * if a circuit (established or pending) exists. In that case, we should not
+   * remove it because it might simply be valid and opened at the previous
+   * scheduled event for the last retry. */
+
+  /* Do we simply have an existing circuit regardless of its state? */
+  if (hs_circ_service_get_intro_circ(ip)) {
+    goto end;
+  }
+
+  /* Getting here means we have _no_ circuits so then return if we have any
+   * remaining retries. */
+  ret = has_no_retries;
+
+ end:
+  /* Meaningful log in case we are about to remove the IP. */
+  if (ret) {
+    log_info(LD_REND, "Intro point %s%s (retried: %u times). "
+                      "Removing it.",
+             describe_intro_point(ip),
+             has_expired ? " has expired" :
+               (has_no_node) ?  " fell off the consensus" : "",
+             ip->circuit_retries);
+  }
+  return ret;
+}
+
+/** Go over the given set of intro points for each service and remove any
+ * invalid ones.
  *
  * If an intro point is removed, the circuit (if any) is immediately close.
  * If a circuit can't be found, the intro point is kept if it hasn't reached
@@ -2405,12 +2425,10 @@ static void
 cleanup_intro_points(hs_service_t *service, time_t now)
 {
   /* List of intro points to close. We can't mark the intro circuits for close
-   * in the modify loop because doing so calls
-   * hs_service_intro_circ_has_closed() which does a digest256map_get() on the
-   * intro points map (that we are iterating over). This can't be done in a
-   * single iteration after a MAP_DEL_CURRENT, the object will still be
-   * returned leading to a use-after-free. So, we close the circuits and free
-   * the intro points after the loop if any. */
+   * in the modify loop because doing so calls back into the HS subsystem and
+   * we need to keep that code path outside of the service/desc loop so those
+   * maps don't get modified during the close making us in a possible
+   * use-after-free situation. */
   smartlist_t *ips_to_free = smartlist_new();
 
   tor_assert(service);
@@ -2421,21 +2439,7 @@ cleanup_intro_points(hs_service_t *service, time_t now)
      * valid and remove any of them that aren't. */
     DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
                                 hs_service_intro_point_t *, ip) {
-      const node_t *node = get_node_from_intro_point(ip);
-      int has_expired = intro_point_should_expire(ip, now);
-
-      /* We cleanup an intro point if it has expired or if we do not know the
-       * node_t anymore (removed from our latest consensus) or if we've
-       * reached the maximum number of retry with a non existing circuit. */
-      if (has_expired || node == NULL ||
-          ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
-        log_info(LD_REND, "Intro point %s%s (retried: %u times). "
-                          "Removing it.",
-                 describe_intro_point(ip),
-                 has_expired ? " has expired" :
-                    (node == NULL) ?  " fell off the consensus" : "",
-                 ip->circuit_retries);
-
+      if (should_remove_intro_point(ip, now)) {
         /* We've retried too many times, remember it as a failed intro point
          * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */
         if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -2472,7 +2476,7 @@ cleanup_intro_points(hs_service_t *service, time_t now)
   smartlist_free(ips_to_free);
 }
 
-/* Set the next rotation time of the descriptors for the given service for the
+/** Set the next rotation time of the descriptors for the given service for the
  * time now. */
 static void
 set_rotation_time(hs_service_t *service)
@@ -2491,7 +2495,7 @@ set_rotation_time(hs_service_t *service)
   }
 }
 
-/* Return true iff the service should rotate its descriptor. The time now is
+/** Return true iff the service should rotate its descriptor. The time now is
  * only used to fetch the live consensus and if none can be found, this
  * returns false. */
 static unsigned int
@@ -2544,7 +2548,7 @@ should_rotate_descriptors(hs_service_t *service, time_t now)
   return 1;
 }
 
-/* Rotate the service descriptors of the given service. The current descriptor
+/** Rotate the service descriptors of the given service. The current descriptor
  * will be freed, the next one put in as the current and finally the next
  * descriptor pointer is NULLified. */
 static void
@@ -2566,7 +2570,7 @@ rotate_service_descriptors(hs_service_t *service)
   set_rotation_time(service);
 }
 
-/* Rotate descriptors for each service if needed. A non existing current
+/** Rotate descriptors for each service if needed. A non existing current
  * descriptor will trigger a descriptor build for the next time period. */
 STATIC void
 rotate_all_descriptors(time_t now)
@@ -2595,7 +2599,7 @@ rotate_all_descriptors(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Scheduled event run from the main loop. Make sure all our services are up
+/** Scheduled event run from the main loop. Make sure all our services are up
  * to date and ready for the other scheduled events. This includes looking at
  * the introduction points status and descriptor rotation time. */
 STATIC void
@@ -2630,7 +2634,7 @@ run_housekeeping_event(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Scheduled event run from the main loop. Make sure all descriptors are up to
+/** Scheduled event run from the main loop. Make sure all descriptors are up to
  * date. Once this returns, each service descriptor needs to be considered for
  * new introduction circuits and then for upload. */
 static void
@@ -2653,7 +2657,7 @@ run_build_descriptor_event(time_t now)
   update_all_descriptors_intro_points(now);
 }
 
-/* For the given service, launch any intro point circuits that could be
+/** For the given service, launch any intro point circuits that could be
  * needed. This considers every descriptor of the service. */
 static void
 launch_intro_point_circuits(hs_service_t *service)
@@ -2707,7 +2711,7 @@ launch_intro_point_circuits(hs_service_t *service)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Don't try to build more than this many circuits before giving up for a
+/** Don't try to build more than this many circuits before giving up for a
  * while. Dynamically calculated based on the configured number of intro
  * points for the given service and how many descriptor exists. The default
  * use case of 3 introduction points and two descriptors will allow 28
@@ -2723,7 +2727,7 @@ get_max_intro_circ_per_period(const hs_service_t *service)
   tor_assert(service->config.num_intro_points <=
              HS_CONFIG_V3_MAX_INTRO_POINTS);
 
-/* For a testing network, allow to do it for the maximum amount so circuit
+/** For a testing network, allow to do it for the maximum amount so circuit
  * creation and rotation and so on can actually be tested without limit. */
 #define MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING -1
   if (get_options()->TestingTorNetwork) {
@@ -2752,7 +2756,7 @@ get_max_intro_circ_per_period(const hs_service_t *service)
   return (count * multiplier);
 }
 
-/* For the given service, return 1 if the service is allowed to launch more
+/** For the given service, return 1 if the service is allowed to launch more
  * introduction circuits else 0 if the maximum has been reached for the retry
  * period of INTRO_CIRC_RETRY_PERIOD. */
 STATIC int
@@ -2798,7 +2802,7 @@ can_service_launch_intro_circuit(hs_service_t *service, time_t now)
   return 1;
 }
 
-/* Scheduled event run from the main loop. Make sure we have all the circuits
+/** Scheduled event run from the main loop. Make sure we have all the circuits
  * we need for each service. */
 static void
 run_build_circuit_event(time_t now)
@@ -2828,7 +2832,7 @@ run_build_circuit_event(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Encode and sign the service descriptor desc and upload it to the given
+/** Encode and sign the service descriptor desc and upload it to the given
  * hidden service directory.  This does nothing if PublishHidServDescriptors
  * is false. */
 static void
@@ -2844,7 +2848,7 @@ upload_descriptor_to_hsdir(const hs_service_t *service,
   /* Let's avoid doing that if tor is configured to not publish. */
   if (!get_options()->PublishHidServDescriptors) {
     log_info(LD_REND, "Service %s not publishing descriptor. "
-                      "PublishHidServDescriptors is set to 1.",
+                      "PublishHidServDescriptors is set to 0.",
              safe_str_client(service->onion_address));
     goto end;
   }
@@ -2958,13 +2962,13 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now,
   /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */
   tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR);
 
-  log_info(LD_REND, "Encrypted revision counter %d to %ld",
-           (int) seconds_since_start_of_srv, (long int) rev_counter);
+  log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64,
+           (int) seconds_since_start_of_srv, rev_counter);
 
   hs_desc->desc->plaintext_data.revision_counter = rev_counter;
 }
 
-/* Encode and sign the service descriptor desc and upload it to the
+/** Encode and sign the service descriptor desc and upload it to the
  * responsible hidden service directories. If for_next_period is true, the set
  * of directories are selected using the next hsdir_index. This does nothing
  * if PublishHidServDescriptors is false. */
@@ -3061,13 +3065,85 @@ service_desc_hsdirs_changed(const hs_service_t *service,
   return should_reupload;
 }
 
-/* Return 1 if the given descriptor from the given service can be uploaded
+/** These are all the reasons why a descriptor upload can't occur. We use
+ * those to log the reason properly with the right rate limiting and for the
+ * right descriptor. */
+typedef enum {
+  LOG_DESC_UPLOAD_REASON_MISSING_IPS = 0,
+  LOG_DESC_UPLOAD_REASON_IP_NOT_ESTABLISHED = 1,
+  LOG_DESC_UPLOAD_REASON_NOT_TIME = 2,
+  LOG_DESC_UPLOAD_REASON_NO_LIVE_CONSENSUS = 3,
+  LOG_DESC_UPLOAD_REASON_NO_DIRINFO = 4,
+} log_desc_upload_reason_t;
+
+/** Maximum number of reasons. This is used to allocate the static array of
+ * all rate limiting objects. */
+#define LOG_DESC_UPLOAD_REASON_MAX LOG_DESC_UPLOAD_REASON_NO_DIRINFO
+
+/** Log the reason why we can't upload the given descriptor for the given
+ * service. This takes a message string (allocated by the caller) and a
+ * reason.
+ *
+ * Depending on the reason and descriptor, different rate limit applies. This
+ * is done because this function will basically be called every second. Each
+ * descriptor for each reason uses its own log rate limit object in order to
+ * avoid message suppression for different reasons and descriptors. */
+static void
+log_cant_upload_desc(const hs_service_t *service,
+                     const hs_service_descriptor_t *desc, const char *msg,
+                     const log_desc_upload_reason_t reason)
+{
+  /* Writing the log every minute shouldn't be too annoying for log rate limit
+   * since this can be emitted every second for each descriptor.
+   *
+   * However, for one specific case, we increase it to 10 minutes because it
+   * is hit constantly, as an expected behavior, which is the reason
+   * indicating that it is not the time to upload. */
+  static ratelim_t limits[2][LOG_DESC_UPLOAD_REASON_MAX + 1] =
+    { { RATELIM_INIT(60), RATELIM_INIT(60), RATELIM_INIT(60 * 10),
+        RATELIM_INIT(60), RATELIM_INIT(60) },
+      { RATELIM_INIT(60), RATELIM_INIT(60), RATELIM_INIT(60 * 10),
+        RATELIM_INIT(60), RATELIM_INIT(60) },
+    };
+  bool is_next_desc = false;
+  unsigned int rlim_pos = 0;
+  ratelim_t *rlim = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(msg);
+
+  /* Make sure the reason value is valid. It should never happen because we
+   * control that value in the code flow but will be apparent during
+   * development if a reason is added but LOG_DESC_UPLOAD_REASON_NUM_ is not
+   * updated. */
+  if (BUG(reason > LOG_DESC_UPLOAD_REASON_MAX)) {
+    return;
+  }
+
+  /* Ease our life. Flag that tells us if the descriptor is the next one. */
+  is_next_desc = (service->desc_next == desc);
+
+  /* Current descriptor is the first element in the ratelimit object array.
+   * The next descriptor is the second element. */
+  rlim_pos = (is_next_desc ? 1 : 0);
+  /* Get the ratelimit object for the reason _and_ right descriptor. */
+  rlim = &limits[rlim_pos][reason];
+
+  log_fn_ratelim(rlim, LOG_INFO, LD_REND,
+                 "Service %s can't upload its %s descriptor: %s",
+                 safe_str_client(service->onion_address),
+                 (is_next_desc) ? "next" : "current", msg);
+}
+
+/** Return 1 if the given descriptor from the given service can be uploaded
  * else return 0 if it can not. */
 static int
 should_service_upload_descriptor(const hs_service_t *service,
                               const hs_service_descriptor_t *desc, time_t now)
 {
-  unsigned int num_intro_points;
+  char *msg = NULL;
+  unsigned int num_intro_points, count_ip_established;
 
   tor_assert(service);
   tor_assert(desc);
@@ -3087,39 +3163,59 @@ should_service_upload_descriptor(const hs_service_t *service,
    * upload descriptor in this case. We need at least one for the service to
    * be reachable. */
   if (desc->missing_intro_points && num_intro_points == 0) {
+    msg = tor_strdup("Missing intro points");
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_MISSING_IPS);
     goto cannot;
   }
 
   /* Check if all our introduction circuit have been established for all the
    * intro points we have selected. */
-  if (count_desc_circuit_established(desc) != num_intro_points) {
+  count_ip_established = count_desc_circuit_established(desc);
+  if (count_ip_established != num_intro_points) {
+    tor_asprintf(&msg, "Intro circuits aren't yet all established (%d/%d).",
+                 count_ip_established, num_intro_points);
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_IP_NOT_ESTABLISHED);
     goto cannot;
   }
 
   /* Is it the right time to upload? */
   if (desc->next_upload_time > now) {
+    tor_asprintf(&msg, "Next upload time is %ld, it is now %ld.",
+                 (long int) desc->next_upload_time, (long int) now);
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_NOT_TIME);
     goto cannot;
   }
 
   /* Don't upload desc if we don't have a live consensus */
   if (!networkstatus_get_reasonably_live_consensus(now,
                                             usable_consensus_flavor())) {
+    msg = tor_strdup("No reasonably live consensus");
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_NO_LIVE_CONSENSUS);
     goto cannot;
   }
 
   /* Do we know enough router descriptors to have adequate vision of the HSDir
      hash ring? */
   if (!router_have_minimum_dir_info()) {
+    msg = tor_strdup("Not enough directory information");
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_NO_DIRINFO);
     goto cannot;
   }
 
   /* Can upload! */
   return 1;
+
  cannot:
+  tor_free(msg);
   return 0;
 }
 
-/* Refresh the given service descriptor meaning this will update every mutable
+/** Refresh the given service descriptor meaning this will update every mutable
  * field that needs to be updated before we upload.
  *
  * This should ONLY be called before uploading a descriptor. It assumes that
@@ -3150,7 +3246,7 @@ refresh_service_descriptor(const hs_service_t *service,
   set_descriptor_revision_counter(desc, now, service->desc_current == desc);
 }
 
-/* Scheduled event run from the main loop. Try to upload the descriptor for
+/** Scheduled event run from the main loop. Try to upload the descriptor for
  * each service. */
 STATIC void
 run_upload_descriptor_event(time_t now)
@@ -3199,7 +3295,7 @@ run_upload_descriptor_event(time_t now)
   consider_republishing_hs_descriptors = 0;
 }
 
-/* Called when the introduction point circuit is done building and ready to be
+/** Called when the introduction point circuit is done building and ready to be
  * used. */
 static void
 service_intro_circ_has_opened(origin_circuit_t *circ)
@@ -3257,7 +3353,7 @@ service_intro_circ_has_opened(origin_circuit_t *circ)
   return;
 }
 
-/* Called when a rendezvous circuit is done building and ready to be used. */
+/** Called when a rendezvous circuit is done building and ready to be used. */
 static void
 service_rendezvous_circ_has_opened(origin_circuit_t *circ)
 {
@@ -3298,7 +3394,7 @@ service_rendezvous_circ_has_opened(origin_circuit_t *circ)
   return;
 }
 
-/* We've been expecting an INTRO_ESTABLISHED cell on this circuit and it just
+/** We've been expecting an INTRO_ESTABLISHED cell on this circuit and it just
  * arrived. Handle the INTRO_ESTABLISHED cell arriving on the given
  * introduction circuit. Return 0 on success else a negative value. */
 static int
@@ -3341,11 +3437,6 @@ service_handle_intro_established(origin_circuit_t *circ,
     goto err;
   }
 
-  /* Flag that we have an established circuit for this intro point. This value
-   * is what indicates the upload scheduled event if we are ready to build the
-   * intro point into the descriptor and upload. */
-  ip->circuit_established = 1;
-
   log_info(LD_REND, "Successfully received an INTRO_ESTABLISHED cell "
                     "on circuit %u for service %s",
            TO_CIRCUIT(circ)->n_circ_id,
@@ -3356,7 +3447,7 @@ service_handle_intro_established(origin_circuit_t *circ,
   return -1;
 }
 
-/* We just received an INTRODUCE2 cell on the established introduction circuit
+/** We just received an INTRODUCE2 cell on the established introduction circuit
  * circ. Handle the cell and return 0 on success else a negative value. */
 static int
 service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
@@ -3394,7 +3485,7 @@ service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
 
   /* The following will parse, decode and launch the rendezvous point circuit.
    * Both current and legacy cells are handled. */
-  if (hs_circ_handle_introduce2(service, circ, ip, desc->desc->subcredential,
+  if (hs_circ_handle_introduce2(service, circ, ip, &desc->desc->subcredential,
                                 payload, payload_len) < 0) {
     goto err;
   }
@@ -3404,7 +3495,7 @@ service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
   return -1;
 }
 
-/* Add to list every filename used by service. This is used by the sandbox
+/** Add to list every filename used by service. This is used by the sandbox
  * subsystem. */
 static void
 service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list)
@@ -3426,7 +3517,7 @@ service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list)
   smartlist_add(list, hs_path_from_filename(s_dir, fname));
 }
 
-/* Return true iff the given service identity key is present on disk. */
+/** Return true iff the given service identity key is present on disk. */
 static int
 service_key_on_disk(const char *directory_path)
 {
@@ -3450,7 +3541,7 @@ service_key_on_disk(const char *directory_path)
   return ret;
 }
 
-/* This is a proxy function before actually calling hs_desc_encode_descriptor
+/** This is a proxy function before actually calling hs_desc_encode_descriptor
  * because we need some preprocessing here */
 static int
 service_encode_descriptor(const hs_service_t *service,
@@ -3481,7 +3572,7 @@ service_encode_descriptor(const hs_service_t *service,
 /* Public API */
 /* ========== */
 
-/* This is called everytime the service map (v2 or v3) changes that is if an
+/** This is called everytime the service map (v2 or v3) changes that is if an
  * element is added or removed. */
 void
 hs_service_map_has_changed(void)
@@ -3492,7 +3583,7 @@ hs_service_map_has_changed(void)
   rescan_periodic_events(get_options());
 }
 
-/* Upload an encoded descriptor in encoded_desc of the given version. This
+/** Upload an encoded descriptor in encoded_desc of the given version. This
  * descriptor is for the service identity_pk and blinded_pk used to setup the
  * directory connection identifier. It is uploaded to the directory hsdir_rs
  * routerstatus_t object.
@@ -3540,7 +3631,7 @@ hs_service_upload_desc_to_dir(const char *encoded_desc,
   directory_request_free(dir_req);
 }
 
-/* Add the ephemeral service using the secret key sk and ports. Both max
+/** Add the ephemeral service using the secret key sk and ports. Both max
  * streams parameter will be set in the newly created service.
  *
  * Ownership of sk and ports is passed to this routine.  Regardless of
@@ -3626,7 +3717,7 @@ hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
   return ret;
 }
 
-/* For the given onion address, delete the ephemeral service. Return 0 on
+/** For the given onion address, delete the ephemeral service. Return 0 on
  * success else -1 on error. */
 int
 hs_service_del_ephemeral(const char *address)
@@ -3676,7 +3767,7 @@ hs_service_del_ephemeral(const char *address)
   return -1;
 }
 
-/* Using the ed25519 public key pk, find a service for that key and return the
+/** Using the ed25519 public key pk, find a service for that key and return the
  * current encoded descriptor as a newly allocated string or NULL if not
  * found. This is used by the control port subsystem. */
 char *
@@ -3702,9 +3793,9 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk)
   return NULL;
 }
 
-/* Return the number of service we have configured and usable. */
-unsigned int
-hs_service_get_num_services(void)
+/** Return the number of service we have configured and usable. */
+MOCK_IMPL(unsigned int,
+hs_service_get_num_services,(void))
 {
   if (hs_service_map == NULL) {
     return 0;
@@ -3712,49 +3803,7 @@ hs_service_get_num_services(void)
   return HT_SIZE(hs_service_map);
 }
 
-/* Called once an introduction circuit is closed. If the circuit doesn't have
- * a v3 identifier, it is ignored. */
-void
-hs_service_intro_circ_has_closed(origin_circuit_t *circ)
-{
-  hs_service_t *service = NULL;
-  hs_service_intro_point_t *ip = NULL;
-  hs_service_descriptor_t *desc = NULL;
-
-  tor_assert(circ);
-
-  if (circ->hs_ident == NULL) {
-    /* This is not a v3 circuit, ignore. */
-    goto end;
-  }
-
-  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
-  if (service == NULL) {
-    /* This is possible if the circuits are closed and the service is
-     * immediately deleted. */
-    log_info(LD_REND, "Unable to find any hidden service associated "
-                      "identity key %s on intro circuit %u.",
-             ed25519_fmt(&circ->hs_ident->identity_pk),
-             TO_CIRCUIT(circ)->n_circ_id);
-    goto end;
-  }
-  if (ip == NULL) {
-    /* The introduction point object has already been removed probably by our
-     * cleanup process so ignore. */
-    goto end;
-  }
-  /* Can't have an intro point object without a descriptor. */
-  tor_assert(desc);
-
-  /* Circuit disappeared so make sure the intro point is updated. By
-   * keeping the object in the descriptor, we'll be able to retry. */
-  ip->circuit_established = 0;
-
- end:
-  return;
-}
-
-/* Given conn, a rendezvous edge connection acting as an exit stream, look up
+/** Given conn, a rendezvous edge connection acting as an exit stream, look up
  * the hidden service for the circuit circ, and look up the port and address
  * based on the connection port. Assign the actual connection address.
  *
@@ -3852,7 +3901,7 @@ hs_service_exports_circuit_id(const ed25519_public_key_t *pk)
   return service->config.circuit_id_protocol;
 }
 
-/* Add to file_list every filename used by a configured hidden service, and to
+/** Add to file_list every filename used by a configured hidden service, and to
  * dir_list every directory path used by a configured hidden service. This is
  * used by the sandbox subsystem to whitelist those. */
 void
@@ -3877,7 +3926,7 @@ hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Called when our internal view of the directory has changed. We might have
+/** Called when our internal view of the directory has changed. We might have
  * received a new batch of descriptors which might affect the shape of the
  * HSDir hash ring. Signal that we should reexamine the hash ring and
  * re-upload our HS descriptors if needed. */
@@ -3894,7 +3943,7 @@ hs_service_dir_info_changed(void)
   }
 }
 
-/* Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and
+/** Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and
  * launch a circuit to the rendezvous point. */
 int
 hs_service_receive_introduce2(origin_circuit_t *circ, const uint8_t *payload,
@@ -3925,7 +3974,7 @@ hs_service_receive_introduce2(origin_circuit_t *circ, const uint8_t *payload,
   return ret;
 }
 
-/* Called when we get an INTRO_ESTABLISHED cell. Mark the circuit as an
+/** Called when we get an INTRO_ESTABLISHED cell. Mark the circuit as an
  * established introduction point. Return 0 on success else a negative value
  * and the circuit is closed. */
 int
@@ -3962,7 +4011,7 @@ hs_service_receive_intro_established(origin_circuit_t *circ,
   return -1;
 }
 
-/* Called when any kind of hidden service circuit is done building thus
+/** Called when any kind of hidden service circuit is done building thus
  * opened. This is the entry point from the circuit subsystem. */
 void
 hs_service_circuit_has_opened(origin_circuit_t *circ)
@@ -3991,7 +4040,7 @@ hs_service_circuit_has_opened(origin_circuit_t *circ)
   }
 }
 
-/* Return the service version by looking at the key in the service directory.
+/** Return the service version by looking at the key in the service directory.
  * If the key is not found or unrecognized, -1 is returned. Else, the service
  * version is returned. */
 int
@@ -4021,7 +4070,7 @@ hs_service_get_version_from_key(const hs_service_t *service)
   return version;
 }
 
-/* Load and/or generate keys for all onion services including the client
+/** Load and/or generate keys for all onion services including the client
  * authorization if any. Return 0 on success, -1 on failure. */
 int
 hs_service_load_all_keys(void)
@@ -4057,7 +4106,51 @@ hs_service_load_all_keys(void)
   return -1;
 }
 
-/* Put all service object in the given service list. After this, the caller
+/** Log the status of introduction points for all version 3 onion services
+ * at log severity <b>severity</b>.
+ */
+void
+hs_service_dump_stats(int severity)
+{
+  origin_circuit_t *circ;
+
+  FOR_EACH_SERVICE_BEGIN(hs) {
+
+    tor_log(severity, LD_GENERAL, "Service configured in %s:",
+            service_escaped_dir(hs));
+    FOR_EACH_DESCRIPTOR_BEGIN(hs, desc) {
+
+      DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                           hs_service_intro_point_t *, ip) {
+        const node_t *intro_node;
+        const char *nickname;
+
+        intro_node = get_node_from_intro_point(ip);
+        if (!intro_node) {
+          tor_log(severity, LD_GENERAL, "  Couldn't find intro point, "
+                  "skipping");
+          continue;
+        }
+        nickname = node_get_nickname(intro_node);
+        if (!nickname) {
+          continue;
+        }
+
+        circ = hs_circ_service_get_intro_circ(ip);
+        if (!circ) {
+          tor_log(severity, LD_GENERAL, "  Intro point at %s: no circuit",
+                  nickname);
+          continue;
+        }
+        tor_log(severity, LD_GENERAL, "  Intro point %s: circuit is %s",
+                nickname, circuit_state_to_string(circ->base_.state));
+      } DIGEST256MAP_FOREACH_END;
+
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+}
+
+/** Put all service object in the given service list. After this, the caller
  * looses ownership of every elements in the list and responsible to free the
  * list pointer. */
 void
@@ -4074,7 +4167,7 @@ hs_service_stage_services(const smartlist_t *service_list)
   smartlist_add_all(hs_service_staging_list, service_list);
 }
 
-/* Allocate and initilize a service object. The service configuration will
+/** Allocate and initilize a service object. The service configuration will
  * contain the default values. Return the newly allocated object pointer. This
  * function can't fail. */
 hs_service_t *
@@ -4092,7 +4185,7 @@ hs_service_new(const or_options_t *options)
   return service;
 }
 
-/* Free the given <b>service</b> object and all its content. This function
+/** Free the given <b>service</b> object and all its content. This function
  * also takes care of wiping service keys from memory. It is safe to pass a
  * NULL pointer. */
 void
@@ -4115,13 +4208,18 @@ hs_service_free_(hs_service_t *service)
     replaycache_free(service->state.replay_cache_rend_cookie);
   }
 
+  /* Free onionbalance subcredentials (if any) */
+  if (service->state.ob_subcreds) {
+    tor_free(service->state.ob_subcreds);
+  }
+
   /* Wipe service keys. */
   memwipe(&service->keys.identity_sk, 0, sizeof(service->keys.identity_sk));
 
   tor_free(service);
 }
 
-/* Periodic callback. Entry point from the main loop to the HS service
+/** Periodic callback. Entry point from the main loop to the HS service
  * subsystem. This is call every second. This is skipped if tor can't build a
  * circuit or the network is disabled. */
 void
@@ -4144,7 +4242,7 @@ hs_service_run_scheduled_events(time_t now)
   run_upload_descriptor_event(now);
 }
 
-/* Initialize the service HS subsystem. */
+/** Initialize the service HS subsystem. */
 void
 hs_service_init(void)
 {
@@ -4161,24 +4259,25 @@ hs_service_init(void)
   hs_service_staging_list = smartlist_new();
 }
 
-/* Release all global storage of the hidden service subsystem. */
+/** Release all global storage of the hidden service subsystem. */
 void
 hs_service_free_all(void)
 {
   rend_service_free_all();
   service_free_all();
+  hs_config_free_all();
 }
 
 #ifdef TOR_UNIT_TESTS
 
-/* Return the global service map size. Only used by unit test. */
+/** Return the global service map size. Only used by unit test. */
 STATIC unsigned int
 get_hs_service_map_size(void)
 {
   return HT_SIZE(hs_service_map);
 }
 
-/* Return the staging list size. Only used by unit test. */
+/** Return the staging list size. Only used by unit test. */
 STATIC int
 get_hs_service_staging_list_size(void)
 {
diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h
index 5f43233ea1..b5bff5bee5 100644
--- a/src/feature/hs/hs_service.h
+++ b/src/feature/hs/hs_service.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,88 +21,92 @@
 /* Trunnel */
 #include "trunnel/hs/cell_establish_intro.h"
 
-/* When loading and configuring a service, this is the default version it will
+#include "ext/ht.h"
+
+/** When loading and configuring a service, this is the default version it will
  * be configured for as it is possible that no HiddenServiceVersion is
  * present. */
 #define HS_SERVICE_DEFAULT_VERSION HS_VERSION_THREE
 
-/* As described in the specification, service publishes their next descriptor
+/** As described in the specification, service publishes their next descriptor
  * at a random time between those two values (in seconds). */
 #define HS_SERVICE_NEXT_UPLOAD_TIME_MIN (60 * 60)
+/** Maximum interval for uploading next descriptor (in seconds). */
 #define HS_SERVICE_NEXT_UPLOAD_TIME_MAX (120 * 60)
 
-/* Service side introduction point. */
+/** Service side introduction point. */
 typedef struct hs_service_intro_point_t {
-  /* Top level intropoint "shared" data between client/service. */
+  /** Top level intropoint "shared" data between client/service. */
   hs_intropoint_t base;
 
-  /* Onion key of the introduction point used to extend to it for the ntor
+  /** Onion key of the introduction point used to extend to it for the ntor
    * handshake. */
   curve25519_public_key_t onion_key;
 
-  /* Authentication keypair used to create the authentication certificate
+  /** Authentication keypair used to create the authentication certificate
    * which is published in the descriptor. */
   ed25519_keypair_t auth_key_kp;
 
-  /* Encryption keypair for the "ntor" type. */
+  /** Encryption keypair for the "ntor" type. */
   curve25519_keypair_t enc_key_kp;
 
-  /* Legacy key if that intro point doesn't support v3. This should be used if
+  /** Legacy key if that intro point doesn't support v3. This should be used if
    * the base object legacy flag is set. */
   crypto_pk_t *legacy_key;
-  /* Legacy key SHA1 public key digest. This should be used only if the base
+  /** Legacy key SHA1 public key digest. This should be used only if the base
    * object legacy flag is set. */
   uint8_t legacy_key_digest[DIGEST_LEN];
 
-  /* Amount of INTRODUCE2 cell accepted from this intro point. */
+  /** Amount of INTRODUCE2 cell accepted from this intro point. */
   uint64_t introduce2_count;
 
-  /* Maximum number of INTRODUCE2 cell this intro point should accept. */
+  /** Maximum number of INTRODUCE2 cell this intro point should accept. */
   uint64_t introduce2_max;
 
-  /* The time at which this intro point should expire and stop being used. */
+  /** The time at which this intro point should expire and stop being used. */
   time_t time_to_expire;
 
-  /* The amount of circuit creation we've made to this intro point. This is
+  /** The amount of circuit creation we've made to this intro point. This is
    * incremented every time we do a circuit relaunch on this intro point which
    * is triggered when the circuit dies but the node is still in the
    * consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give up on it. */
   uint32_t circuit_retries;
 
-  /* Set if this intro point has an established circuit. */
-  unsigned int circuit_established : 1;
-
-  /* Replay cache recording the encrypted part of an INTRODUCE2 cell that the
+  /** Replay cache recording the encrypted part of an INTRODUCE2 cell that the
    * circuit associated with this intro point has received. This is used to
    * prevent replay attacks. */
   replaycache_t *replay_cache;
+
+  /** Support the INTRO2 DoS defense. If set, the DoS extension described by
+   * proposal 305 is sent. */
+  unsigned int support_intro2_dos_defense : 1;
 } hs_service_intro_point_t;
 
-/* Object handling introduction points of a service. */
+/** Object handling introduction points of a service. */
 typedef struct hs_service_intropoints_t {
-  /* The time at which we've started our retry period to build circuits. We
+  /** The time at which we've started our retry period to build circuits. We
    * don't want to stress circuit creation so we can only retry for a certain
    * time and then after we stop and wait. */
   time_t retry_period_started;
 
-  /* Number of circuit we've launched during a single retry period. */
+  /** Number of circuit we've launched during a single retry period. */
   unsigned int num_circuits_launched;
 
-  /* Contains the current hs_service_intro_point_t objects indexed by
+  /** Contains the current hs_service_intro_point_t objects indexed by
    * authentication public key. */
   digest256map_t *map;
 
-  /* Contains node's identity key digest that were introduction point for this
+  /** Contains node's identity key digest that were introduction point for this
    * descriptor but were retried to many times. We keep those so we avoid
    * re-picking them over and over for a circuit retry period.
    * XXX: Once we have #22173, change this to only use ed25519 identity. */
   digestmap_t *failed_id;
 } hs_service_intropoints_t;
 
-/* Representation of a service descriptor.
+/** Representation of a service descriptor.
  *
  * Some elements of the descriptor are mutable whereas others are immutable:
-
+ *
  * Immutable elements are initialized once when the descriptor is built (when
  * service descriptors gets rotated). This means that these elements are
  * initialized once and then they don't change for the lifetime of the
@@ -117,40 +121,42 @@ typedef struct hs_service_intropoints_t {
  * update_service_descriptor_intro_points().
  */
 typedef struct hs_service_descriptor_t {
-  /* Immutable: Client authorization ephemeral keypair. */
+  /** Immutable: Client authorization ephemeral keypair. */
   curve25519_keypair_t auth_ephemeral_kp;
 
-  /* Immutable: Descriptor cookie used to encrypt the descriptor, when the
+  /** Immutable: Descriptor cookie used to encrypt the descriptor, when the
    * client authorization is enabled */
   uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
 
-  /* Immutable: Descriptor signing keypair. */
+  /** Immutable: Descriptor signing keypair. */
   ed25519_keypair_t signing_kp;
 
-  /* Immutable: Blinded keypair derived from the master identity public key. */
+  /** Immutable: Blinded keypair derived from the master identity public
+   * key. */
   ed25519_keypair_t blinded_kp;
 
-  /* Immutable: The time period number this descriptor has been created for. */
+  /** Immutable: The time period number this descriptor has been created
+   * for. */
   uint64_t time_period_num;
 
   /** Immutable: The OPE cipher for encrypting revision counters for this
    *  descriptor.  Tied to the descriptor blinded key. */
   struct crypto_ope_t *ope_cipher;
 
-  /* Mutable: Decoded descriptor. This object is used for encoding when the
+  /** Mutable: Decoded descriptor. This object is used for encoding when the
    * service publishes the descriptor. */
   hs_descriptor_t *desc;
 
-  /* Mutable: When is the next time when we should upload the descriptor. */
+  /** Mutable: When is the next time when we should upload the descriptor. */
   time_t next_upload_time;
 
-  /* Mutable: Introduction points assign to this descriptor which contains
+  /** Mutable: Introduction points assign to this descriptor which contains
    * hs_service_intropoints_t object indexed by authentication key (the RSA key
    * if the node is legacy). */
   hs_service_intropoints_t intro_points;
 
-  /* Mutable: True iff we have missing intro points for this descriptor because
-   * we couldn't pick any nodes. */
+  /** Mutable: True iff we have missing intro points for this descriptor
+   * because we couldn't pick any nodes. */
   unsigned int missing_intro_points : 1;
 
   /** Mutable: List of the responsible HSDirs (their b64ed identity digest)
@@ -160,20 +166,20 @@ typedef struct hs_service_descriptor_t {
   smartlist_t *previous_hsdirs;
 } hs_service_descriptor_t;
 
-/* Service key material. */
+/** Service key material. */
 typedef struct hs_service_keys_t {
-  /* Master identify public key. */
+  /** Master identify public key. */
   ed25519_public_key_t identity_pk;
-  /* Master identity private key. */
+  /** Master identity private key. */
   ed25519_secret_key_t identity_sk;
-  /* True iff the key is kept offline which means the identity_sk MUST not be
+  /** True iff the key is kept offline which means the identity_sk MUST not be
    * used in that case. */
   unsigned int is_identify_key_offline : 1;
 } hs_service_keys_t;
 
 /** Service side configuration of client authorization. */
 typedef struct hs_service_authorized_client_t {
-  /* The client auth public key used to encrypt the descriptor cookie. */
+  /** The client auth public key used to encrypt the descriptor cookie. */
   curve25519_public_key_t client_pk;
 } hs_service_authorized_client_t;
 
@@ -186,115 +192,129 @@ typedef enum {
   HS_CIRCUIT_ID_PROTOCOL_HAPROXY
 } hs_circuit_id_protocol_t;
 
-/* Service configuration. The following are set from the torrc options either
+/** Service configuration. The following are set from the torrc options either
  * set by the configuration file or by the control port. Nothing else should
  * change those values. */
 typedef struct hs_service_config_t {
-  /* Protocol version of the service. Specified by HiddenServiceVersion
+  /** Protocol version of the service. Specified by HiddenServiceVersion
    * option. */
   uint32_t version;
 
-  /* Have we explicitly set HiddenServiceVersion? */
+  /** Have we explicitly set HiddenServiceVersion? */
   unsigned int hs_version_explicitly_set : 1;
 
-  /* List of rend_service_port_config_t */
+  /** List of rend_service_port_config_t */
   smartlist_t *ports;
 
-  /* Path on the filesystem where the service persistent data is stored. NULL
+  /** Path on the filesystem where the service persistent data is stored. NULL
    * if the service is ephemeral. Specified by HiddenServiceDir option. */
   char *directory_path;
 
-  /* The maximum number of simultaneous streams per rendezvous circuit that
+  /** The maximum number of simultaneous streams per rendezvous circuit that
    * are allowed to be created. No limit if 0. Specified by
    * HiddenServiceMaxStreams option. */
   uint64_t max_streams_per_rdv_circuit;
 
-  /* If true, we close circuits that exceed the max_streams_per_rdv_circuit
+  /** If true, we close circuits that exceed the max_streams_per_rdv_circuit
    * limit. Specified by HiddenServiceMaxStreamsCloseCircuit option. */
   unsigned int max_streams_close_circuit : 1;
 
-  /* How many introduction points this service has. Specified by
+  /** How many introduction points this service has. Specified by
    * HiddenServiceNumIntroductionPoints option. */
   unsigned int num_intro_points;
 
-  /* True iff the client auth is enabled. */
+  /** True iff the client auth is enabled. */
   unsigned int is_client_auth_enabled : 1;
 
-  /* List of hs_service_authorized_client_t's of clients that may access this
+  /** List of hs_service_authorized_client_t's of clients that may access this
    * service. Specified by HiddenServiceAuthorizeClient option. */
   smartlist_t *clients;
 
-  /* True iff we allow request made on unknown ports. Specified by
+  /** True iff we allow request made on unknown ports. Specified by
    * HiddenServiceAllowUnknownPorts option. */
   unsigned int allow_unknown_ports : 1;
 
-  /* If true, this service is a Single Onion Service. Specified by
+  /** If true, this service is a Single Onion Service. Specified by
    * HiddenServiceSingleHopMode and HiddenServiceNonAnonymousMode options. */
   unsigned int is_single_onion : 1;
 
-  /* If true, allow group read permissions on the directory_path. Specified by
+  /** If true, allow group read permissions on the directory_path. Specified by
    * HiddenServiceDirGroupReadable option. */
   unsigned int dir_group_readable : 1;
 
-  /* Is this service ephemeral? */
+  /** Is this service ephemeral? */
   unsigned int is_ephemeral : 1;
 
-  /* Does this service export the circuit ID of its clients? */
+  /** Does this service export the circuit ID of its clients? */
   hs_circuit_id_protocol_t circuit_id_protocol;
+
+  /** DoS defenses. For the ESTABLISH_INTRO cell extension. */
+  unsigned int has_dos_defense_enabled : 1;
+  uint32_t intro_dos_rate_per_sec;
+  uint32_t intro_dos_burst_per_sec;
+
+  /** If set, contains the Onion Balance master ed25519 public key (taken from
+   * an .onion addresses) that this tor instance serves as backend. */
+  smartlist_t *ob_master_pubkeys;
 } hs_service_config_t;
 
-/* Service state. */
+/** Service state. */
 typedef struct hs_service_state_t {
-  /* The time at which we've started our retry period to build circuits. We
+  /** The time at which we've started our retry period to build circuits. We
    * don't want to stress circuit creation so we can only retry for a certain
    * time and then after we stop and wait. */
   time_t intro_circ_retry_started_time;
 
-  /* Number of circuit we've launched during a single retry period. This
+  /** Number of circuit we've launched during a single retry period. This
    * should never go over MAX_INTRO_CIRCS_PER_PERIOD. */
   unsigned int num_intro_circ_launched;
 
-  /* Replay cache tracking the REND_COOKIE found in INTRODUCE2 cell to detect
+  /** Replay cache tracking the REND_COOKIE found in INTRODUCE2 cell to detect
    * repeats. Clients may send INTRODUCE1 cells for the same rendezvous point
    * through two or more different introduction points; when they do, this
    * keeps us from launching multiple simultaneous attempts to connect to the
    * same rend point. */
   replaycache_t *replay_cache_rend_cookie;
 
-  /* When is the next time we should rotate our descriptors. This is has to be
+  /** When is the next time we should rotate our descriptors. This is has to be
    * done at the start time of the next SRV protocol run. */
   time_t next_rotation_time;
+
+  /* If this is an onionbalance instance, this is an array of subcredentials
+   * that should be used when decrypting an INTRO2 cell. If this is not an
+   * onionbalance instance, this is NULL.
+   * See [ONIONBALANCE] section in rend-spec-v3.txt for more details . */
+  hs_subcredential_t *ob_subcreds;
+  /* Number of OB subcredentials */
+  size_t n_ob_subcreds;
 } hs_service_state_t;
 
-/* Representation of a service running on this tor instance. */
+/** Representation of a service running on this tor instance. */
 typedef struct hs_service_t {
-  /* Onion address base32 encoded and NUL terminated. We keep it for logging
+  /** Onion address base32 encoded and NUL terminated. We keep it for logging
    * purposes so we don't have to build it everytime. */
   char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
 
-  /* Hashtable node: use to look up the service by its master public identity
+  /** Hashtable node: use to look up the service by its master public identity
    * key in the service global map. */
   HT_ENTRY(hs_service_t) hs_service_node;
 
-  /* Service state which contains various flags and counters. */
+  /** Service state which contains various flags and counters. */
   hs_service_state_t state;
 
-  /* Key material of the service. */
+  /** Key material of the service. */
   hs_service_keys_t keys;
 
-  /* Configuration of the service. */
+  /** Configuration of the service. */
   hs_service_config_t config;
 
-  /* Current descriptor. */
+  /** Current descriptor. */
   hs_service_descriptor_t *desc_current;
-  /* Next descriptor. */
+  /** Next descriptor. */
   hs_service_descriptor_t *desc_next;
-
-  /* XXX: Credential (client auth.) #20700. */
-
 } hs_service_t;
 
-/* For the service global hash map, we define a specific type for it which
+/** For the service global hash map, we define a specific type for it which
  * will make it safe to use and specific to some controlled parameters such as
  * the hashing function and how to compare services. */
 typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht;
@@ -308,9 +328,14 @@ void hs_service_free_all(void);
 /* Service new/free functions. */
 hs_service_t *hs_service_new(const or_options_t *options);
 void hs_service_free_(hs_service_t *service);
+/**
+ * @copydoc hs_service_free_
+ *
+ * Additionally, set the pointer <b>s</b> to NULL.
+ **/
 #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s))
 
-unsigned int hs_service_get_num_services(void);
+MOCK_DECL(unsigned int, hs_service_get_num_services,(void));
 void hs_service_stage_services(const smartlist_t *service_list);
 int hs_service_load_all_keys(void);
 int hs_service_get_version_from_key(const hs_service_t *service);
@@ -330,8 +355,6 @@ int hs_service_receive_introduce2(origin_circuit_t *circ,
                                   const uint8_t *payload,
                                   size_t payload_len);
 
-void hs_service_intro_circ_has_closed(origin_circuit_t *circ);
-
 char *hs_service_lookup_current_desc(const ed25519_public_key_t *pk);
 
 hs_service_add_ephemeral_status_t
@@ -350,6 +373,8 @@ void hs_service_upload_desc_to_dir(const char *encoded_desc,
 hs_circuit_id_protocol_t
 hs_service_exports_circuit_id(const ed25519_public_key_t *pk);
 
+void hs_service_dump_stats(int severity);
+
 #ifdef HS_SERVICE_PRIVATE
 
 #ifdef TOR_UNIT_TESTS
@@ -361,7 +386,10 @@ STATIC hs_service_t *get_first_service(void);
 STATIC hs_service_intro_point_t *service_intro_point_find_by_ident(
                                          const hs_service_t *service,
                                          const hs_ident_circuit_t *ident);
-#endif
+
+MOCK_DECL(STATIC unsigned int, count_desc_circuit_established,
+          (const hs_service_descriptor_t *desc));
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /* Service accessors. */
 STATIC hs_service_t *find_service(hs_service_ht *map,
@@ -369,10 +397,7 @@ STATIC hs_service_t *find_service(hs_service_ht *map,
 STATIC void remove_service(hs_service_ht *map, hs_service_t *service);
 STATIC int register_service(hs_service_ht *map, hs_service_t *service);
 /* Service introduction point functions. */
-STATIC hs_service_intro_point_t *service_intro_point_new(
-                            const extend_info_t *ei,
-                            unsigned int is_legacy,
-                            unsigned int supports_ed25519_link_handshake_any);
+STATIC hs_service_intro_point_t *service_intro_point_new(const node_t *node);
 STATIC void service_intro_point_free_(hs_service_intro_point_t *ip);
 #define service_intro_point_free(ip)                            \
   FREE_AND_NULL(hs_service_intro_point_t,             \
diff --git a/src/feature/hs/hs_stats.c b/src/feature/hs/hs_stats.c
index f24b731328..f9d458d630 100644
--- a/src/feature/hs/hs_stats.c
+++ b/src/feature/hs/hs_stats.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h
index d89440faca..aea2ccf5c2 100644
--- a/src/feature/hs/hs_stats.h
+++ b/src/feature/hs/hs_stats.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,9 +6,13 @@
  * \brief Header file for hs_stats.c
  **/
 
+#ifndef TOR_HS_STATS_H
+#define TOR_HS_STATS_H
+
 void hs_stats_note_introduce2_cell(int is_hsv3);
 uint32_t hs_stats_get_n_introduce2_v3_cells(void);
 uint32_t hs_stats_get_n_introduce2_v2_cells(void);
 void hs_stats_note_service_rendezvous_launch(void);
 uint32_t hs_stats_get_n_rendezvous_launches(void);
 
+#endif /* !defined(TOR_HS_STATS_H) */
diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h
index 7d4116d8bb..6ce0bf5c69 100644
--- a/src/feature/hs/hsdir_index_st.h
+++ b/src/feature/hs/hsdir_index_st.h
@@ -1,24 +1,29 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file hsdir_index_st.h
+ * @brief HS directory index structure
+ **/
+
 #ifndef HSDIR_INDEX_ST_H
 #define HSDIR_INDEX_ST_H
 
-/* Hidden service directory index used in a node_t which is set once we set
+/** Hidden service directory index used in a node_t which is set once we set
  * the consensus. */
 struct hsdir_index_t {
-  /* HSDir index to use when fetching a descriptor. */
+  /** HSDir index to use when fetching a descriptor. */
   uint8_t fetch[DIGEST256_LEN];
 
-  /* HSDir index used by services to store their first and second
+  /** HSDir index used by services to store their first and second
    * descriptor. The first descriptor is chronologically older than the second
    * one and uses older TP and SRV values. */
   uint8_t store_first[DIGEST256_LEN];
+  /** Newer index, for second descriptor. */
   uint8_t store_second[DIGEST256_LEN];
 };
 
-#endif
-
+#endif /* !defined(HSDIR_INDEX_ST_H) */
diff --git a/src/feature/hs/include.am b/src/feature/hs/include.am
new file mode 100644
index 0000000000..af1dc65585
--- /dev/null
+++ b/src/feature/hs/include.am
@@ -0,0 +1,39 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/hs/hs_cache.c		\
+	src/feature/hs/hs_cell.c		\
+	src/feature/hs/hs_circuit.c		\
+	src/feature/hs/hs_circuitmap.c		\
+	src/feature/hs/hs_client.c		\
+	src/feature/hs/hs_common.c		\
+	src/feature/hs/hs_config.c		\
+	src/feature/hs/hs_control.c		\
+	src/feature/hs/hs_descriptor.c		\
+	src/feature/hs/hs_dos.c			\
+	src/feature/hs/hs_ident.c		\
+	src/feature/hs/hs_intropoint.c		\
+	src/feature/hs/hs_ob.c			\
+	src/feature/hs/hs_service.c		\
+	src/feature/hs/hs_stats.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/hs/hs_cache.h			\
+	src/feature/hs/hs_cell.h			\
+	src/feature/hs/hs_circuit.h			\
+	src/feature/hs/hs_circuitmap.h			\
+	src/feature/hs/hs_client.h			\
+	src/feature/hs/hs_common.h			\
+	src/feature/hs/hs_config.h			\
+	src/feature/hs/hs_control.h			\
+	src/feature/hs/hs_descriptor.h			\
+	src/feature/hs/hs_dos.h				\
+	src/feature/hs/hs_ident.h			\
+	src/feature/hs/hs_intropoint.h			\
+	src/feature/hs/hs_ob.h				\
+	src/feature/hs/hs_opts_st.h			\
+	src/feature/hs/hs_options.inc			\
+	src/feature/hs/hs_service.h			\
+	src/feature/hs/hs_stats.h			\
+	src/feature/hs/hsdir_index_st.h
diff --git a/src/feature/hs_common/.may_include b/src/feature/hs_common/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/hs_common/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/hs_common/feature_hs_common.md b/src/feature/hs_common/feature_hs_common.md
new file mode 100644
index 0000000000..3a5e351a0a
--- /dev/null
+++ b/src/feature/hs_common/feature_hs_common.md
@@ -0,0 +1,3 @@
+@dir /feature/hs_common
+@brief feature/hs_common: Common to v2 (old) and v3 (current) onion services
+
diff --git a/src/feature/hs_common/include.am b/src/feature/hs_common/include.am
new file mode 100644
index 0000000000..3bb9225c12
--- /dev/null
+++ b/src/feature/hs_common/include.am
@@ -0,0 +1,10 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/hs_common/replaycache.c	        \
+	src/feature/hs_common/shared_random_client.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/hs_common/replaycache.h		\
+	src/feature/hs_common/shared_random_client.h
diff --git a/src/feature/hs_common/replaycache.c b/src/feature/hs_common/replaycache.c
index 9e8c13b1c5..ab058ce759 100644
--- a/src/feature/hs_common/replaycache.c
+++ b/src/feature/hs_common/replaycache.c
@@ -1,4 +1,4 @@
- /* Copyright (c) 2012-2019, The Tor Project, Inc. */
+ /* Copyright (c) 2012-2020, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/hs_common/replaycache.h b/src/feature/hs_common/replaycache.h
index 01f5e600c2..3a3eed29c0 100644
--- a/src/feature/hs_common/replaycache.h
+++ b/src/feature/hs_common/replaycache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,16 +14,16 @@ typedef struct replaycache_t replaycache_t;
 #ifdef REPLAYCACHE_PRIVATE
 
 struct replaycache_t {
-  /* Scrub interval */
+  /** Scrub interval */
   time_t scrub_interval;
-  /* Last scrubbed */
+  /** Last scrubbed */
   time_t scrubbed;
-  /*
+  /**
    * Horizon
    * (don't return true on digests in the cache but older than this)
    */
   time_t horizon;
-  /*
+  /**
    * Digest map: keys are digests, values are times the digest was last seen
    */
   digest256map_t *digests_seen;
@@ -34,6 +34,11 @@ struct replaycache_t {
 /* replaycache_t free/new */
 
 void replaycache_free_(replaycache_t *r);
+/**
+ * @copydoc replaycache_free_
+ *
+ * Additionally, set the pointer <b>r</b> to NULL.
+ **/
 #define replaycache_free(r) \
   FREE_AND_NULL(replaycache_t, replaycache_free_, (r))
 replaycache_t * replaycache_new(time_t horizon, time_t interval);
diff --git a/src/feature/hs_common/shared_random_client.c b/src/feature/hs_common/shared_random_client.c
index ead5d681a9..4e8a2942fc 100644
--- a/src/feature/hs_common/shared_random_client.c
+++ b/src/feature/hs_common/shared_random_client.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,18 +8,18 @@
  *        as part of the dirauth module.
  **/
 
-#define SHARED_RANDOM_CLIENT_PRIVATE
 #include "feature/hs_common/shared_random_client.h"
 
 #include "app/config/config.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "lib/encoding/binascii.h"
 
 #include "feature/nodelist/networkstatus_st.h"
 
-/* Convert a given srv object to a string for the control port. This doesn't
+/** Convert a given srv object to a string for the control port. This doesn't
  * fail and the srv object MUST be valid. */
 static char *
 srv_to_control_string(const sr_srv_t *srv)
@@ -33,7 +33,25 @@ srv_to_control_string(const sr_srv_t *srv)
   return srv_str;
 }
 
-/* Return the voting interval of the tor vote subsystem. */
+/**
+ * If we have no consensus and we are not an authority, assume that this is
+ * the voting interval.  We should never actually use this: only authorities
+ * should be trying to figure out the schedule when they don't have a
+ * consensus.
+ **/
+#define DEFAULT_NETWORK_VOTING_INTERVAL (3600)
+
+/* This is an unpleasing workaround for tests.  Our unit tests assume that we
+ * are scheduling all of our shared random stuff as if we were a directory
+ * authority, but they do not always set V3AuthoritativeDir.
+ */
+#ifdef TOR_UNIT_TESTS
+#define ASSUME_AUTHORITY_SCHEDULING 1
+#else
+#define ASSUME_AUTHORITY_SCHEDULING 0
+#endif
+
+/** Return the voting interval of the tor vote subsystem. */
 int
 get_voting_interval(void)
 {
@@ -43,45 +61,32 @@ get_voting_interval(void)
                                                 usable_consensus_flavor());
 
   if (consensus) {
+    /* Ideally we have a live consensus and we can just use that. */
+    interval = (int)(consensus->fresh_until - consensus->valid_after);
+  } else if (authdir_mode(get_options()) || ASSUME_AUTHORITY_SCHEDULING) {
+    /* If we don't have a live consensus and we're an authority,
+     * we should believe our own view of what the schedule ought to be. */
+    interval = dirauth_sched_get_configured_interval();
+  } else if ((consensus = networkstatus_get_latest_consensus())) {
+    /* If we're a client, then maybe a latest consensus is good enough?
+     * It's better than falling back to the non-consensus case. */
     interval = (int)(consensus->fresh_until - consensus->valid_after);
   } else {
-    /* Same for both a testing and real network. We voluntarily ignore the
-     * InitialVotingInterval since it complexifies things and it doesn't
-     * affect the SR protocol. */
-    interval = get_options()->V3AuthVotingInterval;
+    /* We should never be reaching this point, since a client should never
+     * call this code unless they have some kind of a consensus. All we can
+     * do is hope that this network is using the default voting interval. */
+    tor_assert_nonfatal_unreached_once();
+    interval = DEFAULT_NETWORK_VOTING_INTERVAL;
   }
   tor_assert(interval > 0);
   return interval;
 }
 
-/* Given the current consensus, return the start time of the current round of
- * the SR protocol. For example, if it's 23:47:08, the current round thus
- * started at 23:47:00 for a voting interval of 10 seconds.
- *
- * This function uses the consensus voting schedule to derive its results,
- * instead of the actual consensus we are currently using, so it should be used
- * for voting purposes. */
-time_t
-get_start_time_of_current_round(void)
-{
-  const or_options_t *options = get_options();
-  int voting_interval = get_voting_interval();
-  /* First, get the start time of the next round */
-  time_t next_start = voting_schedule_get_next_valid_after_time();
-  /* Now roll back next_start by a voting interval to find the start time of
-     the current round. */
-  time_t curr_start = voting_schedule_get_start_of_next_interval(
-                                     next_start - voting_interval - 1,
-                                     voting_interval,
-                                     options->TestingV3AuthVotingStartOffset);
-  return curr_start;
-}
-
 /*
  * Public API
  */
 
-/* Encode the given shared random value and put it in dst. Destination
+/** Encode the given shared random value and put it in dst. Destination
  * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */
 void
 sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
@@ -102,7 +107,7 @@ sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
   strlcpy(dst, buf, dst_len);
 }
 
-/* Return the current SRV string representation for the control port. Return a
+/** Return the current SRV string representation for the control port. Return a
  * newly allocated string on success containing the value else "" if not found
  * or if we don't have a valid consensus yet. */
 char *
@@ -118,7 +123,7 @@ sr_get_current_for_control(void)
   return srv_str;
 }
 
-/* Return the previous SRV string representation for the control port. Return
+/** Return the previous SRV string representation for the control port. Return
  * a newly allocated string on success containing the value else "" if not
  * found or if we don't have a valid consensus yet. */
 char *
@@ -134,7 +139,7 @@ sr_get_previous_for_control(void)
   return srv_str;
 }
 
-/* Return current shared random value from the latest consensus. Caller can
+/** Return current shared random value from the latest consensus. Caller can
  * NOT keep a reference to the returned pointer. Return NULL if none. */
 const sr_srv_t *
 sr_get_current(const networkstatus_t *ns)
@@ -158,7 +163,7 @@ sr_get_current(const networkstatus_t *ns)
   return NULL;
 }
 
-/* Return previous shared random value from the latest consensus. Caller can
+/** Return previous shared random value from the latest consensus. Caller can
  * NOT keep a reference to the returned pointer. Return NULL if none. */
 const sr_srv_t *
 sr_get_previous(const networkstatus_t *ns)
@@ -182,7 +187,7 @@ sr_get_previous(const networkstatus_t *ns)
   return NULL;
 }
 
-/* Parse a list of arguments from a SRV value either from a vote, consensus
+/** Parse a list of arguments from a SRV value either from a vote, consensus
  * or from our disk state and return a newly allocated srv object. NULL is
  * returned on error.
  *
@@ -252,8 +257,19 @@ sr_state_get_start_time_of_current_protocol_run(void)
                                                 usable_consensus_flavor());
   if (ns) {
     beginning_of_curr_round = ns->valid_after;
+  } else if (authdir_mode(get_options()) || ASSUME_AUTHORITY_SCHEDULING) {
+    beginning_of_curr_round = dirauth_sched_get_cur_valid_after_time();
   } else {
-    beginning_of_curr_round = get_start_time_of_current_round();
+    /* voting_interval comes from get_voting_interval(), so if we're in
+     * this case as a client, we already tried to get the voting interval
+     * from the latest_consensus and gave a bug warning if we couldn't.
+     *
+     * We wouldn't want to look at the latest consensus's valid_after time,
+     * since that would be out of date. */
+    beginning_of_curr_round = voting_sched_get_start_of_interval_after(
+                                             approx_time() - voting_interval,
+                                             voting_interval,
+                                             0);
   }
 
   /* Get current SR protocol round */
@@ -295,4 +311,3 @@ sr_state_get_protocol_run_duration(void)
   int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
   return total_protocol_rounds * get_voting_interval();
 }
-
diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h
index 95fe2c65ab..37a086d590 100644
--- a/src/feature/hs_common/shared_random_client.h
+++ b/src/feature/hs_common/shared_random_client.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,11 +38,9 @@ time_t sr_state_get_start_time_of_current_protocol_run(void);
 time_t sr_state_get_start_time_of_previous_protocol_run(void);
 unsigned int sr_state_get_phase_duration(void);
 unsigned int sr_state_get_protocol_run_duration(void);
-time_t get_start_time_of_current_round(void);
 
 #ifdef TOR_UNIT_TESTS
 
 #endif /* TOR_UNIT_TESTS */
 
-#endif /* TOR_SHARED_RANDOM_CLIENT_H */
-
+#endif /* !defined(TOR_SHARED_RANDOM_CLIENT_H) */
diff --git a/src/feature/keymgt/.may_include b/src/feature/keymgt/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/keymgt/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/keymgt/feature_keymgt.md b/src/feature/keymgt/feature_keymgt.md
new file mode 100644
index 0000000000..1eac7cca50
--- /dev/null
+++ b/src/feature/keymgt/feature_keymgt.md
@@ -0,0 +1,3 @@
+@dir /feature/keymgt
+@brief feature/keymgt: Store keys for relays, authorities, etc.
+
diff --git a/src/feature/keymgt/include.am b/src/feature/keymgt/include.am
new file mode 100644
index 0000000000..bc9beaa523
--- /dev/null
+++ b/src/feature/keymgt/include.am
@@ -0,0 +1,8 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/keymgt/loadkey.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/keymgt/loadkey.h
diff --git a/src/feature/keymgt/loadkey.c b/src/feature/keymgt/loadkey.c
index a8cbf0e582..7958bd964f 100644
--- a/src/feature/keymgt/loadkey.c
+++ b/src/feature/keymgt/loadkey.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,7 +33,7 @@
 /** Try to read an RSA key from <b>fname</b>.  If <b>fname</b> doesn't exist
  * and <b>generate</b> is true, create a new RSA key and save it in
  * <b>fname</b>.  Return the read/created key, or NULL on error.  Log all
- * errors at level <b>severity</b>. If <b>created_out/b> is non-NULL and a
+ * errors at level <b>severity</b>. If <b>created_out</b> is non-NULL and a
  * new key was created, set *<b>created_out</b> to true.
  */
 crypto_pk_t *
diff --git a/src/feature/keymgt/loadkey.h b/src/feature/keymgt/loadkey.h
index 8beee57a20..5a8ca32dea 100644
--- a/src/feature/keymgt/loadkey.h
+++ b/src/feature/keymgt/loadkey.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -52,4 +52,4 @@ int read_encrypted_secret_key(ed25519_secret_key_t *out,
 int write_encrypted_secret_key(const ed25519_secret_key_t *out,
                                const char *fname);
 
-#endif
+#endif /* !defined(TOR_LOADKEY_H) */
diff --git a/src/feature/nodelist/.may_include b/src/feature/nodelist/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/nodelist/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c
index 7a065662a7..97e44d53e3 100644
--- a/src/feature/nodelist/authcert.c
+++ b/src/feature/nodelist/authcert.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -46,7 +46,7 @@
 #include "feature/nodelist/networkstatus_voter_info_st.h"
 #include "feature/nodelist/node_st.h"
 
-DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
+DECLARE_TYPED_DIGESTMAP_FNS(dsmap, digest_ds_map_t, download_status_t)
 #define DSMAP_FOREACH(map, keyvar, valvar) \
   DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
                     valvar)
@@ -380,7 +380,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
   int added_trusted_cert = 0;
 
   for (s = contents; *s; s = eos) {
-    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s),
+                                                              &eos);
     cert_list_t *cl;
     if (!cert) {
       failure_code = -1;
@@ -463,11 +464,13 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
           (ds->addr != cert->addr ||
            ds->dir_port != cert->dir_port)) {
         char *a = tor_dup_ip(cert->addr);
-        log_notice(LD_DIR, "Updating address for directory authority %s "
-                   "from %s:%d to %s:%d based on certificate.",
-                   ds->nickname, ds->address, (int)ds->dir_port,
-                   a, cert->dir_port);
-        tor_free(a);
+        if (a) {
+          log_notice(LD_DIR, "Updating address for directory authority %s "
+                     "from %s:%d to %s:%d based on certificate.",
+                     ds->nickname, ds->address, (int)ds->dir_port,
+                     a, cert->dir_port);
+          tor_free(a);
+        }
         ds->addr = cert->addr;
         ds->dir_port = cert->dir_port;
       }
diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h
index 2effdb06e6..33065589ba 100644
--- a/src/feature/nodelist/authcert.h
+++ b/src/feature/nodelist/authcert.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -57,4 +57,4 @@ MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
 
 void authcert_free_all(void);
 
-#endif
+#endif /* !defined(TOR_AUTHCERT_H) */
diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h
index 68a84bc452..9145b12bbf 100644
--- a/src/feature/nodelist/authority_cert_st.h
+++ b/src/feature/nodelist/authority_cert_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file authority_cert_st.h
+ * @brief Authority certificate structure.
+ **/
+
 #ifndef AUTHORITY_CERT_ST_H
 #define AUTHORITY_CERT_ST_H
 
@@ -28,5 +33,4 @@ struct authority_cert_t {
   uint16_t dir_port;
 };
 
-#endif
-
+#endif /* !defined(AUTHORITY_CERT_ST_H) */
diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h
index b04a1abc7d..5f35a490a5 100644
--- a/src/feature/nodelist/desc_store_st.h
+++ b/src/feature/nodelist/desc_store_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file desc_store_st.h
+ * @brief Routerinfo/extrainfo storage structure.
+ **/
+
 #ifndef DESC_STORE_ST_H
 #define DESC_STORE_ST_H
 
@@ -36,4 +41,4 @@ struct desc_store_t {
   size_t bytes_dropped;
 };
 
-#endif
+#endif /* !defined(DESC_STORE_ST_H) */
diff --git a/src/feature/nodelist/describe.c b/src/feature/nodelist/describe.c
index 5c376408c0..00896d5a44 100644
--- a/src/feature/nodelist/describe.c
+++ b/src/feature/nodelist/describe.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,66 +9,108 @@
  * \brief Format short descriptions of relays.
  */
 
+#define DESCRIBE_PRIVATE
+
 #include "core/or/or.h"
 #include "feature/nodelist/describe.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "core/or/extend_info_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
-
-/**
- * Longest allowed output of format_node_description, plus 1 character for
- * NUL.  This allows space for:
- * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
- * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
- * plus a terminating NUL.
- */
-#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+#include "feature/nodelist/microdesc_st.h"
 
 /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
  * hold a human-readable description of a node with identity digest
- * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
- * and address <b>addr</b> or <b>addr32h</b>.
+ * <b>id_digest</b>, nickname <b>nickname</b>, and addresses <b>addr32h</b> and
+ * <b>addr</b>.
  *
  * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
- * NULL.  The <b>addr32h</b> field is optional and may be set to 0.
+ * NULL or the null address.  The <b>addr32h</b> field is optional and may be
+ * set to 0.
  *
  * Return a pointer to the front of <b>buf</b>.
+ * If buf is NULL, return a string constant describing the error.
  */
-static const char *
+STATIC const char *
 format_node_description(char *buf,
                         const char *id_digest,
-                        int is_named,
                         const char *nickname,
                         const tor_addr_t *addr,
                         uint32_t addr32h)
 {
-  char *cp;
+  size_t rv = 0;
+  bool has_addr = addr && !tor_addr_is_null(addr);
 
   if (!buf)
     return "<NULL BUFFER>";
 
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
-  cp = buf+1+HEX_DIGEST_LEN;
+  memset(buf, 0, NODE_DESC_BUF_LEN);
+
+  if (!id_digest) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<NULL ID DIGEST>", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    return buf;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", NODE_DESC_BUF_LEN);
+  tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  id_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   if (nickname) {
-    buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
-    strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
-    cp += strlen(cp);
+    rv = strlcat(buf, "~", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    rv = strlcat(buf, nickname, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
-  if (addr32h || addr) {
-    memcpy(cp, " at ", 4);
-    cp += 4;
-    if (addr) {
-      tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
-    } else {
-      struct in_addr in;
-      in.s_addr = htonl(addr32h);
-      tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
-    }
+  if (addr32h || has_addr) {
+    rv = strlcat(buf, " at ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
+  if (addr32h) {
+    int ntoa_rv = 0;
+    char ipv4_addr_str[INET_NTOA_BUF_LEN];
+    memset(ipv4_addr_str, 0, sizeof(ipv4_addr_str));
+    struct in_addr in;
+    memset(&in, 0, sizeof(in));
+
+    in.s_addr = htonl(addr32h);
+    ntoa_rv = tor_inet_ntoa(&in, ipv4_addr_str, sizeof(ipv4_addr_str));
+    tor_assert_nonfatal(ntoa_rv >= 0);
+
+    rv = strlcat(buf, ipv4_addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  /* Both addresses are valid */
+  if (addr32h && has_addr) {
+    rv = strlcat(buf, " and ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  if (has_addr) {
+    const char *str_rv = NULL;
+    char addr_str[TOR_ADDR_BUF_LEN];
+    memset(addr_str, 0, sizeof(addr_str));
+
+    str_rv = tor_addr_to_str(addr_str, addr, sizeof(addr_str), 1);
+    tor_assert_nonfatal(str_rv == addr_str);
+
+    rv = strlcat(buf, addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   return buf;
 }
 
@@ -84,11 +126,11 @@ router_describe(const routerinfo_t *ri)
 
   if (!ri)
     return "<null>";
+
   return format_node_description(buf,
                                  ri->cache_info.identity_digest,
-                                 0,
                                  ri->nickname,
-                                 NULL,
+                                 &ri->ipv6_addr,
                                  ri->addr);
 }
 
@@ -103,25 +145,33 @@ node_describe(const node_t *node)
   static char buf[NODE_DESC_BUF_LEN];
   const char *nickname = NULL;
   uint32_t addr32h = 0;
-  int is_named = 0;
+  const tor_addr_t *ipv6_addr = NULL;
 
   if (!node)
     return "<null>";
 
   if (node->rs) {
     nickname = node->rs->nickname;
-    is_named = node->rs->is_named;
     addr32h = node->rs->addr;
+    ipv6_addr = &node->rs->ipv6_addr;
+    /* Support consensus versions less than 28, when IPv6 addresses were in
+     * microdescs. This code can be removed when 0.2.9 is no longer supported,
+     * and the MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC macro is removed. */
+    if (node->md && tor_addr_is_null(ipv6_addr)) {
+      ipv6_addr = &node->md->ipv6_addr;
+    }
   } else if (node->ri) {
     nickname = node->ri->nickname;
     addr32h = node->ri->addr;
+    ipv6_addr = &node->ri->ipv6_addr;
+  } else {
+    return "<null rs and ri>";
   }
 
   return format_node_description(buf,
                                  node->identity,
-                                 is_named,
                                  nickname,
-                                 NULL,
+                                 ipv6_addr,
                                  addr32h);
 }
 
@@ -137,11 +187,11 @@ routerstatus_describe(const routerstatus_t *rs)
 
   if (!rs)
     return "<null>";
+
   return format_node_description(buf,
                                  rs->identity_digest,
-                                 rs->is_named,
                                  rs->nickname,
-                                 NULL,
+                                 &rs->ipv6_addr,
                                  rs->addr);
 }
 
@@ -157,9 +207,9 @@ extend_info_describe(const extend_info_t *ei)
 
   if (!ei)
     return "<null>";
+
   return format_node_description(buf,
                                  ei->identity_digest,
-                                 0,
                                  ei->nickname,
                                  &ei->addr,
                                  0);
@@ -175,9 +225,39 @@ extend_info_describe(const extend_info_t *ei)
 void
 router_get_verbose_nickname(char *buf, const routerinfo_t *router)
 {
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
-                DIGEST_LEN);
-  buf[1+HEX_DIGEST_LEN] = '~';
-  strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
+  size_t rv = 0;
+
+  if (!buf)
+    return;
+
+  memset(buf, 0, MAX_VERBOSE_NICKNAME_LEN+1);
+
+  if (!router) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<null>", MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+    return;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  router->cache_info.identity_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+  }
+
+  rv = strlcat(buf, "~", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  rv = strlcat(buf, router->nickname, MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
 }
diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h
index 018af6470e..d0fa1af263 100644
--- a/src/feature/nodelist/describe.h
+++ b/src/feature/nodelist/describe.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,36 @@ const char *node_describe(const struct node_t *node);
 const char *router_describe(const struct routerinfo_t *ri);
 const char *routerstatus_describe(const struct routerstatus_t *ri);
 
-#endif
+void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
+
+#if defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS)
+
+/**
+ * Longest allowed output for an IPv4 address "255.255.255.255", with NO
+ * terminating NUL.
+ */
+#define IPV4_BUF_LEN_NO_NUL 15
+
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL.  This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " 255.255.255.255 and [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN \
+  (MAX_VERBOSE_NICKNAME_LEN+4+IPV4_BUF_LEN_NO_NUL+5+TOR_ADDR_BUF_LEN)
+
+#endif /* defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS) */
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC const char *format_node_description(char *buf,
+                                           const char *id_digest,
+                                           const char *nickname,
+                                           const tor_addr_t *addr,
+                                           uint32_t addr32h);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_DESCRIBE_H) */
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
index b4abffad67..4317491043 100644
--- a/src/feature/nodelist/dirlist.c
+++ b/src/feature/nodelist/dirlist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,8 +27,9 @@
 #include "core/or/or.h"
 
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/or/policies.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/dirlist.h"
@@ -70,7 +71,7 @@ add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir)
 }
 
 /** Go over the trusted directory server list and add their address(es) to the
- * nodelist address set. This is called every time a new consensus is set. */
+ * nodelist address set. This is called everytime a new consensus is set. */
 MOCK_IMPL(void,
 dirlist_add_trusted_dir_addresses, (void))
 {
@@ -362,6 +363,9 @@ trusted_dir_server_new(const char *nickname, const char *address,
     }
     if (!hostname)
       hostname = tor_dup_ip(a);
+
+    if (!hostname)
+      return NULL;
   } else {
     if (tor_lookup_hostname(address, &a)) {
       log_warn(LD_CONFIG,
diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h
index 527af35427..9201e76a9c 100644
--- a/src/feature/nodelist/dirlist.h
+++ b/src/feature/nodelist/dirlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -46,4 +46,4 @@ void dirlist_free_all(void);
 
 MOCK_DECL(void, dirlist_add_trusted_dir_addresses, (void));
 
-#endif
+#endif /* !defined(TOR_DIRLIST_H) */
diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h
index 66e32c422f..4bde9d89ec 100644
--- a/src/feature/nodelist/document_signature_st.h
+++ b/src/feature/nodelist/document_signature_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file document_signature_st.h
+ * @brief Authority signature structure
+ **/
+
 #ifndef DOCUMENT_SIGNATURE_ST_H
 #define DOCUMENT_SIGNATURE_ST_H
 
@@ -25,5 +30,4 @@ struct document_signature_t {
                                      * as good. */
 };
 
-#endif
-
+#endif /* !defined(DOCUMENT_SIGNATURE_ST_H) */
diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h
index c54277b05e..6bd6232cd8 100644
--- a/src/feature/nodelist/extrainfo_st.h
+++ b/src/feature/nodelist/extrainfo_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file extrainfo_st.h
+ * @brief A relay's extra-info structure.
+ **/
+
 #ifndef EXTRAINFO_ST_H
 #define EXTRAINFO_ST_H
 
@@ -26,5 +31,4 @@ struct extrainfo_t {
   size_t pending_sig_len;
 };
 
-#endif
-
+#endif /* !defined(EXTRAINFO_ST_H) */
diff --git a/src/feature/nodelist/feature_nodelist.md b/src/feature/nodelist/feature_nodelist.md
new file mode 100644
index 0000000000..9d715308c2
--- /dev/null
+++ b/src/feature/nodelist/feature_nodelist.md
@@ -0,0 +1,2 @@
+@dir /feature/nodelist
+@brief feature/nodelist: Download and manage a list of relays
diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c
index 75cab7a0af..ca4a312639 100644
--- a/src/feature/nodelist/fmt_routerstatus.c
+++ b/src/feature/nodelist/fmt_routerstatus.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,64 +14,19 @@
 #include "core/or/or.h"
 #include "feature/nodelist/fmt_routerstatus.h"
 
-/* #include "lib/container/buffers.h" */
-/* #include "app/config/config.h" */
-/* #include "app/config/confparse.h" */
-/* #include "core/or/channel.h" */
-/* #include "core/or/channeltls.h" */
-/* #include "core/or/command.h" */
-/* #include "core/mainloop/connection.h" */
-/* #include "core/or/connection_or.h" */
-/* #include "feature/dircache/conscache.h" */
-/* #include "feature/dircache/consdiffmgr.h" */
-/* #include "feature/control/control.h" */
-/* #include "feature/dircache/directory.h" */
-/* #include "feature/dircache/dirserv.h" */
-/* #include "feature/hibernate/hibernate.h" */
-/* #include "feature/dirauth/keypin.h" */
-/* #include "core/mainloop/mainloop.h" */
-/* #include "feature/nodelist/microdesc.h" */
-/* #include "feature/nodelist/networkstatus.h" */
-/* #include "feature/nodelist/nodelist.h" */
 #include "core/or/policies.h"
-/* #include "core/or/protover.h" */
-/* #include "feature/stats/rephist.h" */
-/* #include "feature/relay/router.h" */
-/* #include "feature/nodelist/dirlist.h" */
 #include "feature/nodelist/routerlist.h"
-
-/* #include "feature/nodelist/routerparse.h" */
-/* #include "feature/nodelist/routerset.h" */
-/* #include "feature/nodelist/torcert.h" */
-/* #include "feature/dircommon/voting_schedule.h" */
-
 #include "feature/dirauth/dirvote.h"
 
-/* #include "feature/dircache/cached_dir_st.h" */
-/* #include "feature/dircommon/dir_connection_st.h" */
-/* #include "feature/nodelist/extrainfo_st.h" */
-/* #include "feature/nodelist/microdesc_st.h" */
-/* #include "feature/nodelist/node_st.h" */
 #include "feature/nodelist/routerinfo_st.h"
-/* #include "feature/nodelist/routerlist_st.h" */
-/* #include "core/or/tor_version_st.h" */
 #include "feature/nodelist/vote_routerstatus_st.h"
 
-/* #include "lib/compress/compress.h" */
-/* #include "lib/container/order.h" */
 #include "lib/crypt_ops/crypto_format.h"
-/* #include "lib/encoding/confline.h" */
-
-/* #include "lib/encoding/keyval.h" */
 
 /** Helper: write the router-status information in <b>rs</b> into a newly
  * allocated character buffer.  Use the same format as in network-status
  * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
  *
- * consensus_method is the current consensus method when format is
- * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other
- * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD.
- *
  * Return 0 on success, -1 on failure.
  *
  * The format argument has one of the following values:
@@ -88,7 +43,6 @@ char *
 routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                           const char *protocols,
                           routerstatus_format_type_t format,
-                          int consensus_method,
                           const vote_routerstatus_t *vrs)
 {
   char *summary;
@@ -99,6 +53,10 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
   char digest64[BASE64_DIGEST_LEN+1];
   smartlist_t *chunks = smartlist_new();
 
+  const char *ip_str = fmt_addr32(rs->addr);
+  if (ip_str[0] == '\0')
+    goto err;
+
   format_iso_time(published, rs->published_on);
   digest_to_base64(identity64, rs->identity_digest);
   digest_to_base64(digest64, rs->descriptor_digest);
@@ -110,7 +68,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                    (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
                    (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
                    published,
-                   fmt_addr32(rs->addr),
+                   ip_str,
                    (int)rs->or_port,
                    (int)rs->dir_port);
 
@@ -119,12 +77,6 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
    * networkstatus_type_t values, with an additional control port value
    * added -MP */
 
-  /* V3 microdesc consensuses only have "a" lines in later consensus methods
-   */
-  if (format == NS_V3_CONSENSUS_MICRODESC &&
-      consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS)
-    goto done;
-
   /* Possible "a" line. At most one for now. */
   if (!tor_addr_is_null(&rs->ipv6_addr)) {
     smartlist_add_asprintf(chunks, "a %s\n",
@@ -135,7 +87,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     goto done;
 
   smartlist_add_asprintf(chunks,
-                   "s%s%s%s%s%s%s%s%s%s%s\n",
+                   "s%s%s%s%s%s%s%s%s%s%s%s\n",
                   /* These must stay in alphabetical order. */
                    rs->is_authority?" Authority":"",
                    rs->is_bad_exit?" BadExit":"",
@@ -145,6 +97,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                    rs->is_hs_dir?" HSDir":"",
                    rs->is_flagged_running?" Running":"",
                    rs->is_stable?" Stable":"",
+                   rs->is_staledesc?" StaleDesc":"",
                    rs->is_v2_dir?" V2Dir":"",
                    rs->is_valid?" Valid":"");
 
@@ -164,6 +117,8 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     if (format != NS_CONTROL_PORT) {
       /* Blow up more or less nicely if we didn't get anything or not the
        * thing we expected.
+       * This should be kept in sync with the function
+       * routerstatus_has_visibly_changed and the struct routerstatus_t
        */
       if (!desc) {
         char id[HEX_DIGEST_LEN+1];
@@ -232,7 +187,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     }
 
     if (format == NS_V3_VOTE && vrs) {
-      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
+      if (fast_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
         smartlist_add_strdup(chunks, "id ed25519 none\n");
       } else {
         char ed_b64[BASE64_DIGEST256_LEN+1];
diff --git a/src/feature/nodelist/fmt_routerstatus.h b/src/feature/nodelist/fmt_routerstatus.h
index ddd7a7cf37..a007989af3 100644
--- a/src/feature/nodelist/fmt_routerstatus.h
+++ b/src/feature/nodelist/fmt_routerstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,7 +35,6 @@ char *routerstatus_format_entry(
                               const char *version,
                               const char *protocols,
                               routerstatus_format_type_t format,
-                              int consensus_method,
                               const vote_routerstatus_t *vrs);
 
 #endif /* !defined(TOR_FMT_ROUTERSTATUS_H) */
diff --git a/src/feature/nodelist/include.am b/src/feature/nodelist/include.am
new file mode 100644
index 0000000000..2f5d58ec1c
--- /dev/null
+++ b/src/feature/nodelist/include.am
@@ -0,0 +1,49 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/nodelist/authcert.c		\
+	src/feature/nodelist/describe.c		\
+	src/feature/nodelist/dirlist.c		\
+	src/feature/nodelist/microdesc.c	\
+	src/feature/nodelist/networkstatus.c	\
+	src/feature/nodelist/nickname.c		\
+	src/feature/nodelist/nodefamily.c	\
+	src/feature/nodelist/nodelist.c		\
+	src/feature/nodelist/node_select.c	\
+	src/feature/nodelist/routerinfo.c	\
+	src/feature/nodelist/routerlist.c	\
+	src/feature/nodelist/routerset.c	\
+	src/feature/nodelist/fmt_routerstatus.c	\
+	src/feature/nodelist/torcert.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/nodelist/authcert.h			\
+	src/feature/nodelist/authority_cert_st.h	\
+	src/feature/nodelist/describe.h			\
+	src/feature/nodelist/desc_store_st.h		\
+	src/feature/nodelist/dirlist.h			\
+	src/feature/nodelist/document_signature_st.h	\
+	src/feature/nodelist/extrainfo_st.h		\
+	src/feature/nodelist/microdesc.h		\
+	src/feature/nodelist/microdesc_st.h		\
+	src/feature/nodelist/networkstatus.h		\
+	src/feature/nodelist/networkstatus_sr_info_st.h	\
+	src/feature/nodelist/networkstatus_st.h		\
+	src/feature/nodelist/networkstatus_voter_info_st.h	\
+	src/feature/nodelist/nickname.h			\
+	src/feature/nodelist/node_st.h			\
+	src/feature/nodelist/nodefamily.h		\
+	src/feature/nodelist/nodefamily_st.h		\
+	src/feature/nodelist/nodelist.h			\
+	src/feature/nodelist/node_select.h		\
+	src/feature/nodelist/routerinfo.h		\
+	src/feature/nodelist/routerinfo_st.h		\
+	src/feature/nodelist/routerlist.h		\
+	src/feature/nodelist/routerlist_st.h		\
+	src/feature/nodelist/routerset.h		\
+	src/feature/nodelist/fmt_routerstatus.h		\
+	src/feature/nodelist/routerstatus_st.h		\
+	src/feature/nodelist/signed_descriptor_st.h	\
+	src/feature/nodelist/torcert.h			\
+	src/feature/nodelist/vote_routerstatus_st.h
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
index dafaabb5e5..cf7732b8dc 100644
--- a/src/feature/nodelist/microdesc.c
+++ b/src/feature/nodelist/microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2019, The Tor Project, Inc. */
+/* Copyright (c) 2009-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,11 +18,13 @@
 #include "feature/client/entrynodes.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dlstatus.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dirparse/microdesc_parse.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -69,6 +71,8 @@ struct microdesc_cache_t {
 };
 
 static microdesc_cache_t *get_microdesc_cache_noload(void);
+static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                              const char *activity);
 
 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
 static inline unsigned int
@@ -86,10 +90,10 @@ microdesc_eq_(microdesc_t *a, microdesc_t *b)
 }
 
 HT_PROTOTYPE(microdesc_map, microdesc_t, node,
-             microdesc_hash_, microdesc_eq_)
+             microdesc_hash_, microdesc_eq_);
 HT_GENERATE2(microdesc_map, microdesc_t, node,
              microdesc_hash_, microdesc_eq_, 0.6,
-             tor_reallocarray_, tor_free_)
+             tor_reallocarray_, tor_free_);
 
 /************************* md fetch fail cache *****************************/
 
@@ -110,8 +114,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest)
 
   /* If we have a reasonably live consensus, then most of our dirservers should
    * still be caching all the microdescriptors in it. Reasonably live
-   * consensuses are up to a day old. But microdescriptors expire 7 days after
-   * the last consensus that referenced them. */
+   * consensuses are up to a day old (or a day in the future). But
+   * microdescriptors expire 7 days after the last consensus that referenced
+   * them. */
   if (!networkstatus_get_reasonably_live_consensus(approx_time(),
                                                    FLAV_MICRODESC)) {
     return;
@@ -221,6 +226,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
   }
 
   md->off = tor_fd_getpos(fd);
+  warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
+                    "dumping a microdescriptor");
   written = write_all_to_fd(fd, md->body, md->bodylen);
   if (written != (ssize_t)md->bodylen) {
     written = written < 0 ? 0 : written;
@@ -480,6 +487,27 @@ microdesc_cache_clear(microdesc_cache_t *cache)
   cache->bytes_dropped = 0;
 }
 
+static void
+warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                  const char *activity)
+{
+  const char *nul_found = memchr(inp, 0, len);
+  if (BUG(nul_found)) {
+    log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
+             "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
+             activity, offset, (nul_found - inp), len);
+    const char *start_excerpt_at, *eos = inp + len;
+    if ((nul_found - inp) >= 16)
+      start_excerpt_at = nul_found - 16;
+    else
+      start_excerpt_at = inp;
+    size_t excerpt_len = MIN(32, eos - start_excerpt_at);
+    char tmp[65];
+    base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
+    log_warn(LD_BUG, "      surrounding string: %s", tmp);
+  }
+}
+
 /** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
  * for the first time.  Return 0 on success, -1 on failure. */
 int
@@ -497,6 +525,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
 
   mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
   if (mm) {
+    warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
     added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
                                     SAVED_IN_CACHE, 0, -1, NULL);
     if (added) {
@@ -508,7 +537,9 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   journal_content = read_file_to_str(cache->journal_fname,
                                      RFTS_IGNORE_MISSING, &st);
   if (journal_content) {
-    cache->journal_len = (size_t) st.st_size;
+    cache->journal_len = strlen(journal_content);
+    warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
+                      "reading microdesc journal");
     added = microdescs_add_to_cache(cache, journal_content,
                                     journal_content+st.st_size,
                                     SAVED_IN_JOURNAL, 0, -1, NULL);
@@ -544,8 +575,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
   size_t bytes_dropped = 0;
   time_t now = time(NULL);
 
-  /* If we don't know a live consensus, don't believe last_listed values: we
-   * might be starting up after being down for a while. */
+  /* If we don't know a reasonably live consensus, don't believe last_listed
+   * values: we might be starting up after being down for a while. */
   if (! force &&
       ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
       return;
@@ -884,10 +915,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno)
   if (md->body && md->saved_location != SAVED_IN_CACHE)
     tor_free(md->body);
 
-  if (md->family) {
-    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
-    smartlist_free(md->family);
-  }
+  nodefamily_free(md->family);
   short_policy_free(md->exit_policy);
   short_policy_free(md->ipv6_exit_policy);
 
@@ -943,7 +971,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
       continue;
     if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
       continue;
-    if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
+    if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
       continue;
     /* XXXX Also skip if we're a noncache and wouldn't use this router.
      * XXXX NM Microdesc
@@ -970,9 +998,10 @@ update_microdesc_downloads(time_t now)
 
   if (should_delay_dir_fetches(options, NULL))
     return;
-  if (directory_too_idle_to_fetch_descriptors(options, now))
+  if (dirclient_too_idle_to_fetch_descriptors(options, now))
     return;
 
+  /* Give up if we don't have a reasonably live consensus. */
   consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
   if (!consensus)
     return;
diff --git a/src/feature/nodelist/microdesc.h b/src/feature/nodelist/microdesc.h
index c18099d540..b352f58e34 100644
--- a/src/feature/nodelist/microdesc.h
+++ b/src/feature/nodelist/microdesc.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h
index bb8b23d664..410403e965 100644
--- a/src/feature/nodelist/microdesc_st.h
+++ b/src/feature/nodelist/microdesc_st.h
@@ -1,16 +1,24 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file microdesc_st.h
+ * @brief Microdescriptor structure
+ **/
+
 #ifndef MICRODESC_ST_H
 #define MICRODESC_ST_H
 
 struct curve25519_public_key_t;
 struct ed25519_public_key_t;
+struct nodefamily_t;
 struct short_policy_t;
 
+#include "ext/ht.h"
+
 /** A microdescriptor is the smallest amount of information needed to build a
  * circuit through a router.  They are generated by the directory authorities,
  * using information from the uploaded routerinfo documents.  They are not
@@ -32,6 +40,8 @@ struct microdesc_t {
   unsigned int no_save : 1;
   /** If true, this microdesc has an entry in the microdesc_map */
   unsigned int held_in_map : 1;
+  /** True iff the exit policy for this router rejects everything. */
+  unsigned int policy_is_reject_star : 1;
   /** Reference count: how many node_ts have a reference to this microdesc? */
   unsigned int held_by_nodes;
 
@@ -69,12 +79,12 @@ struct microdesc_t {
   tor_addr_t ipv6_addr;
   /** As routerinfo_t.ipv6_orport */
   uint16_t ipv6_orport;
-  /** As routerinfo_t.family */
-  smartlist_t *family;
+  /** As routerinfo_t.family, with readable members parsed. */
+  struct nodefamily_t *family;
   /** IPv4 exit policy summary */
   struct short_policy_t *exit_policy;
   /** IPv6 exit policy summary */
   struct short_policy_t *ipv6_exit_policy;
 };
 
-#endif
+#endif /* !defined(MICRODESC_ST_H) */
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
index c74acd8b74..e07d58c91c 100644
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,6 +44,7 @@
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitstats.h"
@@ -57,16 +58,18 @@
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/fmt_routerstatus.h"
@@ -81,6 +84,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
+#include "feature/dirauth/dirauth_periodic.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
@@ -98,6 +102,7 @@
 #include "feature/nodelist/routerlist_st.h"
 #include "feature/dirauth/vote_microdesc_hash_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -116,8 +121,6 @@ STATIC networkstatus_t *current_md_consensus = NULL;
 typedef struct consensus_waiting_for_certs_t {
   /** The consensus itself. */
   networkstatus_t *consensus;
-  /** The encoded version of the consensus, nul-terminated. */
-  char *body;
   /** When did we set the current value of consensus_waiting_for_certs?  If
    * this is too recent, we shouldn't try to fetch a new consensus for a
    * little while, to give ourselves time to get certificates for this one. */
@@ -179,6 +182,10 @@ static void update_consensus_bootstrap_multiple_downloads(
 static int networkstatus_check_required_protocols(const networkstatus_t *ns,
                                                   int client_mode,
                                                   char **warning_out);
+static int reload_consensus_from_file(const char *fname,
+                                      const char *flavor,
+                                      unsigned flags,
+                                      const char *source_dir);
 
 /** Forget that we've warned about anything networkstatus-related, so we will
  * give fresh warnings if the same behavior happens again. */
@@ -210,14 +217,11 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
-/**
- * Read and and return the cached consensus of type <b>flavorname</b>.  If
- * <b>unverified</b> is true, get the one we haven't verified. Return NULL if
- * the file isn't there. */
-static char *
-networkstatus_read_cached_consensus_impl(int flav,
-                                         const char *flavorname,
-                                         int unverified_consensus)
+/** Return the filename used to cache the consensus of a given flavor */
+MOCK_IMPL(char *,
+networkstatus_get_cache_fname,(int flav,
+                               const char *flavorname,
+                               int unverified_consensus))
 {
   char buf[128];
   const char *prefix;
@@ -232,21 +236,35 @@ networkstatus_read_cached_consensus_impl(int flav,
     tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
   }
 
-  char *filename = get_cachedir_fname(buf);
-  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  return get_cachedir_fname(buf);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static tor_mmap_t *
+networkstatus_map_cached_consensus_impl(int flav,
+                                        const char *flavorname,
+                                        int unverified_consensus)
+{
+  char *filename = networkstatus_get_cache_fname(flav,
+                                                 flavorname,
+                                                 unverified_consensus);
+  tor_mmap_t *result = tor_mmap_file(filename);
   tor_free(filename);
   return result;
 }
 
-/** Return a new string containing the current cached consensus of flavor
- * <b>flavorname</b>. */
-char *
-networkstatus_read_cached_consensus(const char *flavorname)
- {
+/** Map the file containing the current cached consensus of flavor
+ * <b>flavorname</b> */
+tor_mmap_t *
+networkstatus_map_cached_consensus(const char *flavorname)
+{
   int flav = networkstatus_parse_flavor_name(flavorname);
   if (flav < 0)
     return NULL;
-  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+  return networkstatus_map_cached_consensus_impl(flav, flavorname, 0);
 }
 
 /** Read every cached v3 consensus networkstatus from the disk. */
@@ -259,25 +277,15 @@ router_reload_consensus_networkstatus(void)
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
     const char *flavor = networkstatus_get_flavor_name(flav);
-    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
-        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
-                 flavor);
-      }
-      tor_free(s);
-    }
+    char *fname = networkstatus_get_cache_fname(flav, flavor, 0);
+    reload_consensus_from_file(fname, flavor, flags, NULL);
+    tor_free(fname);
 
-    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor,
-                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
-                                     NULL)) {
-        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
-                 "from cache", flavor);
-      }
-      tor_free(s);
-    }
+    fname = networkstatus_get_cache_fname(flav, flavor, 1);
+    reload_consensus_from_file(fname, flavor,
+                               flags | NSSET_WAS_WAITING_FOR_CERTS,
+                               NULL);
+    tor_free(fname);
   }
 
   update_certificate_downloads(time(NULL));
@@ -713,8 +721,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest)
 
 /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or
  * NULL if none was found. */
-const routerstatus_t *
-networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+MOCK_IMPL(const routerstatus_t *,
+networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest))
 {
   return networkstatus_vote_find_mutable_entry(ns, digest);
 }
@@ -1156,7 +1164,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
       }
     }
 
-    if (directory_fetches_dir_info_early(options)) {
+    if (dirclient_fetches_dir_info_early(options)) {
       /* We want to cache the next one at some point after this one
        * is no longer fresh... */
       start = (time_t)(c->fresh_until + min_sec_before_caching);
@@ -1178,7 +1186,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
 
       /* If we're a bridge user, make use of the numbers we just computed
        * to choose the rest of the interval *after* them. */
-      if (directory_fetches_dir_info_later(options)) {
+      if (dirclient_fetches_dir_info_later(options)) {
         /* Give all the *clients* enough time to download the consensus. */
         start = (time_t)(start + dl_interval + min_sec_before_caching);
         /* But try to get it before ours actually expires. */
@@ -1377,7 +1385,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor))
 }
 
 /** Return the most recent consensus that we have downloaded, or NULL if we
- * don't have one. */
+ * don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus,(void))
 {
@@ -1388,7 +1396,7 @@ networkstatus_get_latest_consensus,(void))
 }
 
 /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL
- * if we don't have one. */
+ * if we don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
 {
@@ -1422,10 +1430,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now)
   return (ns->valid_after <= now && now <= ns->valid_until);
 }
 
-/** Determine if <b>consensus</b> is valid or expired recently enough that
- * we can still use it.
+/** Determine if <b>consensus</b> is valid, or expired recently enough, or not
+ * too far in the future, so that we can still use it.
  *
- * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old or
+ * too new.
  */
 int
 networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
@@ -1434,29 +1443,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
   if (BUG(!consensus))
     return 0;
 
-  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+  return networkstatus_valid_after_is_reasonably_live(consensus->valid_after,
+                                                      now) &&
+         networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
                                                       now);
 }
 
+#define REASONABLY_LIVE_TIME (24*60*60)
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_after
+ * time, and checks to see if it is in the past, or not too far in the future.
+ */
+int
+networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                             time_t now)
+{
+  return (now >= valid_after - REASONABLY_LIVE_TIME);
+}
+
 /** As networkstatus_consensus_reasonably_live, but takes a valid_until
- * time rather than an entire consensus. */
+ * time, and checks to see if it is in the future, or not too far in the past.
+ */
 int
 networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                              time_t now)
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   return (now <= valid_until + REASONABLY_LIVE_TIME);
 }
 
 /** As networkstatus_get_live_consensus(), but is way more tolerant of expired
- * consensuses. */
+ *  and future consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_reasonably_live_consensus,(time_t now, int flavor))
 {
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
-      consensus->valid_after <= now &&
       networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
@@ -1517,7 +1539,7 @@ networkstatus_consensus_can_use_extra_fallbacks,(const or_options_t *options))
              >= smartlist_len(router_get_trusted_dir_servers()));
   /* If we don't fetch from the authorities, and we have additional mirrors,
    * we can use them. */
-  return (!directory_fetches_from_authorities(options)
+  return (!dirclient_fetches_from_authorities(options)
           && (smartlist_len(router_get_fallback_dir_servers())
               > smartlist_len(router_get_trusted_dir_servers())));
 }
@@ -1557,36 +1579,16 @@ networkstatus_consensus_is_already_downloading(const char *resource)
   return answer;
 }
 
-/* Does the current, reasonably live consensus have IPv6 addresses?
- * Returns 1 if there is a reasonably live consensus and its consensus method
- * includes IPv6 addresses in the consensus.
- * Otherwise, if there is no consensus, or the method does not include IPv6
- * addresses, returns 0. */
-int
-networkstatus_consensus_has_ipv6(const or_options_t* options)
-{
-  const networkstatus_t *cons = networkstatus_get_reasonably_live_consensus(
-                                                    approx_time(),
-                                                    usable_consensus_flavor());
-
-  /* If we have no consensus, we have no IPv6 in it */
-  if (!cons) {
-    return 0;
-  }
-
-  /* Different flavours of consensus gained IPv6 at different times */
-  if (we_use_microdescriptors_for_circuits(options)) {
-    return
-       cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
-  } else {
-    return 1;
-  }
-}
-
-/** Given two router status entries for the same router identity, return 1 if
- * if the contents have changed between them. Otherwise, return 0. */
-static int
-routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b)
+/** Given two router status entries for the same router identity, return 1
+ * if the contents have changed between them. Otherwise, return 0.
+ * It only checks for fields that are output by control port.
+ * This should be kept in sync with the struct routerstatus_t
+ * and the printing function routerstatus_format_entry in
+ * NS_CONTROL_PORT mode.
+ **/
+STATIC int
+routerstatus_has_visibly_changed(const routerstatus_t *a,
+                                 const routerstatus_t *b)
 {
   tor_assert(tor_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN));
 
@@ -1605,9 +1607,14 @@ routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b)
          a->is_valid != b->is_valid ||
          a->is_possible_guard != b->is_possible_guard ||
          a->is_bad_exit != b->is_bad_exit ||
-         a->is_hs_dir != b->is_hs_dir;
-  // XXXX this function needs a huge refactoring; it has gotten out
-  // XXXX of sync with routerstatus_t, and it will do so again.
+         a->is_hs_dir != b->is_hs_dir ||
+         a->is_staledesc != b->is_staledesc ||
+         a->has_bandwidth != b->has_bandwidth ||
+         a->published_on != b->published_on ||
+         a->ipv6_orport != b->ipv6_orport ||
+         a->is_v2_dir != b->is_v2_dir ||
+         a->bandwidth_kb != b->bandwidth_kb ||
+         tor_addr_compare(&a->ipv6_addr, &b->ipv6_addr, CMP_EXACT);
 }
 
 /** Notify controllers of any router status entries that changed between
@@ -1639,7 +1646,7 @@ notify_control_networkstatus_changed(const networkstatus_t *old_c,
                      tor_memcmp(rs_old->identity_digest,
                             rs_new->identity_digest, DIGEST_LEN),
                      smartlist_add(changed, (void*) rs_new)) {
-    if (routerstatus_has_changed(rs_old, rs_new))
+    if (routerstatus_has_visibly_changed(rs_old, rs_new))
       smartlist_add(changed, (void*)rs_new);
   } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new);
 
@@ -1655,6 +1662,7 @@ notify_before_networkstatus_changes(const networkstatus_t *old_c,
   notify_control_networkstatus_changed(old_c, new_c);
   dos_consensus_has_changed(new_c);
   relay_consensus_has_changed(new_c);
+  hs_dos_consensus_has_changed(new_c);
 }
 
 /* Called after a new consensus has been put in the global state. It is safe
@@ -1725,6 +1733,44 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
 #endif /* defined(TOR_UNIT_TESTS) */
 
 /**
+ * Helper: Read the current consensus of type <b>flavor</b> from
+ * <b>fname</b>.  Flags and return values are as for
+ * networkstatus_set_current_consensus().
+ **/
+static int
+reload_consensus_from_file(const char *fname,
+                           const char *flavor,
+                           unsigned flags,
+                           const char *source_dir)
+{
+  tor_mmap_t *map = tor_mmap_file(fname);
+  if (!map)
+    return 0;
+
+  int rv = networkstatus_set_current_consensus(map->data, map->size,
+                                               flavor, flags, source_dir);
+#ifdef _WIN32
+  if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) {
+    log_notice(LD_GENERAL, "Looks like the above failures are probably "
+               "because of a CRLF in consensus file %s; falling back to "
+               "read_file_to_string. Nothing to worry about: this file "
+               "was probably saved by an earlier version of Tor.",
+               escaped(fname));
+    char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+    rv = networkstatus_set_current_consensus(content, strlen(content),
+                                             flavor, flags, source_dir);
+    tor_free(content);
+  }
+#endif /* defined(_WIN32) */
+  if (rv < -1) {
+    log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s",
+             escaped(fname));
+  }
+  tor_munmap_file(map);
+  return rv;
+}
+
+/**
  * Helper for handle_missing_protocol_warning: handles either the
  * client case (if <b>is_client</b> is set) or the server case otherwise.
  */
@@ -1841,6 +1887,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor,
  */
 int
 networkstatus_set_current_consensus(const char *consensus,
+                                    size_t consensus_len,
                                     const char *flavor,
                                     unsigned flags,
                                     const char *source_dir)
@@ -1869,7 +1916,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   /* Make sure it's parseable. */
-  c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS);
+  c = networkstatus_parse_vote_from_string(consensus,
+                                           consensus_len,
+                                           NULL, NS_TYPE_CONSENSUS);
   if (!c) {
     log_warn(LD_DIR, "Unable to parse networkstatus consensus");
     result = -2;
@@ -1957,14 +2006,12 @@ networkstatus_set_current_consensus(const char *consensus,
           c->valid_after > current_valid_after) {
         waiting = &consensus_waiting_for_certs[flav];
         networkstatus_vote_free(waiting->consensus);
-        tor_free(waiting->body);
         waiting->consensus = c;
         free_consensus = 0;
-        waiting->body = tor_strdup(consensus);
         waiting->set_at = now;
         waiting->dl_failed = 0;
         if (!from_cache) {
-          write_str_to_file(unverified_fname, consensus, 0);
+          write_bytes_to_file(unverified_fname, consensus, consensus_len, 1);
         }
         if (dl_certs)
           authority_certs_fetch_missing(c, now, source_dir);
@@ -1976,9 +2023,9 @@ networkstatus_set_current_consensus(const char *consensus,
          * latest consensus. */
         if (was_waiting_for_certs && from_cache)
           if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
           }
       }
       goto done;
@@ -1991,9 +2038,9 @@ networkstatus_set_current_consensus(const char *consensus,
       }
       if (was_waiting_for_certs && (r < -1) && from_cache) {
         if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
         }
       }
       goto done;
@@ -2055,16 +2102,12 @@ networkstatus_set_current_consensus(const char *consensus,
       waiting->consensus->valid_after <= c->valid_after) {
     networkstatus_vote_free(waiting->consensus);
     waiting->consensus = NULL;
-    if (consensus != waiting->body)
-      tor_free(waiting->body);
-    else
-      waiting->body = NULL;
     waiting->set_at = 0;
     waiting->dl_failed = 0;
     if (unlink(unverified_fname) != 0) {
-      log_warn(LD_FS,
-               "Failed to unlink %s: %s",
-               unverified_fname, strerror(errno));
+      log_debug(LD_FS,
+                "Failed to unlink %s: %s",
+                unverified_fname, strerror(errno));
     }
   }
 
@@ -2077,12 +2120,11 @@ networkstatus_set_current_consensus(const char *consensus,
      * the first thing we need to do is recalculate the voting schedule static
      * object so we can use the timings in there needed by some subsystems
      * such as hidden service and shared random. */
-    voting_schedule_recalculate_timing(options, now);
+    dirauth_sched_recalculate_timing(options, now);
     reschedule_dirvote(options);
 
     nodelist_set_consensus(c);
 
-    /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
     update_consensus_networkstatus_fetch_time(now);
 
     /* Change the cell EWMA settings */
@@ -2095,6 +2137,7 @@ networkstatus_set_current_consensus(const char *consensus,
     circuit_build_times_new_consensus_params(
                                get_circuit_build_times_mutable(), c);
     channelpadding_new_consensus_params(c);
+    circpad_new_consensus_params(c);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
@@ -2108,22 +2151,23 @@ networkstatus_set_current_consensus(const char *consensus,
   if (we_want_to_fetch_flavor(options, flav)) {
     if (dir_server_mode(get_options())) {
       dirserv_set_cached_consensus_networkstatus(consensus,
+                                                 consensus_len,
                                                  flavor,
                                                  &c->digests,
                                                  c->digest_sha3_as_signed,
                                                  c->valid_after);
 
-      consdiffmgr_add_consensus(consensus, c);
+      consdiffmgr_add_consensus(consensus, consensus_len, c);
     }
   }
 
   if (!from_cache) {
-    write_str_to_file(consensus_fname, consensus, 0);
+    write_bytes_to_file(consensus_fname, consensus, consensus_len, 1);
   }
 
   warn_early_consensus(c, flavor, now);
 
-  /* We got a new consesus. Reset our md fetch fail cache */
+  /* We got a new consensus. Reset our md fetch fail cache */
   microdesc_reset_outdated_dirservers_list();
 
   router_dir_info_changed();
@@ -2154,14 +2198,10 @@ networkstatus_note_certs_arrived(const char *source_dir)
     if (!waiting->consensus)
       continue;
     if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
-      char *waiting_body = waiting->body;
-      if (!networkstatus_set_current_consensus(
-                                 waiting_body,
-                                 flavor_name,
-                                 NSSET_WAS_WAITING_FOR_CERTS,
-                                 source_dir)) {
-        tor_free(waiting_body);
-      }
+      char *fname = networkstatus_get_cache_fname(i, flavor_name, 1);
+      reload_consensus_from_file(fname, flavor_name,
+                                 NSSET_WAS_WAITING_FOR_CERTS, source_dir);
+      tor_free(fname);
     }
   }
 }
@@ -2311,10 +2351,52 @@ char *
 networkstatus_getinfo_helper_single(const routerstatus_t *rs)
 {
   return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT,
-                                   ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
                                    NULL);
 }
 
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
+ *
+ * We assume that node-\>is_running has already been set, e.g. by
+ *   dirserv_set_router_is_running(ri, now);
+ */
+void
+set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                 const node_t *node,
+                                 const routerinfo_t *ri)
+{
+  memset(rs, 0, sizeof(routerstatus_t));
+
+  rs->is_authority =
+    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
+
+  /* Set by compute_performance_thresholds or from consensus */
+  rs->is_exit = node->is_exit;
+  rs->is_stable = node->is_stable;
+  rs->is_fast = node->is_fast;
+  rs->is_flagged_running = node->is_running;
+  rs->is_valid = node->is_valid;
+  rs->is_possible_guard = node->is_possible_guard;
+  rs->is_bad_exit = node->is_bad_exit;
+  rs->is_hs_dir = node->is_hs_dir;
+  rs->is_named = rs->is_unnamed = 0;
+
+  rs->published_on = ri->cache_info.published_on;
+  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
+  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  rs->addr = ri->addr;
+  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
+  rs->or_port = ri->or_port;
+  rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
+
+  tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
+  rs->ipv6_orport = ri->ipv6_orport;
+}
+
 /** Alloc and return a string describing routerstatuses for the most
  * recent info of each router we know about that is of purpose
  * <b>purpose_string</b>. Return NULL if unrecognized purpose.
@@ -2331,7 +2413,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   smartlist_t *statuses;
   const uint8_t purpose = router_purpose_from_string(purpose_string);
   routerstatus_t rs;
-  const int bridge_auth = authdir_mode_bridge(get_options());
 
   if (purpose == ROUTER_PURPOSE_UNKNOWN) {
     log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
@@ -2348,11 +2429,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
       continue;
     if (ri->purpose != purpose)
       continue;
-    /* TODO: modifying the running flag in a getinfo is a bad idea */
-    if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
-      dirserv_set_router_is_running(ri, now);
-    /* then generate and write out status lines for each of them */
-    set_routerstatus_from_routerinfo(&rs, node, ri, now, 0);
+    set_routerstatus_from_routerinfo(&rs, node, ri);
     smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs));
   } SMARTLIST_FOREACH_END(ri);
 
@@ -2362,41 +2439,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   return answer;
 }
 
-/** Write out router status entries for all our bridge descriptors. */
-void
-networkstatus_dump_bridge_status_to_file(time_t now)
-{
-  char *status = networkstatus_getinfo_by_purpose("bridge", now);
-  char *fname = NULL;
-  char *thresholds = NULL;
-  char *published_thresholds_and_status = NULL;
-  char published[ISO_TIME_LEN+1];
-  const routerinfo_t *me = router_get_my_routerinfo();
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *fingerprint_line = NULL;
-
-  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
-                                      fingerprint, 0) >= 0) {
-    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
-  } else {
-    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
-  }
-  format_iso_time(published, now);
-  dirserv_compute_bridge_flag_thresholds();
-  thresholds = dirserv_get_flag_thresholds_line();
-  tor_asprintf(&published_thresholds_and_status,
-               "published %s\nflag-thresholds %s\n%s%s",
-               published, thresholds, fingerprint_line ? fingerprint_line : "",
-               status);
-  fname = get_datadir_fname("networkstatus-bridges");
-  write_str_to_file(fname,published_thresholds_and_status,0);
-  tor_free(thresholds);
-  tor_free(published_thresholds_and_status);
-  tor_free(fname);
-  tor_free(status);
-  tor_free(fingerprint_line);
-}
-
 /* DOCDOC get_net_param_from_list */
 static int32_t
 get_net_param_from_list(smartlist_t *net_params, const char *param_name,
@@ -2668,6 +2710,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
   const char *required, *recommended;
   char *missing = NULL;
 
+  const bool consensus_postdates_this_release =
+    ns->valid_after >= tor_get_approx_release_date();
+
   tor_assert(warning_out);
 
   if (client_mode) {
@@ -2685,7 +2730,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
                  "%s on the Tor network. The missing protocols are: %s",
                  func, missing);
     tor_free(missing);
-    return 1;
+    return consensus_postdates_this_release ? 1 : 0;
   }
 
   if (! protover_all_supported(recommended, &missing)) {
@@ -2718,6 +2763,49 @@ networkstatus_free_all(void)
       networkstatus_vote_free(waiting->consensus);
       waiting->consensus = NULL;
     }
-    tor_free(waiting->body);
   }
 }
+
+/** Return the start of the next interval of size <b>interval</b> (in
+ * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
+ * starts a fresh interval, and if the last interval of a day would be
+ * truncated to less than half its size, it is rolled into the
+ * previous interval. */
+time_t
+voting_sched_get_start_of_interval_after(time_t now, int interval,
+                                           int offset)
+{
+  struct tm tm;
+  time_t midnight_today=0;
+  time_t midnight_tomorrow;
+  time_t next;
+
+  tor_gmtime_r(&now, &tm);
+  tm.tm_hour = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
+
+  if (tor_timegm(&tm, &midnight_today) < 0) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
+    // LCOV_EXCL_STOP
+  }
+  midnight_tomorrow = midnight_today + (24*60*60);
+
+  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
+
+  /* Intervals never cross midnight. */
+  if (next > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  /* If the interval would only last half as long as it's supposed to, then
+   * skip over to the next day. */
+  if (next + interval/2 > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  next += offset;
+  if (next - interval > now)
+    next -= interval;
+
+  return next;
+}
diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h
index 9e7b0f1bb0..ce050aeadc 100644
--- a/src/feature/nodelist/networkstatus.h
+++ b/src/feature/nodelist/networkstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,10 @@
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
-char *networkstatus_read_cached_consensus(const char *flavorname);
+MOCK_DECL(char *,networkstatus_get_cache_fname,(int flav,
+                                                const char *flavorname,
+                                                int unverified_consensus));
+tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
 void routerstatus_free_(routerstatus_t *rs);
 #define routerstatus_free(rs) \
@@ -40,8 +43,9 @@ int compare_digest_to_routerstatus_entry(const void *_key,
                                          const void **_member);
 int compare_digest_to_vote_routerstatus_entry(const void *_key,
                                               const void **_member);
-const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns,
-                                              const char *digest);
+MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,(
+                                              networkstatus_t *ns,
+                                              const char *digest));
 routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns,
                                               const char *digest);
 int networkstatus_vote_find_entry_idx(networkstatus_t *ns,
@@ -87,6 +91,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
 int networkstatus_is_live(const networkstatus_t *ns, time_t now);
 int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
                                             time_t now);
+int networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                                 time_t now);
 int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                                  time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus,
@@ -98,7 +104,6 @@ int networkstatus_consensus_can_use_multiple_directories(
 MOCK_DECL(int, networkstatus_consensus_can_use_extra_fallbacks,(
                                                 const or_options_t *options));
 int networkstatus_consensus_is_already_downloading(const char *resource);
-int networkstatus_consensus_has_ipv6(const or_options_t* options);
 
 #define NSSET_FROM_CACHE 1
 #define NSSET_WAS_WAITING_FOR_CERTS 2
@@ -106,6 +111,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options);
 #define NSSET_ACCEPT_OBSOLETE 8
 #define NSSET_REQUIRE_FLAVOR 16
 int networkstatus_set_current_consensus(const char *consensus,
+                                        size_t consensus_len,
                                         const char *flavor,
                                         unsigned flags,
                                         const char *source_dir);
@@ -118,7 +124,6 @@ void signed_descs_update_status_from_consensus_networkstatus(
 
 char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
 char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
-void networkstatus_dump_bridge_status_to_file(time_t now);
 MOCK_DECL(int32_t, networkstatus_get_param,
           (const networkstatus_t *ns, const char *param_name,
            int32_t default_val, int32_t min_val, int32_t max_val));
@@ -145,6 +150,13 @@ void vote_routerstatus_free_(vote_routerstatus_t *rs);
 #define vote_routerstatus_free(rs) \
   FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
 
+void set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                      const node_t *node,
+                                      const routerinfo_t *ri);
+time_t voting_sched_get_start_of_interval_after(time_t now,
+                                                  int interval,
+                                                  int offset);
+
 #ifdef NETWORKSTATUS_PRIVATE
 #ifdef TOR_UNIT_TESTS
 STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
@@ -154,7 +166,8 @@ STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor,
 extern networkstatus_t *current_ns_consensus;
 extern networkstatus_t *current_md_consensus;
 #endif /* defined(TOR_UNIT_TESTS) */
+STATIC int routerstatus_has_visibly_changed(const routerstatus_t *a,
+                                    const routerstatus_t *b);
 #endif /* defined(NETWORKSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_NETWORKSTATUS_H) */
-
diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h
index 677d8ed811..04d0dfe8f6 100644
--- a/src/feature/nodelist/networkstatus_sr_info_st.h
+++ b/src/feature/nodelist/networkstatus_sr_info_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file networkstatus_sr_info_st.h
+ * @brief Shared-randomness structure.
+ **/
+
 #ifndef NETWORKSTATUS_SR_INFO_ST_H
 #define NETWORKSTATUS_SR_INFO_ST_H
 
@@ -19,5 +24,4 @@ struct networkstatus_sr_info_t {
   smartlist_t *commits;
 };
 
-#endif
-
+#endif /* !defined(NETWORKSTATUS_SR_INFO_ST_H) */
diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h
index 6160f12361..021168d3ca 100644
--- a/src/feature/nodelist/networkstatus_st.h
+++ b/src/feature/nodelist/networkstatus_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file networkstatus_st.h
+ * @brief Networkstatus consensus/vote structure.
+ **/
+
 #ifndef NETWORKSTATUS_ST_H
 #define NETWORKSTATUS_ST_H
 
@@ -99,6 +104,9 @@ struct networkstatus_t {
 
   /** List of key=value strings from the headers of the bandwidth list file */
   smartlist_t *bw_file_headers;
+
+  /** A SHA256 digest of the bandwidth file used in a vote. */
+  uint8_t bw_file_digest256[DIGEST256_LEN];
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_ST_H) */
diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h
index 4037fcdeca..b4d0b1dd17 100644
--- a/src/feature/nodelist/networkstatus_voter_info_st.h
+++ b/src/feature/nodelist/networkstatus_voter_info_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file networkstatus_voter_info_st.h
+ * @brief Single consensus voter structure.
+ **/
+
 #ifndef NETWORKSTATUS_VOTER_INFO_ST_H
 #define NETWORKSTATUS_VOTER_INFO_ST_H
 
@@ -27,4 +32,4 @@ struct networkstatus_voter_info_t {
   smartlist_t *sigs;
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_VOTER_INFO_ST_H) */
diff --git a/src/feature/nodelist/nickname.c b/src/feature/nodelist/nickname.c
index 5378b749ca..c022dd6bc4 100644
--- a/src/feature/nodelist/nickname.c
+++ b/src/feature/nodelist/nickname.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/nodelist/nickname.h b/src/feature/nodelist/nickname.h
index 9bdc6b50e8..11c6416f3a 100644
--- a/src/feature/nodelist/nickname.h
+++ b/src/feature/nodelist/nickname.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,4 +16,4 @@ int is_legal_nickname(const char *s);
 int is_legal_nickname_or_hexdigest(const char *s);
 int is_legal_hexdigest(const char *s);
 
-#endif
+#endif /* !defined(TOR_NICKNAME_H) */
diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c
index 7b9e241e5b..e831248413 100644
--- a/src/feature/nodelist/node_select.c
+++ b/src/feature/nodelist/node_select.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,6 +19,7 @@
 #include "core/or/reasons.h"
 #include "feature/client/entrynodes.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/dirlist.h"
@@ -30,6 +31,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
+#include "lib/container/bitarray.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/math/fp.h"
 
@@ -146,7 +148,7 @@ router_pick_dirserver_generic(smartlist_t *sourcelist,
       try_ip_pref = 0;                                                        \
       goto retry_label;                                                       \
     }                                                                         \
-  STMT_END                                                                    \
+  STMT_END
 
 /* Common retry code for router_pick_directory_server_impl and
  * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
@@ -321,7 +323,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
 
   const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
   const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
-  const int must_have_or = directory_must_use_begindir(options);
+  const int must_have_or = dirclient_must_use_begindir(options);
 
   /* Find all the running dirservers we know about. */
   SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
@@ -630,6 +632,7 @@ compute_weighted_bandwidths(const smartlist_t *sl,
   }
 
   weight_scale = networkstatus_get_weight_scale_param(NULL);
+  tor_assert(weight_scale >= 1);
 
   if (rule == WEIGHT_FOR_GUARD) {
     Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
@@ -871,6 +874,58 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
   nodelist_add_node_and_family(sl, node);
 }
 
+/**
+ * Remove every node_t that appears in <b>excluded</b> from <b>sl</b>.
+ *
+ * Behaves like smartlist_subtract, but uses nodelist_idx values to deliver
+ * linear performance when smartlist_subtract would be quadratic.
+ **/
+static void
+nodelist_subtract(smartlist_t *sl, const smartlist_t *excluded)
+{
+  const smartlist_t *nodelist = nodelist_get_list();
+  const int nodelist_len = smartlist_len(nodelist);
+  bitarray_t *excluded_idx = bitarray_init_zero(nodelist_len);
+
+  /* We haven't used nodelist_idx in this way previously, so I'm going to be
+   * paranoid in this code, and check that nodelist_idx is correct for every
+   * node before we use it.  If we fail, we fall back to smartlist_subtract().
+   */
+
+  /* Set the excluded_idx bit corresponding to every excluded node...
+   */
+  SMARTLIST_FOREACH_BEGIN(excluded, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    bitarray_set(excluded_idx, idx);
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Then remove them from sl.
+   */
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    if (bitarray_is_set(excluded_idx, idx)) {
+      SMARTLIST_DEL_CURRENT(sl, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  bitarray_free(excluded_idx);
+  return;
+
+ internal_error:
+  log_warn(LD_BUG, "Internal error prevented us from using the fast method "
+           "for subtracting nodelists. Falling back to the quadratic way.");
+  smartlist_subtract(sl, excluded);
+  bitarray_free(excluded_idx);
+}
+
 /** Return a random running node from the nodelist. Never
  * pick a node that is in
  * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
@@ -905,6 +960,7 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
   const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
 
+  const smartlist_t *node_list = nodelist_get_list();
   smartlist_t *sl=smartlist_new(),
     *excludednodes=smartlist_new();
   const node_t *choice = NULL;
@@ -915,17 +971,17 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   rule = weight_for_exit ? WEIGHT_FOR_EXIT :
     (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
 
-  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+  SMARTLIST_FOREACH_BEGIN(node_list, const node_t *, node) {
     if (node_allows_single_hop_exits(node)) {
       /* Exclude relays that allow single hop exit circuits. This is an
        * obsolete option since 0.2.9.2-alpha and done by default in
        * 0.3.1.0-alpha. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     } else if (rendezvous_v3 &&
                !node_supports_v3_rendezvous_point(node)) {
       /* Exclude relays that do not support to rendezvous for a hidden service
        * version 3. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     }
   } SMARTLIST_FOREACH_END(node);
 
@@ -942,19 +998,11 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
            "We found %d running nodes.",
             smartlist_len(sl));
 
-  smartlist_subtract(sl,excludednodes);
-  log_debug(LD_CIRC,
-            "We removed %d excludednodes, leaving %d nodes.",
-            smartlist_len(excludednodes),
-            smartlist_len(sl));
-
   if (excludedsmartlist) {
-    smartlist_subtract(sl,excludedsmartlist);
-    log_debug(LD_CIRC,
-              "We removed %d excludedsmartlist, leaving %d nodes.",
-              smartlist_len(excludedsmartlist),
-              smartlist_len(sl));
+    smartlist_add_all(excludednodes, excludedsmartlist);
   }
+  nodelist_subtract(sl, excludednodes);
+
   if (excludedset) {
     routerset_subtract_nodes(sl,excludedset);
     log_debug(LD_CIRC,
@@ -1074,7 +1122,7 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
 
   const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
   const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
-  const int must_have_or = directory_must_use_begindir(options);
+  const int must_have_or = dirclient_must_use_begindir(options);
 
   SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
     {
diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h
index ed7450b92c..2e67f990f6 100644
--- a/src/feature/nodelist/node_select.h
+++ b/src/feature/nodelist/node_select.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -97,6 +97,6 @@ STATIC const routerstatus_t *router_pick_directory_server_impl(
                                            int *n_busy_out);
 STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
                                           int serverdesc, int microdesc);
-#endif
+#endif /* defined(NODE_SELECT_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NODE_SELECT_H) */
diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h
index 53ffde29e4..b1ec4db202 100644
--- a/src/feature/nodelist/node_st.h
+++ b/src/feature/nodelist/node_st.h
@@ -1,14 +1,20 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file node_st.h
+ * @brief Node information structure.
+ **/
+
 #ifndef NODE_ST_H
 #define NODE_ST_H
 
 #include "feature/hs/hsdir_index_st.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
+#include "ext/ht.h"
 
 /** A node_t represents a Tor router.
  *
@@ -99,4 +105,4 @@ struct node_t {
   struct hsdir_index_t hsdir_index;
 };
 
-#endif
+#endif /* !defined(NODE_ST_H) */
diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c
new file mode 100644
index 0000000000..feaa3730dc
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.c
@@ -0,0 +1,416 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.c
+ * \brief Code to manipulate encoded, reference-counted node families.  We
+ *  use these tricks to save space, since these families would otherwise
+ *  require a large number of tiny allocations.
+ **/
+
+#include "core/or/or.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
+#include "feature/nodelist/nodefamily_st.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/routerlist.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * Allocate and return a blank node family with space to hold <b>n_members</b>
+ * members.
+ */
+static nodefamily_t *
+nodefamily_alloc(int n_members)
+{
+  size_t alloc_len = offsetof(nodefamily_t, family_members) +
+    NODEFAMILY_ARRAY_SIZE(n_members);
+  nodefamily_t *nf = tor_malloc_zero(alloc_len);
+  nf->n_members = n_members;
+  return nf;
+}
+
+/**
+ * Hashtable hash implementation.
+ */
+static inline unsigned int
+nodefamily_hash(const nodefamily_t *nf)
+{
+  return (unsigned) siphash24g(nf->family_members,
+                               NODEFAMILY_ARRAY_SIZE(nf->n_members));
+}
+
+/**
+ * Hashtable equality implementation.
+ */
+static inline unsigned int
+nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b)
+{
+  return (a->n_members == b->n_members) &&
+    fast_memeq(a->family_members, b->family_members,
+               NODEFAMILY_ARRAY_SIZE(a->n_members));
+}
+
+static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families
+  = HT_INITIALIZER();
+
+HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             nodefamily_eq);
+HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             node_family_eq, 0.6, tor_reallocarray_, tor_free_);
+
+/**
+ * Parse the family declaration in <b>s</b>, returning the canonical
+ * <b>nodefamily_t</b> for its members.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * If NF_WARN_MALFORMED is set in <b>flags</b>, warn about any
+ * elements that we can't parse.  (By default, we log at info.)
+ *
+ * If NF_REJECT_MALFORMED is set in <b>flags</b>, treat any unparseable
+ * elements as an error. (By default, we simply omit them.)
+ **/
+nodefamily_t *
+nodefamily_parse(const char *s, const uint8_t *rsa_id_self,
+                 unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Canonicalize the family list <b>s</b>, returning a newly allocated string.
+ *
+ * The canonicalization rules are fully specified in dir-spec.txt, but,
+ * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are
+ * truncated, nicknames are put into lowercase, unrecognized entries are left
+ * alone, and everything is sorted.
+ **/
+char *
+nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                        unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_t *result_members = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags,
+                                             result_members);
+
+  char *formatted = nodefamily_format(nf);
+  smartlist_split_string(result_members, formatted, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  smartlist_sort_strings(result_members);
+  char *combined = smartlist_join_strings(result_members, " ", 0, NULL);
+
+  nodefamily_free(nf);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp));
+  smartlist_free(result_members);
+  tor_free(formatted);
+
+  return combined;
+}
+
+/**
+ * qsort helper for encoded nodefamily elements.
+ **/
+static int
+compare_members(const void *a, const void *b)
+{
+  return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN);
+}
+
+/**
+ * Parse the member strings in <b>members</b>, returning their canonical
+ * <b>nodefamily_t</b>.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * The <b>flags</b> element is interpreted as in nodefamily_parse().
+ *
+ * If <b>unrecognized</b> is provided, fill it copies of any unrecognized
+ * members.  (Note that malformed $hexids are not considered unrecognized.)
+ **/
+nodefamily_t *
+nodefamily_from_members(const smartlist_t *members,
+                        const uint8_t *rsa_id_self,
+                        unsigned flags,
+                        smartlist_t *unrecognized_out)
+{
+  const int n_self = rsa_id_self ? 1 : 0;
+  int n_bad_elements = 0;
+  int n_members = smartlist_len(members) + n_self;
+  nodefamily_t *tmp = nodefamily_alloc(n_members);
+  uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0);
+
+  SMARTLIST_FOREACH_BEGIN(members, const char *, cp) {
+    bool bad_element = true;
+    if (is_legal_nickname(cp)) {
+      ptr[0] = NODEFAMILY_BY_NICKNAME;
+      tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname
+      memcpy(ptr+1, cp, strlen(cp));
+      tor_strlower((char*) ptr+1);
+      bad_element = false;
+    } else if (is_legal_hexdigest(cp)) {
+      char digest_buf[DIGEST_LEN];
+      char nn_buf[MAX_NICKNAME_LEN+1];
+      char nn_char=0;
+      if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) {
+        bad_element = false;
+        ptr[0] = NODEFAMILY_BY_RSA_ID;
+        memcpy(ptr+1, digest_buf, DIGEST_LEN);
+      }
+    } else {
+      if (unrecognized_out)
+        smartlist_add_strdup(unrecognized_out, cp);
+    }
+
+    if (bad_element) {
+      const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO;
+      log_fn(severity, LD_GENERAL,
+             "Bad element %s while parsing a node family.",
+             escaped(cp));
+      ++n_bad_elements;
+    } else {
+      ptr += NODEFAMILY_MEMBER_LEN;
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  if (n_bad_elements && (flags & NF_REJECT_MALFORMED))
+    goto err;
+
+  if (rsa_id_self) {
+    /* Add self. */
+    ptr[0] = NODEFAMILY_BY_RSA_ID;
+    memcpy(ptr+1, rsa_id_self, DIGEST_LEN);
+  }
+
+  n_members -= n_bad_elements;
+
+  /* Sort tmp into canonical order. */
+  qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN,
+        compare_members);
+
+  /* Remove duplicates. */
+  int i;
+  for (i = 0; i < n_members-1; ++i) {
+    uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i);
+    uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1);
+    if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) {
+      memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN);
+      --n_members;
+      --i;
+    }
+  }
+  int n_members_alloc = tmp->n_members;
+  tmp->n_members = n_members;
+
+  /* See if we already allocated this family. */
+  nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp);
+  if (found) {
+    /* If we did, great: incref it and return it. */
+    ++found->refcnt;
+    tor_free(tmp);
+    return found;
+  } else {
+    /* If not, insert it into the hashtable. */
+    if (n_members_alloc != n_members) {
+      /* Compact the family if needed */
+      nodefamily_t *tmp2 = nodefamily_alloc(n_members);
+      memcpy(tmp2->family_members, tmp->family_members,
+             n_members * NODEFAMILY_MEMBER_LEN);
+      tor_free(tmp);
+      tmp = tmp2;
+    }
+
+    tmp->refcnt = 1;
+    HT_INSERT(nodefamily_map, &the_node_families, tmp);
+    return tmp;
+  }
+
+ err:
+  tor_free(tmp);
+  return NULL;
+}
+
+/**
+ * Drop our reference to <b>family</b>, freeing it if there are no more
+ * references.
+ */
+void
+nodefamily_free_(nodefamily_t *family)
+{
+  if (family == NULL)
+    return;
+
+  --family->refcnt;
+
+  if (family->refcnt == 0) {
+    HT_REMOVE(nodefamily_map, &the_node_families, family);
+    tor_free(family);
+  }
+}
+
+/**
+ * Return true iff <b>family</b> contains the SHA1 RSA1024 identity
+ * <b>rsa_id</b>.
+ */
+bool
+nodefamily_contains_rsa_id(const nodefamily_t *family,
+                           const uint8_t *rsa_id)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    if (ptr[0] == NODEFAMILY_BY_RSA_ID &&
+        fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff <b>family</b> contains the nickname <b>name</b>.
+ */
+bool
+nodefamily_contains_nickname(const nodefamily_t *family,
+                             const char *name)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    // note that the strcasecmp() is safe because there is always at least one
+    // NUL in the encoded nickname, because all legal nicknames are less than
+    // DIGEST_LEN bytes long.
+    if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true if <b>family</b> contains the nickname or the RSA ID for
+ * <b>node</b>
+ **/
+bool
+nodefamily_contains_node(const nodefamily_t *family,
+                         const node_t *node)
+{
+  return
+    nodefamily_contains_nickname(family, node_get_nickname(node))
+    ||
+    nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node));
+}
+
+/**
+ * Look up every entry in <b>family</b>, and add add the corresponding
+ * node_t to <b>out</b>.
+ **/
+void
+nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                  smartlist_t *out)
+{
+  if (!family)
+    return;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    const node_t *node = NULL;
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED);
+        break;
+      case NODEFAMILY_BY_RSA_ID:
+        node = node_get_by_id((char*)ptr+1);
+        break;
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+    if (node)
+      smartlist_add(out, (void *)node);
+  }
+}
+
+/**
+ * Encode <b>family</b> as a space-separated string.
+ */
+char *
+nodefamily_format(const nodefamily_t *family)
+{
+  if (!family)
+    return tor_strdup("");
+
+  unsigned i;
+  smartlist_t *sl = smartlist_new();
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        smartlist_add_strdup(sl, (char*)ptr+1);
+        break;
+      case NODEFAMILY_BY_RSA_ID: {
+        char buf[HEX_DIGEST_LEN+2];
+        buf[0]='$';
+        base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN);
+        tor_strupper(buf);
+        smartlist_add_strdup(sl, buf);
+        break;
+      }
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+  }
+
+  char *result = smartlist_join_strings(sl, " ", 0, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Free all storage held in the nodefamily map.
+ **/
+void
+nodefamily_free_all(void)
+{
+  HT_CLEAR(nodefamily_map, &the_node_families);
+}
diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h
new file mode 100644
index 0000000000..16e161ba82
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.h
+ * \brief Header file for nodefamily.c.
+ **/
+
+#ifndef TOR_NODEFAMILY_H
+#define TOR_NODEFAMILY_H
+
+#include "lib/malloc/malloc.h"
+#include <stdbool.h>
+
+typedef struct nodefamily_t nodefamily_t;
+struct node_t;
+struct smartlist_t;
+
+#define NF_WARN_MALFORMED    (1u<<0)
+#define NF_REJECT_MALFORMED  (1u<<1)
+
+nodefamily_t *nodefamily_parse(const char *s,
+                               const uint8_t *rsa_id_self,
+                               unsigned flags);
+nodefamily_t *nodefamily_from_members(const struct smartlist_t *members,
+                                      const uint8_t *rsa_id_self,
+                                      unsigned flags,
+                                      smartlist_t *unrecognized_out);
+void nodefamily_free_(nodefamily_t *family);
+#define nodefamily_free(family) \
+  FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family))
+
+bool nodefamily_contains_rsa_id(const nodefamily_t *family,
+                                const uint8_t *rsa_id);
+bool nodefamily_contains_nickname(const nodefamily_t *family,
+                                  const char *name);
+bool nodefamily_contains_node(const nodefamily_t *family,
+                              const struct node_t *node);
+void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                       struct smartlist_t *out);
+char *nodefamily_format(const nodefamily_t *family);
+char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                              unsigned flags);
+
+void nodefamily_free_all(void);
+
+#endif /* !defined(TOR_NODEFAMILY_H) */
diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h
new file mode 100644
index 0000000000..c581c917a9
--- /dev/null
+++ b/src/feature/nodelist/nodefamily_st.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file nodefamily_st.h
+ * @brief Compact node-family structure
+ **/
+
+#ifndef TOR_NODEFAMILY_ST_H
+#define TOR_NODEFAMILY_ST_H
+
+#include "orconfig.h"
+#include "ht.h"
+
+struct nodefamily_t {
+  /** Entry for this nodefamily_t within the hashtable. */
+  HT_ENTRY(nodefamily_t) ht_ent;
+  /** Reference count.  (The hashtable is not treated as a reference */
+  uint32_t refcnt;
+  /** Number of items encoded in <b>family_members</b>. */
+  uint32_t n_members;
+  /* A byte-array encoding the members of this family.  We encode each member
+   * as one byte to indicate whether it's a nickname or a fingerprint, plus
+   * DIGEST_LEN bytes of data.  The entries are lexically sorted.
+   */
+  uint8_t family_members[FLEXIBLE_ARRAY_MEMBER];
+};
+
+#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN)
+
+/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID.
+ */
+#define NODEFAMILY_BY_RSA_ID 0
+/** Tag byte, indicates that the following bytes are a NUL-padded nickname.
+ */
+#define NODEFAMILY_BY_NICKNAME 1
+
+/**
+ * Number of bytes to allocate in the array for a nodefamily_t with N members.
+ **/
+#define NODEFAMILY_ARRAY_SIZE(n) \
+  ((n) * NODEFAMILY_MEMBER_LEN)
+
+/**
+ * Pointer to the i'th member of <b>nf</b>, as encoded.
+ */
+#define NODEFAMILY_MEMBER_PTR(nf, i) \
+  (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN]))
+
+#endif /* !defined(TOR_NODEFAMILY_ST_H) */
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
index 9d56f2c3aa..7ebc4f5fda 100644
--- a/src/feature/nodelist/nodelist.c
+++ b/src/feature/nodelist/nodelist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,9 +49,9 @@
 #include "core/or/protover.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
-#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/nodelist/describe.h"
@@ -59,6 +59,7 @@
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
@@ -156,9 +157,9 @@ node_id_eq(const node_t *node1, const node_t *node2)
   return tor_memeq(node1->identity, node2->identity, DIGEST_LEN);
 }
 
-HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq)
+HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq);
 HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 static inline unsigned int
 node_ed_id_hash(const node_t *node)
@@ -173,9 +174,9 @@ node_ed_id_eq(const node_t *node1, const node_t *node2)
 }
 
 HT_PROTOTYPE(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
-             node_ed_id_eq)
+             node_ed_id_eq);
 HT_GENERATE2(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
-             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_)
+             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** The global nodelist. */
 static nodelist_t *the_nodelist=NULL;
@@ -702,7 +703,8 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH(the_nodelist->nodes, node_t *, node,
                     node->rs = NULL);
 
-  /* Conservatively estimate that every node will have 2 addresses. */
+  /* Conservatively estimate that every node will have 2 addresses (v4 and
+   * v6). Then we add the number of configured trusted authorities we have. */
   int estimated_addresses = smartlist_len(ns->routerstatus_list) *
                             get_estimated_address_per_node();
   estimated_addresses += (get_n_authorities(V3_DIRINFO | BRIDGE_DIRINFO) *
@@ -1045,7 +1047,7 @@ nodelist_ensure_freshness(networkstatus_t *ns)
 /** Return a list of a node_t * for every node we know about.  The caller
  * MUST NOT modify the list. (You can set and clear flags in the nodes if
  * you must, but you must not add or remove nodes.) */
-MOCK_IMPL(smartlist_t *,
+MOCK_IMPL(const smartlist_t *,
 nodelist_get_list,(void))
 {
   init_nodelist();
@@ -1148,8 +1150,8 @@ node_get_by_nickname,(const char *nickname, unsigned flags))
 
 /** Return the Ed25519 identity key for the provided node, or NULL if it
  * doesn't have one. */
-const ed25519_public_key_t *
-node_get_ed25519_id(const node_t *node)
+MOCK_IMPL(const ed25519_public_key_t *,
+node_get_ed25519_id,(const node_t *node))
 {
   const ed25519_public_key_t *ri_pk = NULL;
   const ed25519_public_key_t *md_pk = NULL;
@@ -1207,7 +1209,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
 /** Dummy object that should be unreturnable.  Used to ensure that
  * node_get_protover_summary_flags() always returns non-NULL. */
 static const protover_summary_flags_t zero_protover_flags = {
-  0,0,0,0,0,0,0
+  0,0,0,0,0,0,0,0,0
 };
 
 /** Return the protover_summary_flags for a given node. */
@@ -1232,9 +1234,9 @@ node_get_protover_summary_flags(const node_t *node)
  * by ed25519 ID during the link handshake.  If <b>compatible_with_us</b>,
  * it needs to be using a link authentication method that we understand.
  * If not, any plausible link authentication method will do. */
-int
-node_supports_ed25519_link_authentication(const node_t *node,
-                                          int compatible_with_us)
+MOCK_IMPL(int,
+node_supports_ed25519_link_authentication,(const node_t *node,
+                                           int compatible_with_us))
 {
   if (! node_get_ed25519_id(node))
     return 0;
@@ -1267,6 +1269,17 @@ node_supports_ed25519_hs_intro(const node_t *node)
   return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
 }
 
+/** Return true iff <b>node</b> supports the DoS ESTABLISH_INTRO cell
+ * extenstion. */
+int
+node_supports_establish_intro_dos_extension(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->
+                           supports_establish_intro_dos_extension;
+}
+
 /** Return true iff <b>node</b> supports to be a rendezvous point for hidden
  * service version 3 (HSRend=2). */
 int
@@ -1290,6 +1303,102 @@ node_get_rsa_id_digest(const node_t *node)
   return (const uint8_t*)node->identity;
 }
 
+/* Returns a new smartlist with all possible link specifiers from node:
+ *  - legacy ID is mandatory thus MUST be present in node;
+ *  - include ed25519 link specifier if present in the node, and the node
+ *    supports ed25519 link authentication, and:
+ *    - if direct_conn is true, its link versions are compatible with us,
+ *    - if direct_conn is false, regardless of its link versions;
+ *  - include IPv4 link specifier, if the primary address is not IPv4, log a
+ *    BUG() warning, and return an empty smartlist;
+ *  - include IPv6 link specifier if present in the node.
+ *
+ * If node is NULL, returns an empty smartlist.
+ *
+ * The smartlist must be freed using link_specifier_smartlist_free(). */
+MOCK_IMPL(smartlist_t *,
+node_get_link_specifier_smartlist,(const node_t *node, bool direct_conn))
+{
+  link_specifier_t *ls;
+  tor_addr_port_t ap;
+  smartlist_t *lspecs = smartlist_new();
+
+  if (!node)
+    return lspecs;
+
+  /* Get the relay's IPv4 address. */
+  node_get_prim_orport(node, &ap);
+
+  /* We expect the node's primary address to be a valid IPv4 address.
+   * This conforms to the protocol, which requires either an IPv4 or IPv6
+   * address (or both). */
+  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
+      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
+    return lspecs;
+  }
+
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
+  link_specifier_set_un_ipv4_port(ls, ap.port);
+  /* Four bytes IPv4 and two bytes port. */
+  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
+                            sizeof(ap.port));
+  smartlist_add(lspecs, ls);
+
+  /* Legacy ID is mandatory and will always be present in node. */
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
+         link_specifier_getlen_un_legacy_id(ls));
+  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
+  smartlist_add(lspecs, ls);
+
+  /* ed25519 ID is only included if the node has it, and the node declares a
+   protocol version that supports ed25519 link authentication.
+   If direct_conn is true, we also require that the node's link version is
+   compatible with us. (Otherwise, we will be sending the ed25519 key
+   to another tor, which may support different link versions.) */
+  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
+      node_supports_ed25519_link_authentication(node, direct_conn)) {
+    ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, LS_ED25519_ID);
+    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
+           link_specifier_getlen_un_ed25519_id(ls));
+    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
+    smartlist_add(lspecs, ls);
+  }
+
+  /* Check for IPv6. If so, include it as well. */
+  if (node_has_ipv6_orport(node)) {
+    ls = link_specifier_new();
+    node_get_pref_ipv6_orport(node, &ap);
+    link_specifier_set_ls_type(ls, LS_IPV6);
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
+    smartlist_add(lspecs, ls);
+  }
+
+  return lspecs;
+}
+
+/* Free a link specifier list. */
+void
+link_specifier_smartlist_free_(smartlist_t *ls_list)
+{
+  if (!ls_list)
+    return;
+
+  SMARTLIST_FOREACH(ls_list, link_specifier_t *, lspec,
+                    link_specifier_free(lspec));
+  smartlist_free(ls_list);
+}
+
 /** Return the nickname of <b>node</b>, or NULL if we can't find one. */
 const char *
 node_get_nickname(const node_t *node)
@@ -1429,8 +1538,7 @@ node_exit_policy_rejects_all(const node_t *node)
   if (node->ri)
     return node->ri->policy_is_reject_star;
   else if (node->md)
-    return node->md->exit_policy == NULL ||
-      short_policy_is_reject_star(node->md->exit_policy);
+    return node->md->policy_is_reject_star;
   else
     return 1;
 }
@@ -1605,19 +1713,6 @@ node_is_me(const node_t *node)
   return router_digest_is_me(node->identity);
 }
 
-/** Return <b>node</b> declared family (as a list of names), or NULL if
- * the node didn't declare a family. */
-const smartlist_t *
-node_get_declared_family(const node_t *node)
-{
-  if (node->ri && node->ri->declared_family)
-    return node->ri->declared_family;
-  else if (node->md && node->md->family)
-    return node->md->family;
-  else
-    return NULL;
-}
-
 /* Does this node have a valid IPv6 address?
  * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for
  * checking specific ports. */
@@ -1877,7 +1972,7 @@ microdesc_has_curve25519_onion_key(const microdesc_t *md)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -1957,7 +2052,7 @@ node_set_country(node_t *node)
 void
 nodelist_refresh_countries(void)
 {
-  smartlist_t *nodes = nodelist_get_list();
+  const smartlist_t *nodes = nodelist_get_list();
   SMARTLIST_FOREACH(nodes, node_t *, node,
                     node_set_country(node));
 }
@@ -1968,6 +2063,9 @@ int
 addrs_in_same_network_family(const tor_addr_t *a1,
                              const tor_addr_t *a2)
 {
+  if (tor_addr_is_null(a1) || tor_addr_is_null(a2))
+    return 0;
+
   switch (tor_addr_family(a1)) {
     case AF_INET:
       return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
@@ -1983,7 +2081,7 @@ addrs_in_same_network_family(const tor_addr_t *a1,
  * (case-insensitive), or if <b>node's</b> identity key digest
  * matches a hexadecimal value stored in <b>nickname</b>.  Return
  * false otherwise. */
-static int
+STATIC int
 node_nickname_matches(const node_t *node, const char *nickname)
 {
   const char *n = node_get_nickname(node);
@@ -1995,7 +2093,7 @@ node_nickname_matches(const node_t *node, const char *nickname)
 }
 
 /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
-static inline int
+STATIC int
 node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
 {
   if (!lst) return 0;
@@ -2006,6 +2104,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
   return 0;
 }
 
+/** Return true iff n1's declared family contains n2. */
+STATIC int
+node_family_contains(const node_t *n1, const node_t *n2)
+{
+  if (n1->ri && n1->ri->declared_family) {
+    return node_in_nickname_smartlist(n1->ri->declared_family, n2);
+  } else if (n1->md) {
+    return nodefamily_contains_node(n1->md->family, n2);
+  } else {
+    return 0;
+  }
+}
+
+/**
+ * Return true iff <b>node</b> has declared a nonempty family.
+ **/
+STATIC bool
+node_has_declared_family(const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    return true;
+  }
+
+  if (node->md && node->md->family) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Add to <b>out</b> every node_t that is listed by <b>node</b> as being in
+ * its family.  (Note that these nodes are not in node's family unless they
+ * also agree that node is in their family.)
+ **/
+STATIC void
+node_lookup_declared_family(smartlist_t *out, const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) {
+      const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED);
+      if (n2) {
+        smartlist_add(out, (node_t *)n2);
+      }
+    } SMARTLIST_FOREACH_END(name);
+    return;
+  }
+
+  if (node->md && node->md->family) {
+    nodefamily_add_nodes_to_smartlist(node->md->family, out);
+  }
+}
+
 /** Return true iff r1 and r2 are in the same family, but not the same
  * router. */
 int
@@ -2018,19 +2171,20 @@ nodes_in_same_family(const node_t *node1, const node_t *node2)
     tor_addr_t a1, a2;
     node_get_addr(node1, &a1);
     node_get_addr(node2, &a2);
-    if (addrs_in_same_network_family(&a1, &a2))
+
+    tor_addr_port_t ap6_1, ap6_2;
+    node_get_pref_ipv6_orport(node1, &ap6_1);
+    node_get_pref_ipv6_orport(node2, &ap6_2);
+
+    if (addrs_in_same_network_family(&a1, &a2) ||
+        addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr))
       return 1;
   }
 
   /* Are they in the same family because the agree they are? */
-  {
-    const smartlist_t *f1, *f2;
-    f1 = node_get_declared_family(node1);
-    f2 = node_get_declared_family(node2);
-    if (f1 && f2 &&
-        node_in_nickname_smartlist(f1, node2) &&
-        node_in_nickname_smartlist(f2, node1))
-      return 1;
+  if (node_family_contains(node1, node2) &&
+      node_family_contains(node2, node1)) {
+    return 1;
   }
 
   /* Are they in the same family because the user says they are? */
@@ -2058,13 +2212,10 @@ void
 nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
 {
   const smartlist_t *all_nodes = nodelist_get_list();
-  const smartlist_t *declared_family;
   const or_options_t *options = get_options();
 
   tor_assert(node);
 
-  declared_family = node_get_declared_family(node);
-
   /* Let's make sure that we have the node itself, if it's a real node. */
   {
     const node_t *real_node = node_get_by_id(node->identity);
@@ -2075,35 +2226,35 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
   /* First, add any nodes with similar network addresses. */
   if (options->EnforceDistinctSubnets) {
     tor_addr_t node_addr;
+    tor_addr_port_t node_ap6;
     node_get_addr(node, &node_addr);
+    node_get_pref_ipv6_orport(node, &node_ap6);
 
     SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) {
       tor_addr_t a;
+      tor_addr_port_t ap6;
       node_get_addr(node2, &a);
-      if (addrs_in_same_network_family(&a, &node_addr))
+      node_get_pref_ipv6_orport(node2, &ap6);
+      if (addrs_in_same_network_family(&a, &node_addr) ||
+          addrs_in_same_network_family(&ap6.addr, &node_ap6.addr))
         smartlist_add(sl, (void*)node2);
     } SMARTLIST_FOREACH_END(node2);
   }
 
-  /* Now, add all nodes in the declared_family of this node, if they
+  /* Now, add all nodes in the declared family of this node, if they
    * also declare this node to be in their family. */
-  if (declared_family) {
+  if (node_has_declared_family(node)) {
+    smartlist_t *declared_family = smartlist_new();
+    node_lookup_declared_family(declared_family, node);
+
     /* Add every r such that router declares familyness with node, and node
      * declares familyhood with router. */
-    SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
-      const node_t *node2;
-      const smartlist_t *family2;
-      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
-        continue;
-      if (!(family2 = node_get_declared_family(node2)))
-        continue;
-      SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) {
-          if (node_nickname_matches(node, name2)) {
-            smartlist_add(sl, (void*)node2);
-            break;
-          }
-      } SMARTLIST_FOREACH_END(name2);
-    } SMARTLIST_FOREACH_END(name);
+    SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) {
+      if (node_family_contains(node2, node)) {
+        smartlist_add(sl, (void*)node2);
+      }
+    } SMARTLIST_FOREACH_END(node2);
+    smartlist_free(declared_family);
   }
 
   /* If the user declared any families locally, honor those too. */
@@ -2408,7 +2559,7 @@ compute_frac_paths_available(const networkstatus_t *consensus,
   const int authdir = authdir_mode_v3(options);
 
   count_usable_descriptors(num_present_out, num_usable_out,
-                           mid, consensus, now, NULL,
+                           mid, consensus, now, options->MiddleNodes,
                            USABLE_DESCRIPTOR_ALL);
   log_debug(LD_NET,
             "%s: %d present, %d usable",
@@ -2610,7 +2761,7 @@ count_loading_descriptors_progress(void)
   if (fraction > 1.0)
     return 0; /* it's not the number of descriptors holding us back */
   return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int)
-    (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 -
+    (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 -
                BOOTSTRAP_STATUS_LOADING_DESCRIPTORS));
 }
 
@@ -2697,14 +2848,14 @@ update_router_have_minimum_dir_info(void)
   /* If paths have just become available in this update. */
   if (res && !have_min_dir_info) {
     control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO");
-    control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+    control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
     log_info(LD_DIR,
              "We now have enough directory information to build circuits.");
   }
 
   /* If paths have just become unavailable in this update. */
   if (!res && have_min_dir_info) {
-    int quiet = directory_too_idle_to_fetch_descriptors(options, now);
+    int quiet = dirclient_too_idle_to_fetch_descriptors(options, now);
     tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR,
         "Our directory information is no longer up-to-date "
         "enough to build circuits: %s", dir_info_status);
diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h
index bc09731ce2..0e06326a9c 100644
--- a/src/feature/nodelist/nodelist.h
+++ b/src/feature/nodelist/nodelist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,16 +71,23 @@ const char *node_get_platform(const node_t *node);
 uint32_t node_get_prim_addr_ipv4h(const node_t *node);
 void node_get_address_string(const node_t *node, char *cp, size_t len);
 long node_get_declared_uptime(const node_t *node);
-const smartlist_t *node_get_declared_family(const node_t *node);
-const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
+MOCK_DECL(const struct ed25519_public_key_t *,node_get_ed25519_id,
+          (const node_t *node));
 int node_ed25519_id_matches(const node_t *node,
                             const struct ed25519_public_key_t *id);
-int node_supports_ed25519_link_authentication(const node_t *node,
-                                              int compatible_with_us);
+MOCK_DECL(int,node_supports_ed25519_link_authentication,
+          (const node_t *node,
+           int compatible_with_us));
 int node_supports_v3_hsdir(const node_t *node);
 int node_supports_ed25519_hs_intro(const node_t *node);
 int node_supports_v3_rendezvous_point(const node_t *node);
+int node_supports_establish_intro_dos_extension(const node_t *node);
 const uint8_t *node_get_rsa_id_digest(const node_t *node);
+MOCK_DECL(smartlist_t *,node_get_link_specifier_smartlist,(const node_t *node,
+                                                           bool direct_conn));
+void link_specifier_smartlist_free_(smartlist_t *ls_list);
+#define link_specifier_smartlist_free(ls_list) \
+  FREE_AND_NULL(smartlist_t, link_specifier_smartlist_free_, (ls_list))
 
 int node_has_ipv6_addr(const node_t *node);
 int node_has_ipv6_orport(const node_t *node);
@@ -100,7 +107,7 @@ const struct curve25519_public_key_t *node_get_curve25519_onion_key(
                                   const node_t *node);
 crypto_pk_t *node_get_rsa_onion_key(const node_t *node);
 
-MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
+MOCK_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 /* Temporary during transition to multiple addresses.  */
 void node_get_addr(const node_t *node, tor_addr_t *addr_out);
@@ -158,10 +165,16 @@ int count_loading_descriptors_progress(void);
 
 #ifdef NODELIST_PRIVATE
 
+STATIC int node_nickname_matches(const node_t *node, const char *nickname);
+STATIC int node_in_nickname_smartlist(const smartlist_t *lst,
+                                      const node_t *node);
+STATIC int node_family_contains(const node_t *n1, const node_t *n2);
+STATIC bool node_has_declared_family(const node_t *node);
+STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node);
+
 #ifdef TOR_UNIT_TESTS
 
-STATIC void
-node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/feature/nodelist/routerinfo.c b/src/feature/nodelist/routerinfo.c
index 975b503615..0bf2a977f5 100644
--- a/src/feature/nodelist/routerinfo.c
+++ b/src/feature/nodelist/routerinfo.c
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerinfo.c
+ * @brief Manipulate full router descriptors.
+ **/
+
 #include "core/or/or.h"
 
 #include "feature/nodelist/nodelist.h"
diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h
index bfa28c7754..604e478999 100644
--- a/src/feature/nodelist/routerinfo.h
+++ b/src/feature/nodelist/routerinfo.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,11 +17,9 @@ void router_get_prim_orport(const routerinfo_t *router,
 int router_has_orport(const routerinfo_t *router,
                       const tor_addr_port_t *orport);
 
-void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
-
 smartlist_t *router_get_all_orports(const routerinfo_t *ri);
 
 const char *router_purpose_to_string(uint8_t p);
 uint8_t router_purpose_from_string(const char *s);
 
-#endif
+#endif /* !defined(TOR_ROUTERINFO_H) */
diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h
index 59656818c1..36ead50e33 100644
--- a/src/feature/nodelist/routerinfo_st.h
+++ b/src/feature/nodelist/routerinfo_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerinfo_st.h
+ * @brief Router descriptor structure.
+ **/
+
 #ifndef ROUTERINFO_ST_H
 #define ROUTERINFO_ST_H
 
@@ -21,9 +26,6 @@ struct routerinfo_t {
   uint16_t dir_port; /**< Port for HTTP directory connections. */
 
   /** A router's IPv6 address, if it has one. */
-  /* XXXXX187 Actually these should probably be part of a list of addresses,
-   * not just a special case.  Use abstractions to access these; don't do it
-   * directly. */
   tor_addr_t ipv6_addr;
   uint16_t ipv6_orport;
 
@@ -112,4 +114,4 @@ struct routerinfo_t {
   uint8_t purpose;
 };
 
-#endif
+#endif /* !defined(ROUTERINFO_ST_H) */
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
index c7fa868929..96bea5a670 100644
--- a/src/feature/nodelist/routerlist.c
+++ b/src/feature/nodelist/routerlist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -67,12 +67,13 @@
 #include "core/mainloop/mainloop.h"
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/authcert.h"
@@ -116,9 +117,9 @@
 /* Typed wrappers for different digestmap types; used to avoid type
  * confusion. */
 
-DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t)
-DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t)
-DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(sdmap, digest_sd_map_t, signed_descriptor_t)
+DECLARE_TYPED_DIGESTMAP_FNS(rimap, digest_ri_map_t, routerinfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(eimap, digest_ei_map_t, extrainfo_t)
 #define SDMAP_FOREACH(map, keyvar, valvar)                              \
   DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \
                     valvar)
@@ -160,7 +161,7 @@ static time_t last_descriptor_download_attempted = 0;
  *
  * From time to time, we replace "cached-descriptors" with a new file
  * containing only the live, non-superseded descriptors, and clear
- * cached-routers.new.
+ * cached-descriptors.new.
  *
  * On startup, we read both files.
  */
@@ -1463,12 +1464,13 @@ router_descriptor_is_older_than,(const routerinfo_t *router, int seconds))
 }
 
 /** Add <b>router</b> to the routerlist, if we don't already have it.  Replace
- * older entries (if any) with the same key.  Note: Callers should not hold
- * their pointers to <b>router</b> if this function fails; <b>router</b>
- * will either be inserted into the routerlist or freed. Similarly, even
- * if this call succeeds, they should not hold their pointers to
- * <b>router</b> after subsequent calls with other routerinfo's -- they
- * might cause the original routerinfo to get freed.
+ * older entries (if any) with the same key.
+ *
+ * Note: Callers should not hold their pointers to <b>router</b> if this
+ * function fails; <b>router</b> will either be inserted into the routerlist or
+ * freed. Similarly, even if this call succeeds, they should not hold their
+ * pointers to <b>router</b> after subsequent calls with other routerinfo's --
+ * they might cause the original routerinfo to get freed.
  *
  * Returns the status for the operation. Might set *<b>msg</b> if it wants
  * the poster of the router to know something.
@@ -1930,6 +1932,8 @@ routerlist_remove_old_routers(void)
 void
 routerlist_descriptors_added(smartlist_t *sl, int from_cache)
 {
+  // XXXX use pubsub mechanism here.
+
   tor_assert(sl);
   control_event_descriptors_changed(sl);
   SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) {
@@ -2405,7 +2409,7 @@ max_dl_per_request(const or_options_t *options, int purpose)
   }
   /* If we're going to tunnel our connections, we can ask for a lot more
    * in a request. */
-  if (directory_must_use_begindir(options)) {
+  if (dirclient_must_use_begindir(options)) {
     max = 500;
   }
   return max;
@@ -2448,7 +2452,7 @@ launch_descriptor_downloads(int purpose,
   if (!n_downloadable)
     return;
 
-  if (!directory_fetches_dir_info_early(options)) {
+  if (!dirclient_fetches_dir_info_early(options)) {
     if (n_downloadable >= MAX_DL_TO_DELAY) {
       log_debug(LD_DIR,
                 "There are enough downloadable %ss to launch requests.",
@@ -2539,7 +2543,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
   int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0,
     n_inprogress=0, n_in_oldrouters=0;
 
-  if (directory_too_idle_to_fetch_descriptors(options, now))
+  if (dirclient_too_idle_to_fetch_descriptors(options, now))
     goto done;
   if (!consensus)
     goto done;
@@ -2559,8 +2563,15 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
   map = digestmap_new();
   list_pending_descriptor_downloads(map, 0);
   SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, void *, rsp) {
-      routerstatus_t *rs =
-        is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp;
+      routerstatus_t *rs;
+      vote_routerstatus_t *vrs;
+      if (is_vote) {
+        rs = &(((vote_routerstatus_t *)rsp)->status);
+        vrs = rsp;
+      } else {
+        rs = rsp;
+        vrs = NULL;
+      }
       signed_descriptor_t *sd;
       if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) {
         const routerinfo_t *ri;
@@ -2585,7 +2596,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
         ++n_delayed; /* Not ready for retry. */
         continue;
       }
-      if (authdir && dirserv_would_reject_router(rs)) {
+      if (authdir && is_vote && dirserv_would_reject_router(rs, vrs)) {
         ++n_would_reject;
         continue; /* We would throw it out immediately. */
       }
@@ -2915,7 +2926,7 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2)
       (r1->bandwidthburst != r2->bandwidthburst))
     return 0;
 
-  /* Did more than 12 hours pass? */
+  /* Has enough time passed between the publication times? */
   if (r1->cache_info.published_on + ROUTER_MAX_COSMETIC_TIME_DIFFERENCE
       < r2->cache_info.published_on)
     return 0;
@@ -2975,7 +2986,7 @@ routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey,
   digest256_matches = tor_memeq(ei->digest256,
                                 sd->extra_info_digest256, DIGEST256_LEN);
   digest256_matches |=
-    tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
+    fast_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
 
   /* The identity must match exactly to have been generated at the same time
    * by the same router. */
@@ -3059,7 +3070,7 @@ routerinfo_has_curve25519_onion_key(const routerinfo_t *ri)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -3227,6 +3238,8 @@ refresh_all_country_info(void)
     routerset_refresh_countries(options->EntryNodes);
   if (options->ExitNodes)
     routerset_refresh_countries(options->ExitNodes);
+  if (options->MiddleNodes)
+    routerset_refresh_countries(options->MiddleNodes);
   if (options->ExcludeNodes)
     routerset_refresh_countries(options->ExcludeNodes);
   if (options->ExcludeExitNodes)
diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h
index 5771ebb1ab..81a2343540 100644
--- a/src/feature/nodelist/routerlist.h
+++ b/src/feature/nodelist/routerlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -37,9 +37,12 @@ typedef enum was_router_added_t {
   ROUTER_WAS_NOT_WANTED = -6,
   /* Router descriptor was rejected because it was older than
    * OLD_ROUTER_DESC_MAX_AGE. */
-  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
-  /* DOCDOC */
-  ROUTER_CERTS_EXPIRED = -8
+  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'ROUTER_IS_ALREADY_KNOWN' */
+  /* Some certs on this router are expired. */
+  ROUTER_CERTS_EXPIRED = -8,
+  /* We couldn't format the annotations for this router. This is a directory
+   * authority bug. */
+  ROUTER_AUTHDIR_BUG_ANNOTATIONS = -10
 } was_router_added_t;
 
 /** How long do we avoid using a directory server after it's given us a 503? */
diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h
index 7446ead3cb..ec8933c7cb 100644
--- a/src/feature/nodelist/routerlist_st.h
+++ b/src/feature/nodelist/routerlist_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerlist_st.h
+ * @brief Router descriptor list structure.
+ **/
+
 #ifndef ROUTERLIST_ST_H
 #define ROUTERLIST_ST_H
 
@@ -36,5 +41,4 @@ struct routerlist_t {
   desc_store_t extrainfo_store;
 };
 
-#endif
-
+#endif /* !defined(ROUTERLIST_ST_H) */
diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c
index 55e2756959..2e06ecbf04 100644
--- a/src/feature/nodelist/routerset.c
+++ b/src/feature/nodelist/routerset.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
-n * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,7 +17,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
  *
  * Routersets are typically used for user-specified restrictions, and
  * are created by invoking routerset_new and routerset_parse from
- * config.c and confparse.c.  To use a routerset, invoke one of
+ * config.c and confmgt.c.  To use a routerset, invoke one of
  * routerset_contains_...() functions , or use
  * routerstatus_get_all_nodes() / routerstatus_subtract_nodes() to
  * manipulate a smartlist of node_t pointers.
@@ -34,6 +34,9 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/nickname.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
 #include "lib/geoip/geoip.h"
 
 #include "core/or/addr_policy_st.h"
@@ -41,6 +44,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
+#include "lib/confmgt/var_type_def_st.h"
 
 /** Return a new empty routerset. */
 routerset_t *
@@ -378,7 +382,7 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
   } else {
     /* We need to iterate over the routerlist to get all the ones of the
      * right kind. */
-    smartlist_t *nodes = nodelist_get_list();
+    const smartlist_t *nodes = nodelist_get_list();
     SMARTLIST_FOREACH(nodes, const node_t *, node, {
         if (running_only && !node->is_running)
           continue;
@@ -461,3 +465,111 @@ routerset_free_(routerset_t *routerset)
   bitarray_free(routerset->countries);
   tor_free(routerset);
 }
+
+/**
+ * config helper: parse a routerset-typed variable.
+ *
+ * Takes as input as a single line in <b>line</b>; writes its results into a
+ * routerset_t** passed as <b>target</b>.  On success return 0; on failure
+ * return -1 and store an error message into *<b>errmsg</b>.
+ **/
+/*
+ * Warning: For this type, the default value (NULL) and "" are sometimes
+ * considered different values.  That is generally risky, and best avoided for
+ * other types in the future.  For cases where we want the default to be "all
+ * routers" (like EntryNodes) we should add a new routerset value indicating
+ * "all routers" (see #31908)
+ */
+static int
+routerset_kv_parse(void *target, const config_line_t *line, char **errmsg,
+                  const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)target;
+  routerset_free(*p); // clear the old value, if any.
+  routerset_t *rs = routerset_new();
+  if (routerset_parse(rs, line->value, line->key) < 0) {
+    routerset_free(rs);
+    *errmsg = tor_strdup("Invalid router list.");
+    return -1;
+  } else {
+    if (routerset_is_empty(rs)) {
+      /* Represent empty sets as NULL. */
+      routerset_free(rs);
+    }
+    *p = rs;
+    return 0;
+  }
+}
+
+/**
+ * config helper: encode a routerset-typed variable.
+ *
+ * Return a newly allocated string containing the value of the
+ * routerset_t** passed as <b>value</b>.
+ */
+static char *
+routerset_encode(const void *value, const void *params)
+{
+  (void)params;
+  const routerset_t **p = (const routerset_t**)value;
+  return routerset_to_string(*p);
+}
+
+/**
+ * config helper: free and clear a routerset-typed variable.
+ *
+ * Clear the routerset_t** passed as <b>value</b>.
+ */
+static void
+routerset_clear(void *value, const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)value;
+  routerset_free(*p); // sets *p to NULL.
+}
+
+/**
+ * config helper: copy a routerset-typed variable.
+ *
+ * Takes it input from a routerset_t** in <b>src</b>; writes its output to a
+ * routerset_t** in <b>dest</b>.  Returns 0 on success, -1 on (impossible)
+ * failure.
+ **/
+static int
+routerset_copy(void *dest, const void *src, const void *params)
+{
+  (void)params;
+  routerset_t **output = (routerset_t**)dest;
+  const routerset_t *input = *(routerset_t**)src;
+  routerset_free(*output); // sets *output to NULL
+  if (! routerset_is_empty(input)) {
+    *output = routerset_new();
+    routerset_union(*output, input);
+  }
+  return 0;
+}
+
+/**
+ * Function table to implement a routerset_t-based configuration type.
+ **/
+static const var_type_fns_t routerset_type_fns = {
+  .kv_parse = routerset_kv_parse,
+  .encode = routerset_encode,
+  .clear = routerset_clear,
+  .copy = routerset_copy
+};
+
+/**
+ * Definition of a routerset_t-based configuration type.
+ *
+ * Values are mapped to and from strings using the format defined in
+ * routerset_parse(): nicknames, IP address patterns, and fingerprints--with
+ * optional space, separated by commas.
+ *
+ * Empty sets are represented as NULL.
+ **/
+const var_type_def_t ROUTERSET_type_defn = {
+  .name = "RouterList",
+  .fns = &routerset_type_fns
+};
diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h
index ca8b6fed93..0e4fedf64e 100644
--- a/src/feature/nodelist/routerset.h
+++ b/src/feature/nodelist/routerset.h
@@ -1,10 +1,10 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file routerlist.h
+ * \file routerset.h
  * \brief Header file for routerset.c
  **/
 
@@ -44,6 +44,10 @@ void routerset_free_(routerset_t *routerset);
 #define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
 int routerset_len(const routerset_t *set);
 
+struct var_type_def_t;
+extern const struct var_type_def_t ROUTERSET_type_defn;
+typedef routerset_t *config_decl_ROUTERSET;
+
 #ifdef ROUTERSET_PRIVATE
 #include "lib/container/bitarray.h"
 
diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h
index 288edf5982..735c754b31 100644
--- a/src/feature/nodelist/routerstatus_st.h
+++ b/src/feature/nodelist/routerstatus_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerstatus_st.h
+ * @brief Routerstatus (consensus entry) structure
+ **/
+
 #ifndef ROUTERSTATUS_ST_H
 #define ROUTERSTATUS_ST_H
 
@@ -12,6 +17,10 @@
 /** Contents of a single router entry in a network status object.
  */
 struct routerstatus_t {
+ /* This should be kept in sync with the function
+ * routerstatus_has_visibly_changed and the printing function
+ * routerstatus_format_entry in NS_CONTROL_PORT mode.
+ */
   time_t published_on; /**< When was this router published? */
   char nickname[MAX_NICKNAME_LEN+1]; /**< The nickname this router says it
                                       * has. */
@@ -47,6 +56,8 @@ struct routerstatus_t {
   unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
                              * or it claims to accept tunnelled dir requests.
                              */
+  unsigned int is_staledesc:1; /** True iff the authorities think this router
+                                * should upload a new descriptor soon. */
 
   unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
   unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
@@ -76,5 +87,4 @@ struct routerstatus_t {
 
 };
 
-#endif
-
+#endif /* !defined(ROUTERSTATUS_ST_H) */
diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h
index bdcebf184a..068f2a733c 100644
--- a/src/feature/nodelist/signed_descriptor_st.h
+++ b/src/feature/nodelist/signed_descriptor_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file signed_descriptor_st.h
+ * @brief Descriptor/extrainfo signature structure
+ **/
+
 #ifndef SIGNED_DESCRIPTOR_ST_H
 #define SIGNED_DESCRIPTOR_ST_H
 
@@ -57,5 +62,4 @@ struct signed_descriptor_t {
   unsigned int send_unencrypted : 1;
 };
 
-#endif
-
+#endif /* !defined(SIGNED_DESCRIPTOR_ST_H) */
diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c
index b0197e9f13..89cc9c88fb 100644
--- a/src/feature/nodelist/torcert.c
+++ b/src/feature/nodelist/torcert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -74,7 +74,7 @@ tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
   tor_assert(real_len == alloc_len);
   tor_assert(real_len > ED25519_SIG_LEN);
   uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
-  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
+  tor_assert(fast_mem_is_zero((char*)sig, ED25519_SIG_LEN));
 
   ed25519_signature_t signature;
   if (ed25519_sign(&signature, encoded,
@@ -290,8 +290,8 @@ tor_cert_describe_signature_status(const tor_cert_t *cert)
 }
 
 /** Return a new copy of <b>cert</b> */
-tor_cert_t *
-tor_cert_dup(const tor_cert_t *cert)
+MOCK_IMPL(tor_cert_t *,
+tor_cert_dup,(const tor_cert_t *cert))
 {
   tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
   if (cert->encoded)
diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h
index 492275b514..f8fba2b794 100644
--- a/src/feature/nodelist/torcert.h
+++ b/src/feature/nodelist/torcert.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file torcert.h
+ * @brief Header for torcert.c
+ **/
+
 #ifndef TORCERT_H_INCLUDED
 #define TORCERT_H_INCLUDED
 
@@ -71,7 +76,7 @@ int tor_cert_checksig(tor_cert_t *cert,
                       const ed25519_public_key_t *pubkey, time_t now);
 const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
 
-tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
+MOCK_DECL(tor_cert_t *,tor_cert_dup,(const tor_cert_t *cert));
 int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 
diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h
index 366754c166..ad0ee3f23b 100644
--- a/src/feature/nodelist/vote_routerstatus_st.h
+++ b/src/feature/nodelist/vote_routerstatus_st.h
@@ -1,9 +1,13 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file vote_routerstatus_st.h
+ * @brief Routerstatus (vote entry) structure
+ **/
 #ifndef VOTE_ROUTERSTATUS_ST_H
 #define VOTE_ROUTERSTATUS_ST_H
 
@@ -38,4 +42,4 @@ struct vote_routerstatus_t {
   uint8_t ed25519_id[ED25519_PUBKEY_LEN];
 };
 
-#endif
+#endif /* !defined(VOTE_ROUTERSTATUS_ST_H) */
diff --git a/src/feature/relay/.may_include b/src/feature/relay/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/relay/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/relay/circuitbuild_relay.c b/src/feature/relay/circuitbuild_relay.c
new file mode 100644
index 0000000000..b89866b477
--- /dev/null
+++ b/src/feature/relay/circuitbuild_relay.c
@@ -0,0 +1,549 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file circuitbuild_relay.c
+ * @brief Implements the details of exteding circuits (by relaying extend
+ * cells as create cells, and answering create cells).
+ *
+ * On the server side, this module handles the logic of responding to
+ * RELAY_EXTEND requests, using circuit_extend() and onionskin_answer().
+ *
+ * The shared client and server code is in core/or/circuitbuild.c.
+ **/
+
+#include "orconfig.h"
+#include "feature/relay/circuitbuild_relay.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/crypto/relay_crypto.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/circuit_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/or_circuit_st.h"
+
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/onion.h"
+#include "core/or/relay.h"
+
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+
+/* Before replying to an extend cell, check the state of the circuit
+ * <b>circ</b>, and the configured tor mode.
+ *
+ * <b>circ</b> must not be NULL.
+ *
+ * If the state and mode are valid, return 0.
+ * Otherwise, if they are invalid, log a protocol warning, and return -1.
+ */
+STATIC int
+circuit_extend_state_valid_helper(const struct circuit_t *circ)
+{
+  if (!server_mode(get_options())) {
+    circuitbuild_warn_client_extend();
+    return -1;
+  }
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  if (circ->n_chan) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "n_chan already set. Bug/attack. Closing.");
+    return -1;
+  }
+
+  if (circ->n_hop) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "conn to next hop already launched. Bug/attack. Closing.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/* Make sure the extend cell <b>ec</b> has an ed25519 link specifier.
+ *
+ * First, check that the RSA node id is valid.
+ * If the node id is valid, add the ed25519 link specifier (if required),
+ * and return 0.
+ *
+ * Otherwise, if the node id is invalid, log a protocol warning,
+ * and return -1.(And do not modify the extend cell.)
+ *
+ * Must be called before circuit_extend_lspec_valid_helper().
+ */
+STATIC int
+circuit_extend_add_ed25519_helper(struct extend_cell_t *ec)
+{
+  IF_BUG_ONCE(!ec) {
+    return -1;
+  }
+
+  /* Check if they asked us for 0000..0000. We support using
+   * an empty fingerprint for the first hop (e.g. for a bridge relay),
+   * but we don't want to let clients send us extend cells for empty
+   * fingerprints -- a) because it opens the user up to a mitm attack,
+   * and b) because it lets an attacker force the relay to hold open a
+   * new TLS connection for each extend request. */
+  if (tor_digest_is_zero((const char*)ec->node_id)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend without specifying an id_digest.");
+    return -1;
+  }
+
+  /* Fill in ed_pubkey if it was not provided and we can infer it from
+   * our networkstatus */
+  if (ed25519_public_key_is_zero(&ec->ed_pubkey)) {
+    const node_t *node = node_get_by_id((const char*)ec->node_id);
+    const ed25519_public_key_t *node_ed_id = NULL;
+    if (node &&
+        node_supports_ed25519_link_authentication(node, 1) &&
+        (node_ed_id = node_get_ed25519_id(node))) {
+      ed25519_pubkey_copy(&ec->ed_pubkey, node_ed_id);
+    }
+  }
+
+  return 0;
+}
+
+/* Check if the address and port in the tor_addr_port_t <b>ap</b> are valid,
+ * and are allowed by the current ExtendAllowPrivateAddresses config.
+ *
+ * If they are valid, return true.
+ * Otherwise, if they are invalid, return false.
+ *
+ * If <b>log_zero_addrs</b> is true, log warnings about zero addresses at
+ * <b>log_level</b>. If <b>log_internal_addrs</b> is true, log warnings about
+ * internal addresses at <b>log_level</b>.
+ */
+static bool
+circuit_extend_addr_port_is_valid(const struct tor_addr_port_t *ap,
+                                  bool log_zero_addrs, bool log_internal_addrs,
+                                  int log_level)
+{
+  /* It's safe to print the family. But we don't want to print the address,
+   * unless specifically configured to do so. (Zero addresses aren't sensitive,
+   * But some internal addresses might be.)*/
+
+  if (!tor_addr_port_is_valid_ap(ap, 0)) {
+    if (log_zero_addrs) {
+      log_fn(log_level, LD_PROTOCOL,
+             "Client asked me to extend to a zero destination port or "
+             "%s address '%s'.",
+             fmt_addr_family(&ap->addr), safe_str(fmt_addrport_ap(ap)));
+    }
+    return false;
+  }
+
+  if (tor_addr_is_internal(&ap->addr, 0) &&
+      !get_options()->ExtendAllowPrivateAddresses) {
+    if (log_internal_addrs) {
+      log_fn(log_level, LD_PROTOCOL,
+             "Client asked me to extend to a private %s address '%s'.",
+             fmt_addr_family(&ap->addr),
+             safe_str(fmt_and_decorate_addr(&ap->addr)));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+/* Before replying to an extend cell, check the link specifiers in the extend
+ * cell <b>ec</b>, which was received on the circuit <b>circ</b>.
+ *
+ * If they are valid, return 0.
+ * Otherwise, if they are invalid, log a protocol warning, and return -1.
+ *
+ * Must be called after circuit_extend_add_ed25519_helper().
+ */
+STATIC int
+circuit_extend_lspec_valid_helper(const struct extend_cell_t *ec,
+                                  const struct circuit_t *circ)
+{
+  IF_BUG_ONCE(!ec) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  /* Check the addresses, without logging */
+  const int ipv4_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                                           false, false, 0);
+  const int ipv6_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                                           false, false, 0);
+  /* We need at least one valid address */
+  if (!ipv4_valid && !ipv6_valid) {
+    /* Now, log the invalid addresses at protocol warning level */
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                      true, true, LOG_PROTOCOL_WARN);
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                      true, true, LOG_PROTOCOL_WARN);
+    /* And fail */
+    return -1;
+  } else if (!ipv4_valid) {
+    /* Always log unexpected internal addresses, but go on to use the other
+     * valid address */
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                      false, true, LOG_PROTOCOL_WARN);
+  } else if (!ipv6_valid) {
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                      false, true, LOG_PROTOCOL_WARN);
+  }
+
+  IF_BUG_ONCE(circ->magic != OR_CIRCUIT_MAGIC) {
+    return -1;
+  }
+
+  const channel_t *p_chan = CONST_TO_OR_CIRCUIT(circ)->p_chan;
+
+  IF_BUG_ONCE(!p_chan) {
+    return -1;
+  }
+
+  /* Next, check if we're being asked to connect to the hop that the
+   * extend cell came from. There isn't any reason for that, and it can
+   * assist circular-path attacks. */
+  if (tor_memeq(ec->node_id, p_chan->identity_digest, DIGEST_LEN)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend back to the previous hop.");
+    return -1;
+  }
+
+  /* Check the previous hop Ed25519 ID too */
+  if (! ed25519_public_key_is_zero(&ec->ed_pubkey) &&
+      ed25519_pubkey_eq(&ec->ed_pubkey, &p_chan->ed25519_identity)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend back to the previous hop "
+           "(by Ed25519 ID).");
+    return -1;
+  }
+
+  return 0;
+}
+
+/* If possible, return a supported, non-NULL IP address.
+ *
+ * If both addresses are supported and non-NULL, choose one uniformly at
+ * random.
+ *
+ * If we have an IPv6-only extend, but IPv6 is not supported, returns NULL.
+ * If both addresses are NULL, also returns NULL. */
+STATIC const tor_addr_port_t *
+circuit_choose_ip_ap_for_extend(const tor_addr_port_t *ipv4_ap,
+                                const tor_addr_port_t *ipv6_ap)
+{
+  const bool ipv6_supported = router_can_extend_over_ipv6(get_options());
+
+  /* If IPv6 is not supported, we can't use the IPv6 address. */
+  if (!ipv6_supported) {
+    ipv6_ap = NULL;
+  }
+
+  /* If there is no IPv6 address, IPv4 is always supported.
+   * Until clients include IPv6 ORPorts, and most relays support IPv6,
+   * this is the most common case. */
+  if (!ipv6_ap) {
+    return ipv4_ap;
+  }
+
+  /* If there is no IPv4 address, return the (possibly NULL) IPv6 address. */
+  if (!ipv4_ap) {
+    return ipv6_ap;
+  }
+
+  /* Now we have an IPv4 and an IPv6 address, and IPv6 is supported.
+   * So make an IPv6 connection at random, with probability 1 in N.
+   *   1 means "always IPv6 (and no IPv4)"
+   *   2 means "equal probability of IPv4 or IPv6"
+   *   ... (and so on) ...
+   *   (UINT_MAX - 1) means "almost always IPv4 (and almost never IPv6)"
+   * To disable IPv6, set ipv6_supported to 0.
+   */
+#define IPV6_CONNECTION_ONE_IN_N 2
+
+  bool choose_ipv6 = crypto_fast_rng_one_in_n(get_thread_fast_rng(),
+                                              IPV6_CONNECTION_ONE_IN_N);
+  if (choose_ipv6) {
+    return ipv6_ap;
+  } else {
+    return ipv4_ap;
+  }
+}
+
+/* When there is no open channel for an extend cell <b>ec</b>, set up the
+ * circuit <b>circ</b> to wait for a new connection.
+ *
+ * If <b>should_launch</b> is true, open a new connection. (Otherwise, we are
+ * already waiting for a new connection to the same relay.)
+ *
+ * Check if IPv6 extends are supported by our current configuration. If they
+ * are, new connections may be made over IPv4 or IPv6. (IPv4 connections are
+ * always supported.)
+ */
+STATIC void
+circuit_open_connection_for_extend(const struct extend_cell_t *ec,
+                                   struct circuit_t *circ,
+                                   int should_launch)
+{
+  /* We have to check circ first, so we can close it on all other failures */
+  IF_BUG_ONCE(!circ) {
+    /* We can't mark a NULL circuit for close. */
+    return;
+  }
+
+  /* Now we know that circ is not NULL */
+  IF_BUG_ONCE(!ec) {
+    circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+    return;
+  }
+
+  /* Check the addresses, without logging */
+  const int ipv4_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                                           false, false, 0);
+  const int ipv6_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                                           false, false, 0);
+
+  IF_BUG_ONCE(!ipv4_valid && !ipv6_valid) {
+    /* circuit_extend_lspec_valid_helper() should have caught this */
+    circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+    return;
+  }
+
+  const tor_addr_port_t *chosen_ap = circuit_choose_ip_ap_for_extend(
+                                        ipv4_valid ? &ec->orport_ipv4 : NULL,
+                                        ipv6_valid ? &ec->orport_ipv6 : NULL);
+  if (!chosen_ap) {
+    /* An IPv6-only extend, but IPv6 is not supported */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Received IPv6-only extend, but we don't have an IPv6 ORPort.");
+    circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+    return;
+  }
+
+  circ->n_hop = extend_info_new(NULL /*nickname*/,
+                                (const char*)ec->node_id,
+                                &ec->ed_pubkey,
+                                NULL, /*onion_key*/
+                                NULL, /*curve25519_key*/
+                                &chosen_ap->addr,
+                                chosen_ap->port);
+
+  circ->n_chan_create_cell = tor_memdup(&ec->create_cell,
+                                        sizeof(ec->create_cell));
+
+  circuit_set_state(circ, CIRCUIT_STATE_CHAN_WAIT);
+
+  if (should_launch) {
+    /* we should try to open a connection */
+    channel_t *n_chan = channel_connect_for_circuit(
+                                                &circ->n_hop->addr,
+                                                circ->n_hop->port,
+                                                circ->n_hop->identity_digest,
+                                                &circ->n_hop->ed_identity);
+    if (!n_chan) {
+      log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
+      circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+      return;
+    }
+    log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
+  }
+}
+
+/** Take the 'extend' <b>cell</b>, pull out addr/port plus the onion
+ * skin and identity digest for the next hop. If we're already connected,
+ * pass the onion skin to the next hop using a create cell; otherwise
+ * launch a new OR connection, and <b>circ</b> will notice when the
+ * connection succeeds or fails.
+ *
+ * Return -1 if we want to warn and tear down the circuit, else return 0.
+ */
+int
+circuit_extend(struct cell_t *cell, struct circuit_t *circ)
+{
+  channel_t *n_chan;
+  relay_header_t rh;
+  extend_cell_t ec;
+  const char *msg = NULL;
+  int should_launch = 0;
+
+  IF_BUG_ONCE(!cell) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  if (circuit_extend_state_valid_helper(circ) < 0)
+    return -1;
+
+  relay_header_unpack(&rh, cell->payload);
+
+  if (extend_cell_parse(&ec, rh.command,
+                        cell->payload+RELAY_HEADER_SIZE,
+                        rh.length) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Can't parse extend cell. Closing circuit.");
+    return -1;
+  }
+
+  if (circuit_extend_add_ed25519_helper(&ec) < 0)
+    return -1;
+
+  if (circuit_extend_lspec_valid_helper(&ec, circ) < 0)
+    return -1;
+
+  /* Check the addresses, without logging */
+  const int ipv4_valid = circuit_extend_addr_port_is_valid(&ec.orport_ipv4,
+                                                           false, false, 0);
+  const int ipv6_valid = circuit_extend_addr_port_is_valid(&ec.orport_ipv6,
+                                                           false, false, 0);
+  IF_BUG_ONCE(!ipv4_valid && !ipv6_valid) {
+    /* circuit_extend_lspec_valid_helper() should have caught this */
+    return -1;
+  }
+
+  n_chan = channel_get_for_extend((const char*)ec.node_id,
+                                  &ec.ed_pubkey,
+                                  ipv4_valid ? &ec.orport_ipv4.addr : NULL,
+                                  ipv6_valid ? &ec.orport_ipv6.addr : NULL,
+                                  &msg,
+                                  &should_launch);
+
+  if (!n_chan) {
+    /* We can't use fmt_addr*() twice in the same function call,
+     * because it uses a static buffer. */
+    log_debug(LD_CIRC|LD_OR, "Next router IPv4 (%s): %s.",
+              fmt_addrport_ap(&ec.orport_ipv4),
+              msg ? msg : "????");
+    log_debug(LD_CIRC|LD_OR, "Next router IPv6 (%s).",
+              fmt_addrport_ap(&ec.orport_ipv6));
+
+    circuit_open_connection_for_extend(&ec, circ, should_launch);
+
+    /* return success. The onion/circuit/etc will be taken care of
+     * automatically (may already have been) whenever n_chan reaches
+     * OR_CONN_STATE_OPEN.
+     */
+    return 0;
+  } else {
+    /* Connection is already established.
+     * So we need to extend the circuit to the next hop. */
+    tor_assert(!circ->n_hop);
+    circ->n_chan = n_chan;
+    log_debug(LD_CIRC,
+              "n_chan is %s.",
+              channel_get_canonical_remote_descr(n_chan));
+
+    if (circuit_deliver_create_cell(circ, &ec.create_cell, 1) < 0)
+      return -1;
+
+    return 0;
+  }
+}
+
+/** On a relay, accept a create cell, initialise a circuit, and send a
+ * created cell back.
+ *
+ * Given:
+ *   - a response payload consisting of:
+ *     - the <b>created_cell</b> and
+ *     - an optional <b>rend_circ_nonce</b>, and
+ *   - <b>keys</b> of length <b>keys_len</b>, which must be
+ *     CPATH_KEY_MATERIAL_LEN;
+ * then:
+ *   - initialize the circuit <b>circ</b>'s cryptographic material,
+ *   - set the circuit's state to open, and
+ *   - send a created cell back on that circuit.
+ *
+ * If we haven't found our ORPorts reachable yet, and the channel meets the
+ * necessary conditions, mark the relevant ORPorts as reachable.
+ *
+ * Returns -1 if cell or circuit initialisation fails.
+ */
+int
+onionskin_answer(struct or_circuit_t *circ,
+                 const created_cell_t *created_cell,
+                 const char *keys, size_t keys_len,
+                 const uint8_t *rend_circ_nonce)
+{
+  cell_t cell;
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!created_cell) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!keys) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!rend_circ_nonce) {
+    return -1;
+  }
+
+  tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN);
+
+  if (created_cell_format(&cell, created_cell) < 0) {
+    log_warn(LD_BUG,"couldn't format created cell (type=%d, len=%d).",
+             (int)created_cell->cell_type, (int)created_cell->handshake_len);
+    return -1;
+  }
+  cell.circ_id = circ->p_circ_id;
+
+  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+
+  log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.",
+            (unsigned int)get_uint32(keys),
+            (unsigned int)get_uint32(keys+20));
+  if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) {
+    log_warn(LD_BUG,"Circuit initialization failed.");
+    return -1;
+  }
+
+  memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN);
+
+  int used_create_fast = (created_cell->cell_type == CELL_CREATED_FAST);
+
+  append_cell_to_circuit_queue(TO_CIRCUIT(circ),
+                               circ->p_chan, &cell, CELL_DIRECTION_IN, 0);
+  log_debug(LD_CIRC,"Finished sending '%s' cell.",
+            used_create_fast ? "created_fast" : "created");
+
+  /* Ignore the local bit when ExtendAllowPrivateAddresses is set:
+   * it violates the assumption that private addresses are local.
+   * Also, many test networks run on local addresses, and
+   * TestingTorNetwork sets ExtendAllowPrivateAddresses. */
+  if ((!channel_is_local(circ->p_chan)
+       || get_options()->ExtendAllowPrivateAddresses)
+      && !channel_is_outgoing(circ->p_chan)) {
+    /* record that we could process create cells from a non-local conn
+     * that we didn't initiate; presumably this means that create cells
+     * can reach us too. */
+    router_orport_found_reachable();
+  }
+
+  return 0;
+}
diff --git a/src/feature/relay/circuitbuild_relay.h b/src/feature/relay/circuitbuild_relay.h
new file mode 100644
index 0000000000..0783161538
--- /dev/null
+++ b/src/feature/relay/circuitbuild_relay.h
@@ -0,0 +1,87 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file circuitbuild_relay.h
+ * @brief Header for feature/relay/circuitbuild_relay.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_CIRCUITBUILD_RELAY_H
+#define TOR_FEATURE_RELAY_CIRCUITBUILD_RELAY_H
+
+#include "lib/cc/torint.h"
+#include "lib/log/log.h"
+
+#include "app/config/config.h"
+
+struct cell_t;
+struct created_cell_t;
+
+struct circuit_t;
+struct or_circuit_t;
+struct extend_cell_t;
+
+/* Log a protocol warning about getting an extend cell on a client. */
+static inline void
+circuitbuild_warn_client_extend(void)
+{
+  log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+         "Got an extend cell, but running as a client. Closing.");
+}
+
+#ifdef HAVE_MODULE_RELAY
+
+int circuit_extend(struct cell_t *cell, struct circuit_t *circ);
+
+int onionskin_answer(struct or_circuit_t *circ,
+                     const struct created_cell_t *created_cell,
+                     const char *keys, size_t keys_len,
+                     const uint8_t *rend_circ_nonce);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+static inline int
+circuit_extend(struct cell_t *cell, struct circuit_t *circ)
+{
+  (void)cell;
+  (void)circ;
+  circuitbuild_warn_client_extend();
+  return -1;
+}
+
+static inline int
+onionskin_answer(struct or_circuit_t *circ,
+                 const struct created_cell_t *created_cell,
+                 const char *keys, size_t keys_len,
+                 const uint8_t *rend_circ_nonce)
+{
+  (void)circ;
+  (void)created_cell;
+  (void)keys;
+  (void)keys_len;
+  (void)rend_circ_nonce;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC int circuit_extend_state_valid_helper(const struct circuit_t *circ);
+STATIC int circuit_extend_add_ed25519_helper(struct extend_cell_t *ec);
+STATIC int circuit_extend_lspec_valid_helper(const struct extend_cell_t *ec,
+                                             const struct circuit_t *circ);
+STATIC const tor_addr_port_t * circuit_choose_ip_ap_for_extend(
+                                             const tor_addr_port_t *ipv4_ap,
+                                             const tor_addr_port_t *ipv6_ap);
+STATIC void circuit_open_connection_for_extend(const struct extend_cell_t *ec,
+                                               struct circuit_t *circ,
+                                               int should_launch);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_CIRCUITBUILD_RELAY_H) */
diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c
index e20a39482f..b83bd9b758 100644
--- a/src/feature/relay/dns.c
+++ b/src/feature/relay/dns.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -59,7 +59,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
@@ -146,9 +146,9 @@ cached_resolve_hash(cached_resolve_t *a)
 }
 
 HT_PROTOTYPE(cache_map, cached_resolve_t, node, cached_resolve_hash,
-             cached_resolves_eq)
+             cached_resolves_eq);
 HT_GENERATE2(cache_map, cached_resolve_t, node, cached_resolve_hash,
-             cached_resolves_eq, 0.6, tor_reallocarray_, tor_free_)
+             cached_resolves_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** Initialize the DNS cache. */
 static void
@@ -268,22 +268,6 @@ has_dns_init_failed(void)
   return nameserver_config_failed;
 }
 
-/** Helper: Given a TTL from a DNS response, determine what TTL to give the
- * OP that asked us to resolve it, and how long to cache that record
- * ourselves. */
-uint32_t
-dns_clip_ttl(uint32_t ttl)
-{
-  /* This logic is a defense against "DefectTor" DNS-based traffic
-   * confirmation attacks, as in https://nymity.ch/tor-dns/tor-dns.pdf .
-   * We only give two values: a "low" value and a "high" value.
-   */
-  if (ttl < MIN_DNS_TTL_AT_EXIT)
-    return MIN_DNS_TTL_AT_EXIT;
-  else
-    return MAX_DNS_TTL_AT_EXIT;
-}
-
 /** Helper: free storage held by an entry in the DNS cache. */
 static void
 free_cached_resolve_(cached_resolve_t *r)
@@ -521,7 +505,7 @@ send_resolved_cell,(edge_connection_t *conn, uint8_t answer_type,
   uint32_t ttl;
 
   buf[0] = answer_type;
-  ttl = dns_clip_ttl(conn->address_ttl);
+  ttl = clip_dns_ttl(conn->address_ttl);
 
   switch (answer_type)
     {
@@ -593,7 +577,7 @@ send_resolved_hostname_cell,(edge_connection_t *conn,
   size_t namelen = strlen(hostname);
 
   tor_assert(namelen < 256);
-  ttl = dns_clip_ttl(conn->address_ttl);
+  ttl = clip_dns_ttl(conn->address_ttl);
 
   buf[0] = RESOLVED_TYPE_HOSTNAME;
   buf[1] = (uint8_t)namelen;
@@ -987,25 +971,6 @@ assert_connection_edge_not_dns_pending(edge_connection_t *conn)
 #endif /* 1 */
 }
 
-/** Log an error and abort if any connection waiting for a DNS resolve is
- * corrupted. */
-void
-assert_all_pending_dns_resolves_ok(void)
-{
-  pending_connection_t *pend;
-  cached_resolve_t **resolve;
-
-  HT_FOREACH(resolve, cache_map, &cache_root) {
-    for (pend = (*resolve)->pending_connections;
-         pend;
-         pend = pend->next) {
-      assert_connection_ok(TO_CONN(pend->conn), 0);
-      tor_assert(!SOCKET_OK(pend->conn->base_.s));
-      tor_assert(!connection_in_array(TO_CONN(pend->conn)));
-    }
-  }
-}
-
 /** Remove <b>conn</b> from the list of connections waiting for conn-\>address.
  */
 void
@@ -1063,7 +1028,7 @@ connection_dns_remove(edge_connection_t *conn)
  * the resolve for <b>address</b> itself, and remove any cached results for
  * <b>address</b> from the cache.
  */
-MOCK_IMPL(void,
+MOCK_IMPL(STATIC void,
 dns_cancel_pending_resolve,(const char *address))
 {
   pending_connection_t *pend;
@@ -1338,7 +1303,7 @@ make_pending_resolve_cached(cached_resolve_t *resolve)
         resolve->ttl_hostname < ttl)
       ttl = resolve->ttl_hostname;
 
-    set_expiry(new_resolve, time(NULL) + dns_clip_ttl(ttl));
+    set_expiry(new_resolve, time(NULL) + clip_dns_ttl(ttl));
   }
 
   assert_cache_ok();
@@ -1360,6 +1325,42 @@ evdns_err_is_transient(int err)
   }
 }
 
+/**
+ * Return number of configured nameservers in <b>the_evdns_base</b>.
+ */
+size_t
+number_of_configured_nameservers(void)
+{
+  return evdns_base_count_nameservers(the_evdns_base);
+}
+
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+/**
+ * Return address of configured nameserver in <b>the_evdns_base</b>
+ * at index <b>idx</b>.
+ */
+tor_addr_t *
+configured_nameserver_address(const size_t idx)
+{
+ struct sockaddr_storage sa;
+ ev_socklen_t sa_len = sizeof(sa);
+
+ if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx,
+                                    (struct sockaddr *)&sa,
+                                    sa_len) > 0) {
+   tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t));
+   if (tor_addr_from_sockaddr(tor_addr,
+                              (const struct sockaddr *)&sa,
+                              NULL) == 0) {
+     return tor_addr;
+   }
+   tor_free(tor_addr);
+ }
+
+ return NULL;
+}
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
 /** Configure eventdns nameservers if force is true, or if the configuration
  * has changed since the last time we called this function, or if we failed on
  * our last attempt.  On Unix, this reads from /etc/resolv.conf or
@@ -1391,16 +1392,23 @@ configure_nameservers(int force)
   evdns_set_log_fn(evdns_log_cb);
   if (conf_fname) {
     log_debug(LD_FS, "stat()ing %s", conf_fname);
-    if (stat(sandbox_intern_string(conf_fname), &st)) {
+    int missing_resolv_conf = 0;
+    int stat_res = stat(sandbox_intern_string(conf_fname), &st);
+
+    if (stat_res) {
       log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s",
                conf_fname, strerror(errno));
-      goto err;
-    }
-    if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname)
+      missing_resolv_conf = 1;
+    } else if (!force && resolv_conf_fname &&
+               !strcmp(conf_fname,resolv_conf_fname)
         && st.st_mtime == resolv_conf_mtime) {
       log_info(LD_EXIT, "No change to '%s'", conf_fname);
       return 0;
     }
+
+    if (stat_res == 0 && st.st_size == 0)
+      missing_resolv_conf = 1;
+
     if (nameservers_configured) {
       evdns_base_search_clear(the_evdns_base);
       evdns_base_clear_nameservers_and_suspend(the_evdns_base);
@@ -1413,20 +1421,34 @@ configure_nameservers(int force)
           sandbox_intern_string("/etc/hosts"));
     }
 #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */
-    log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
-    if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
-        sandbox_intern_string(conf_fname)))) {
-      log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)",
-               conf_fname, conf_fname, r);
-      goto err;
-    }
-    if (evdns_base_count_nameservers(the_evdns_base) == 0) {
-      log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname);
-      goto err;
+
+    if (!missing_resolv_conf) {
+      log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
+      if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
+          sandbox_intern_string(conf_fname)))) {
+        log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers "
+                          "in '%s' (%d)", conf_fname, conf_fname, r);
+
+        if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" -
+          goto err; // in which case we expect libevent to add 127.0.0.1 as
+                    // fallback.
+      }
+      if (evdns_base_count_nameservers(the_evdns_base) == 0) {
+        log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.",
+                 conf_fname);
+      }
+
+      tor_free(resolv_conf_fname);
+      resolv_conf_fname = tor_strdup(conf_fname);
+      resolv_conf_mtime = st.st_mtime;
+    } else {
+      log_warn(LD_EXIT, "Could not read your DNS config from '%s' - "
+                        "please investigate your DNS configuration. "
+                        "This is possibly a problem. Meanwhile, falling"
+                        " back to local DNS at 127.0.0.1.", conf_fname);
+      evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1");
     }
-    tor_free(resolv_conf_fname);
-    resolv_conf_fname = tor_strdup(conf_fname);
-    resolv_conf_mtime = st.st_mtime;
+
     if (nameservers_configured)
       evdns_base_resume(the_evdns_base);
   }
@@ -1569,12 +1591,17 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses,
     } else if (type == DNS_IPv6_AAAA && count) {
       char answer_buf[TOR_ADDR_BUF_LEN];
       char *escaped_address;
+      const char *ip_str;
       struct in6_addr *addrs = addresses;
       tor_addr_from_in6(&addr, &addrs[0]);
-      tor_inet_ntop(AF_INET6, &addrs[0], answer_buf, sizeof(answer_buf));
+      ip_str = tor_inet_ntop(AF_INET6, &addrs[0], answer_buf,
+                             sizeof(answer_buf));
       escaped_address = esc_for_log(string_address);
 
-      if (answer_is_wildcarded(answer_buf)) {
+      if (BUG(ip_str == NULL)) {
+        log_warn(LD_EXIT, "tor_inet_ntop() failed!");
+        result = DNS_ERR_NOTEXIST;
+      } else if (answer_is_wildcarded(answer_buf)) {
         log_debug(LD_EXIT, "eventdns said that %s resolves to ISP-hijacked "
                   "address %s; treating as a failure.",
                   safe_str(escaped_address),
@@ -1841,6 +1868,7 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl,
                               void *addresses, void *arg)
 {
   (void)ttl;
+  const char *ip_str;
   ++n_wildcard_requests;
   if (result == DNS_ERR_NONE && count) {
     char *string_address = arg;
@@ -1850,16 +1878,22 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl,
       for (i = 0; i < count; ++i) {
         char answer_buf[INET_NTOA_BUF_LEN+1];
         struct in_addr in;
+        int ntoa_res;
         in.s_addr = addrs[i];
-        tor_inet_ntoa(&in, answer_buf, sizeof(answer_buf));
-        wildcard_increment_answer(answer_buf);
+        ntoa_res = tor_inet_ntoa(&in, answer_buf, sizeof(answer_buf));
+        tor_assert_nonfatal(ntoa_res >= 0);
+        if (ntoa_res > 0)
+          wildcard_increment_answer(answer_buf);
       }
     } else if (type == DNS_IPv6_AAAA) {
       const struct in6_addr *addrs = addresses;
       for (i = 0; i < count; ++i) {
         char answer_buf[TOR_ADDR_BUF_LEN+1];
-        tor_inet_ntop(AF_INET6, &addrs[i], answer_buf, sizeof(answer_buf));
-        wildcard_increment_answer(answer_buf);
+        ip_str = tor_inet_ntop(AF_INET6, &addrs[i], answer_buf,
+                               sizeof(answer_buf));
+        tor_assert_nonfatal(ip_str);
+        if (ip_str)
+          wildcard_increment_answer(answer_buf);
       }
     }
 
@@ -1994,12 +2028,12 @@ dns_launch_correctness_checks(void)
 
   /* Wait a while before launching requests for test addresses, so we can
    * get the results from checking for wildcarding. */
-  if (! launch_event)
+  if (!launch_event)
     launch_event = tor_evtimer_new(tor_libevent_get_base(),
                                    launch_test_addresses, NULL);
   timeout.tv_sec = 30;
   timeout.tv_usec = 0;
-  if (evtimer_add(launch_event, &timeout)<0) {
+  if (evtimer_add(launch_event, &timeout) < 0) {
     log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking");
   }
 }
@@ -2131,7 +2165,7 @@ dns_cache_handle_oom(time_t now, size_t min_remove_bytes)
     total_bytes_removed += bytes_removed;
 
     /* Increase time_inc by a reasonable fraction. */
-    time_inc += (MAX_DNS_TTL_AT_EXIT / 4);
+    time_inc += (MAX_DNS_TTL / 4);
   } while (total_bytes_removed < min_remove_bytes);
 
   return total_bytes_removed;
diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h
index e4474cdf43..120b75bf8d 100644
--- a/src/feature/relay/dns.h
+++ b/src/feature/relay/dns.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,29 +12,14 @@
 #ifndef TOR_DNS_H
 #define TOR_DNS_H
 
-/** Lowest value for DNS ttl that a server will give. */
-#define MIN_DNS_TTL_AT_EXIT (5*60)
-/** Highest value for DNS ttl that a server will give. */
-#define MAX_DNS_TTL_AT_EXIT (60*60)
-
-/** How long do we keep DNS cache entries before purging them (regardless of
- * their TTL)? */
-#define MAX_DNS_ENTRY_AGE (3*60*60)
-/** How long do we cache/tell clients to cache DNS records when no TTL is
- * known? */
-#define DEFAULT_DNS_TTL (30*60)
+#ifdef HAVE_MODULE_RELAY
 
 int dns_init(void);
 int has_dns_init_failed(void);
-void dns_free_all(void);
-uint32_t dns_clip_ttl(uint32_t ttl);
 int dns_reset(void);
 void connection_dns_remove(edge_connection_t *conn);
 void assert_connection_edge_not_dns_pending(edge_connection_t *conn);
-void assert_all_pending_dns_resolves_ok(void);
-MOCK_DECL(void,dns_cancel_pending_resolve,(const char *question));
 int dns_resolve(edge_connection_t *exitconn);
-void dns_launch_correctness_checks(void);
 int dns_seems_to_be_broken(void);
 int dns_seems_to_be_broken_for_ipv6(void);
 void dns_reset_correctness_checks(void);
@@ -42,9 +27,57 @@ size_t dns_cache_total_allocation(void);
 void dump_dns_mem_usage(int severity);
 size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes);
 
+/* These functions are only used within the feature/relay module, and don't
+ * need stubs. */
+void dns_free_all(void);
+void dns_launch_correctness_checks(void);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define dns_init() (0)
+#define dns_seems_to_be_broken() (0)
+#define has_dns_init_failed() (0)
+#define dns_cache_total_allocation() (0)
+
+#define dns_reset_correctness_checks() STMT_NIL
+
+#define assert_connection_edge_not_dns_pending(conn) \
+  ((void)(conn))
+#define dump_dns_mem_usage(severity)\
+  ((void)(severity))
+#define dns_cache_handle_oom(now, bytes) \
+  ((void)(now), (void)(bytes), 0)
+
+#define connection_dns_remove(conn) \
+  STMT_BEGIN                        \
+  (void)(conn);                     \
+  tor_assert_nonfatal_unreached();  \
+  STMT_END
+
+static inline int
+dns_reset(void)
+{
+  return 0;
+}
+static inline int
+dns_resolve(edge_connection_t *exitconn)
+{
+  (void)exitconn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #ifdef DNS_PRIVATE
 #include "feature/relay/dns_structs.h"
 
+size_t number_of_configured_nameservers(void);
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+tor_addr_t *configured_nameserver_address(const size_t idx);
+#endif
+
+MOCK_DECL(STATIC void,dns_cancel_pending_resolve,(const char *question));
 MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn,
 int is_resolve,or_circuit_t *oncirc, char **hostname_out,
 int *made_connection_pending_out, cached_resolve_t **resolve_out));
@@ -69,4 +102,3 @@ launch_resolve,(cached_resolve_t *resolve));
 #endif /* defined(DNS_PRIVATE) */
 
 #endif /* !defined(TOR_DNS_H) */
-
diff --git a/src/feature/relay/dns_structs.h b/src/feature/relay/dns_structs.h
index e128746f81..27a791b9b3 100644
--- a/src/feature/relay/dns_structs.h
+++ b/src/feature/relay/dns_structs.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,8 @@
 #ifndef TOR_DNS_STRUCTS_H
 #define TOR_DNS_STRUCTS_H
 
+#include "ext/ht.h"
+
 /** Longest hostname we're willing to resolve. */
 #define MAX_ADDRESSLEN 256
 
@@ -99,4 +101,3 @@ typedef struct cached_resolve_t {
 } cached_resolve_t;
 
 #endif /* !defined(TOR_DNS_STRUCTS_H) */
-
diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c
index 136aee3084..2cf30262f5 100644
--- a/src/feature/relay/ext_orport.c
+++ b/src/feature/relay/ext_orport.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,7 @@
 #include "core/or/or.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn)
 
   conn->base_.type = CONN_TYPE_OR;
   TO_CONN(conn)->state = 0; // set the state to a neutral value
-  control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0);
   connection_tls_start_handshake(conn, 1);
 }
 
@@ -606,7 +606,7 @@ connection_ext_or_process_inbuf(or_connection_t *or_conn)
                                              command->body, command->len) < 0)
         goto err;
     } else {
-      log_notice(LD_NET,"Got Extended ORPort command we don't regognize (%u).",
+      log_notice(LD_NET,"Got Extended ORPort command we don't recognize (%u).",
                  command->cmd);
     }
 
@@ -656,6 +656,77 @@ connection_ext_or_start_auth(or_connection_t *or_conn)
   return 0;
 }
 
+/** Global map between Extended ORPort identifiers and OR
+ *  connections. */
+static digestmap_t *orconn_ext_or_id_map = NULL;
+
+/** Remove the Extended ORPort identifier of <b>conn</b> from the
+ *  global identifier list. Also, clear the identifier from the
+ *  connection itself. */
+void
+connection_or_remove_from_ext_or_id_map(or_connection_t *conn)
+{
+  or_connection_t *tmp;
+  if (!orconn_ext_or_id_map)
+    return;
+  if (!conn->ext_or_conn_id)
+    return;
+
+  tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id);
+  if (!tor_digest_is_zero(conn->ext_or_conn_id))
+    tor_assert(tmp == conn);
+
+  memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN);
+}
+
+#ifdef TOR_UNIT_TESTS
+/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such
+ * connection is found. */
+or_connection_t *
+connection_or_get_by_ext_or_id(const char *id)
+{
+  if (!orconn_ext_or_id_map)
+    return NULL;
+  return digestmap_get(orconn_ext_or_id_map, id);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** Deallocate the global Extended ORPort identifier list */
+void
+connection_or_clear_ext_or_id_map(void)
+{
+  digestmap_free(orconn_ext_or_id_map, NULL);
+  orconn_ext_or_id_map = NULL;
+}
+
+/** Creates an Extended ORPort identifier for <b>conn</b> and deposits
+ *  it into the global list of identifiers. */
+void
+connection_or_set_ext_or_identifier(or_connection_t *conn)
+{
+  char random_id[EXT_OR_CONN_ID_LEN];
+  or_connection_t *tmp;
+
+  if (!orconn_ext_or_id_map)
+    orconn_ext_or_id_map = digestmap_new();
+
+  /* Remove any previous identifiers: */
+  if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id))
+      connection_or_remove_from_ext_or_id_map(conn);
+
+  do {
+    crypto_rand(random_id, sizeof(random_id));
+  } while (digestmap_get(orconn_ext_or_id_map, random_id));
+
+  if (!conn->ext_or_conn_id)
+    conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN);
+
+  memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN);
+
+  tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn);
+  tor_assert(!tmp);
+}
+
 /** Free any leftover allocated memory of the ext_orport.c subsystem. */
 void
 ext_orport_free_all(void)
diff --git a/src/feature/relay/ext_orport.h b/src/feature/relay/ext_orport.h
index 7313ebd03d..416c358397 100644
--- a/src/feature/relay/ext_orport.h
+++ b/src/feature/relay/ext_orport.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file ext_orport.h
+ * @brief Header for ext_orport.c
+ **/
+
 #ifndef EXT_ORPORT_H
 #define EXT_ORPORT_H
 
@@ -26,26 +31,56 @@
 #define EXT_OR_CONN_STATE_FLUSHING 5
 #define EXT_OR_CONN_STATE_MAX_ 5
 
-int connection_ext_or_start_auth(or_connection_t *or_conn);
-
-ext_or_cmd_t *ext_or_cmd_new(uint16_t len);
+#ifdef HAVE_MODULE_RELAY
 
-#define ext_or_cmd_free(cmd)                            \
-  FREE_AND_NULL(ext_or_cmd_t, ext_or_cmd_free_, (cmd))
+int connection_ext_or_start_auth(or_connection_t *or_conn);
 
-void ext_or_cmd_free_(ext_or_cmd_t *cmd);
 void connection_or_set_ext_or_identifier(or_connection_t *conn);
 void connection_or_remove_from_ext_or_id_map(or_connection_t *conn);
 void connection_or_clear_ext_or_id_map(void);
-or_connection_t *connection_or_get_by_ext_or_id(const char *id);
-
 int connection_ext_or_finished_flushing(or_connection_t *conn);
 int connection_ext_or_process_inbuf(or_connection_t *or_conn);
+char *get_ext_or_auth_cookie_file_name(void);
 
+/* (No stub needed for these: they are only called within feature/relay.) */
 int init_ext_or_cookie_authentication(int is_enabled);
-char *get_ext_or_auth_cookie_file_name(void);
 void ext_orport_free_all(void);
 
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+static inline int
+connection_ext_or_start_auth(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+static inline int
+connection_ext_or_finished_flushing(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+static inline int
+connection_ext_or_process_inbuf(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+#define connection_or_set_ext_or_identifier(conn) \
+  ((void)(conn))
+#define connection_or_remove_from_ext_or_id_map(conn) \
+  ((void)(conn))
+#define connection_or_clear_ext_or_id_map() \
+  STMT_NIL
+
+#define get_ext_or_auth_cookie_file_name() \
+  (NULL)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #ifdef EXT_ORPORT_PRIVATE
 STATIC int connection_write_ext_or_command(connection_t *conn,
                                            uint16_t command,
@@ -55,9 +90,11 @@ STATIC int handle_client_auth_nonce(const char *client_nonce,
                          size_t client_nonce_len,
                          char **client_hash_out,
                          char **reply_out, size_t *reply_len_out);
+
 #ifdef TOR_UNIT_TESTS
 extern uint8_t *ext_or_auth_cookie;
 extern int ext_or_auth_cookie_is_set;
+or_connection_t *connection_or_get_by_ext_or_id(const char *id);
 #endif
 #endif /* defined(EXT_ORPORT_PRIVATE) */
 
diff --git a/src/feature/relay/feature_relay.md b/src/feature/relay/feature_relay.md
new file mode 100644
index 0000000000..a7f0c2153a
--- /dev/null
+++ b/src/feature/relay/feature_relay.md
@@ -0,0 +1,4 @@
+@dir /feature/relay
+@brief feature/relay: Relay-specific code
+
+(There is also a bunch of relay-specific code in other modules.)
diff --git a/src/feature/relay/include.am b/src/feature/relay/include.am
new file mode 100644
index 0000000000..84bb1ff35e
--- /dev/null
+++ b/src/feature/relay/include.am
@@ -0,0 +1,46 @@
+
+# Legacy shared relay code: migrate to the relay module over time
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/relay/onion_queue.c		\
+	src/feature/relay/relay_find_addr.c	\
+	src/feature/relay/router.c
+
+# The Relay module.
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+MODULE_RELAY_SOURCES = 						\
+	src/feature/relay/circuitbuild_relay.c			\
+	src/feature/relay/dns.c					\
+	src/feature/relay/ext_orport.c		                \
+	src/feature/relay/routermode.c				\
+	src/feature/relay/relay_config.c			\
+	src/feature/relay/relay_handshake.c			\
+	src/feature/relay/relay_periodic.c			\
+	src/feature/relay/relay_sys.c				\
+	src/feature/relay/routerkeys.c				\
+	src/feature/relay/selftest.c				\
+	src/feature/relay/transport_config.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/relay/circuitbuild_relay.h		\
+	src/feature/relay/dns.h				\
+	src/feature/relay/dns_structs.h			\
+	src/feature/relay/ext_orport.h			\
+	src/feature/relay/onion_queue.h			\
+	src/feature/relay/relay_config.h		\
+	src/feature/relay/relay_handshake.h		\
+	src/feature/relay/relay_periodic.h		\
+	src/feature/relay/relay_sys.h			\
+	src/feature/relay/relay_find_addr.h		\
+	src/feature/relay/router.h			\
+	src/feature/relay/routerkeys.h			\
+	src/feature/relay/routermode.h			\
+	src/feature/relay/selftest.h			\
+	src/feature/relay/transport_config.h
+
+if BUILD_MODULE_RELAY
+LIBTOR_APP_A_SOURCES += $(MODULE_RELAY_SOURCES)
+else
+LIBTOR_APP_A_STUB_SOURCES += src/feature/relay/relay_stub.c
+endif
diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c
index 696905cf5e..3cbaa65d28 100644
--- a/src/feature/relay/onion_queue.c
+++ b/src/feature/relay/onion_queue.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,10 +49,12 @@ typedef struct onion_queue_t {
 /** 5 seconds on the onion queue til we just send back a destroy */
 #define ONIONQUEUE_WAIT_CUTOFF 5
 
+TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t);
+typedef struct onion_queue_head_t onion_queue_head_t;
+
 /** Array of queues of circuits waiting for CPU workers. An element is NULL
  * if that queue is empty.*/
-static TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t)
-              ol_list[MAX_ONION_HANDSHAKE_TYPE+1] =
+static onion_queue_head_t ol_list[MAX_ONION_HANDSHAKE_TYPE+1] =
 { TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */
   TOR_TAILQ_HEAD_INITIALIZER(ol_list[1]), /* fast */
   TOR_TAILQ_HEAD_INITIALIZER(ol_list[2]), /* ntor */
@@ -212,10 +214,12 @@ num_ntors_per_tap(void)
 #define MIN_NUM_NTORS_PER_TAP 1
 #define MAX_NUM_NTORS_PER_TAP 100000
 
-  return networkstatus_get_param(NULL, "NumNTorsPerTAP",
-                                 DEFAULT_NUM_NTORS_PER_TAP,
-                                 MIN_NUM_NTORS_PER_TAP,
-                                 MAX_NUM_NTORS_PER_TAP);
+  int result = networkstatus_get_param(NULL, "NumNTorsPerTAP",
+                                       DEFAULT_NUM_NTORS_PER_TAP,
+                                       MIN_NUM_NTORS_PER_TAP,
+                                       MAX_NUM_NTORS_PER_TAP);
+  tor_assert(result > 0);
+  return result;
 }
 
 /** Choose which onion queue we'll pull from next. If one is empty choose
diff --git a/src/feature/relay/onion_queue.h b/src/feature/relay/onion_queue.h
index 0df921e057..08379b2c00 100644
--- a/src/feature/relay/onion_queue.h
+++ b/src/feature/relay/onion_queue.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ int onion_num_pending(uint16_t handshake_type);
 void onion_pending_remove(or_circuit_t *circ);
 void clear_pending_onions(void);
 
-#endif
+#endif /* !defined(TOR_ONION_QUEUE_H) */
diff --git a/src/feature/relay/relay_config.c b/src/feature/relay/relay_config.c
new file mode 100644
index 0000000000..fac6a2f577
--- /dev/null
+++ b/src/feature/relay/relay_config.c
@@ -0,0 +1,1444 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_config.c
+ * @brief Code to interpret the user's configuration of Tor's relay module.
+ **/
+
+#include "orconfig.h"
+#define RELAY_CONFIG_PRIVATE
+#include "feature/relay/relay_config.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confmgt.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/geoip/geoip.h"
+#include "lib/meminfo/meminfo.h"
+#include "lib/osinfo/uname.h"
+#include "lib/process/setuid.h"
+
+/* Required for dirinfo_type_t in or_options_t */
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/mainloop/connection.h"
+#include "core/mainloop/cpuworker.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/connection_or.h"
+#include "core/or/port_cfg_st.h"
+
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/authmode.h"
+
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+
+/** Contents of most recently read DirPortFrontPage file. */
+static char *global_dirfrontpagecontents = NULL;
+
+/* Copied from config.c, we will refactor later in 29211. */
+#define REJECT(arg) \
+  STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
+#if defined(__GNUC__) && __GNUC__ <= 3
+#define COMPLAIN(args...) \
+  STMT_BEGIN log_warn(LD_CONFIG, args); STMT_END
+#else
+#define COMPLAIN(args, ...)                                     \
+  STMT_BEGIN log_warn(LD_CONFIG, args, ##__VA_ARGS__); STMT_END
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
+
+/* Used in the various options_transition_affects* functions. */
+#define YES_IF_CHANGED_BOOL(opt) \
+  if (!CFG_EQ_BOOL(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_INT(opt) \
+  if (!CFG_EQ_INT(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_STRING(opt) \
+  if (!CFG_EQ_STRING(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_LINELIST(opt) \
+  if (!CFG_EQ_LINELIST(old_options, new_options, opt)) return 1;
+
+/** Return the contents of our frontpage string, or NULL if not configured. */
+MOCK_IMPL(const char*,
+relay_get_dirportfrontpage, (void))
+{
+  return global_dirfrontpagecontents;
+}
+
+/** Release all memory and resources held by global relay configuration
+ * structures.
+ */
+void
+relay_config_free_all(void)
+{
+  tor_free(global_dirfrontpagecontents);
+}
+
+/** Return the bandwidthrate that we are going to report to the authorities
+ * based on the config options. */
+uint32_t
+relay_get_effective_bwrate(const or_options_t *options)
+{
+  uint64_t bw = options->BandwidthRate;
+  if (bw > options->MaxAdvertisedBandwidth)
+    bw = options->MaxAdvertisedBandwidth;
+  if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate)
+    bw = options->RelayBandwidthRate;
+  /* config_ensure_bandwidth_cap() makes sure that this cast can't overflow. */
+  return (uint32_t)bw;
+}
+
+/** Return the bandwidthburst that we are going to report to the authorities
+ * based on the config options. */
+uint32_t
+relay_get_effective_bwburst(const or_options_t *options)
+{
+  uint64_t bw = options->BandwidthBurst;
+  if (options->RelayBandwidthBurst > 0 && bw > options->RelayBandwidthBurst)
+    bw = options->RelayBandwidthBurst;
+  /* config_ensure_bandwidth_cap() makes sure that this cast can't overflow. */
+  return (uint32_t)bw;
+}
+
+/** Warn for every Extended ORPort port in <b>ports</b> that is on a
+ *  publicly routable address. */
+void
+port_warn_nonlocal_ext_orports(const smartlist_t *ports, const char *portname)
+{
+  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
+    if (port->type != CONN_TYPE_EXT_OR_LISTENER)
+      continue;
+    if (port->is_unix_addr)
+      continue;
+    /* XXX maybe warn even if address is RFC1918? */
+    if (!tor_addr_is_internal(&port->addr, 1)) {
+      log_warn(LD_CONFIG, "You specified a public address '%s' for %sPort. "
+               "This is not advised; this address is supposed to only be "
+               "exposed on localhost so that your pluggable transport "
+               "proxies can connect to it.",
+               fmt_addrport(&port->addr, port->port), portname);
+    }
+  } SMARTLIST_FOREACH_END(port);
+}
+
+/** Given a list of <b>port_cfg_t</b> in <b>ports</b>, check them for internal
+ * consistency and warn as appropriate.  On Unix-based OSes, set
+ * *<b>n_low_ports_out</b> to the number of sub-1024 ports we will be
+ * binding, and warn if we may be unable to re-bind after hibernation. */
+static int
+check_server_ports(const smartlist_t *ports,
+                   const or_options_t *options,
+                   int *n_low_ports_out)
+{
+  if (BUG(!ports))
+    return -1;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!n_low_ports_out))
+    return -1;
+
+  int n_orport_advertised = 0;
+  int n_orport_advertised_ipv4 = 0;
+  int n_orport_listeners = 0;
+  int n_dirport_advertised = 0;
+  int n_dirport_listeners = 0;
+  int n_low_port = 0;
+  int r = 0;
+
+  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
+    if (port->type == CONN_TYPE_DIR_LISTENER) {
+      if (! port->server_cfg.no_advertise)
+        ++n_dirport_advertised;
+      if (! port->server_cfg.no_listen)
+        ++n_dirport_listeners;
+    } else if (port->type == CONN_TYPE_OR_LISTENER) {
+      if (! port->server_cfg.no_advertise) {
+        ++n_orport_advertised;
+        if (port_binds_ipv4(port))
+          ++n_orport_advertised_ipv4;
+      }
+      if (! port->server_cfg.no_listen)
+        ++n_orport_listeners;
+    } else {
+      continue;
+    }
+#ifndef _WIN32
+    if (!port->server_cfg.no_listen && port->port < 1024)
+      ++n_low_port;
+#endif
+  } SMARTLIST_FOREACH_END(port);
+
+  if (n_orport_advertised && !n_orport_listeners) {
+    log_warn(LD_CONFIG, "We are advertising an ORPort, but not actually "
+             "listening on one.");
+    r = -1;
+  }
+  if (n_orport_listeners && !n_orport_advertised) {
+    log_warn(LD_CONFIG, "We are listening on an ORPort, but not advertising "
+             "any ORPorts. This will keep us from building a %s "
+             "descriptor, and make us impossible to use.",
+             options->BridgeRelay ? "bridge" : "router");
+    r = -1;
+  }
+  if (n_dirport_advertised && !n_dirport_listeners) {
+    log_warn(LD_CONFIG, "We are advertising a DirPort, but not actually "
+             "listening on one.");
+    r = -1;
+  }
+  if (n_dirport_advertised > 1) {
+    log_warn(LD_CONFIG, "Can't advertise more than one DirPort.");
+    r = -1;
+  }
+  if (n_orport_advertised && !n_orport_advertised_ipv4 &&
+      !options->BridgeRelay) {
+    log_warn(LD_CONFIG, "Configured public relay to listen only on an IPv6 "
+             "address. Tor needs to listen on an IPv4 address too.");
+    r = -1;
+  }
+
+  if (n_low_port && options->AccountingMax &&
+      (!have_capability_support() || options->KeepBindCapabilities == 0)) {
+    const char *extra = "";
+    if (options->KeepBindCapabilities == 0 && have_capability_support())
+      extra = ", and you have disabled KeepBindCapabilities.";
+    log_warn(LD_CONFIG,
+          "You have set AccountingMax to use hibernation. You have also "
+          "chosen a low DirPort or OrPort%s."
+          "This combination can make Tor stop "
+          "working when it tries to re-attach the port after a period of "
+          "hibernation. Please choose a different port or turn off "
+          "hibernation unless you know this combination will work on your "
+          "platform.", extra);
+  }
+
+  if (n_low_ports_out)
+    *n_low_ports_out = n_low_port;
+
+  return r;
+}
+
+/** Parse all relay ports from <b>options</b>. On success, add parsed ports to
+ * <b>ports</b>, and return 0.  On failure, set *<b>msg</b> to a newly
+ * allocated string describing the problem, and return -1.
+ **/
+int
+port_parse_ports_relay(or_options_t *options,
+                  char **msg,
+                  smartlist_t *ports_out,
+                  int *have_low_ports_out)
+{
+  int retval = -1;
+  smartlist_t *ports = smartlist_new();
+  int n_low_ports = 0;
+
+  if (BUG(!options))
+    goto err;
+
+  if (BUG(!msg))
+    goto err;
+
+  if (BUG(!ports_out))
+    goto err;
+
+  if (BUG(!have_low_ports_out))
+    goto err;
+
+  if (options->ClientOnly) {
+    retval = 0;
+    goto err;
+  }
+
+  if (port_parse_config(ports,
+                        options->ORPort_lines,
+                        "OR", CONN_TYPE_OR_LISTENER,
+                        "0.0.0.0", 0,
+                        CL_PORT_SERVER_OPTIONS) < 0) {
+    *msg = tor_strdup("Invalid ORPort configuration");
+    goto err;
+  }
+  if (port_parse_config(ports,
+                        options->ExtORPort_lines,
+                        "ExtOR", CONN_TYPE_EXT_OR_LISTENER,
+                        "127.0.0.1", 0,
+                        CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) {
+    *msg = tor_strdup("Invalid ExtORPort configuration");
+    goto err;
+  }
+  if (port_parse_config(ports,
+                        options->DirPort_lines,
+                        "Dir", CONN_TYPE_DIR_LISTENER,
+                        "0.0.0.0", 0,
+                        CL_PORT_SERVER_OPTIONS) < 0) {
+    *msg = tor_strdup("Invalid DirPort configuration");
+    goto err;
+  }
+
+  if (check_server_ports(ports, options, &n_low_ports) < 0) {
+    *msg = tor_strdup("Misconfigured server ports");
+    goto err;
+  }
+
+  smartlist_add_all(ports_out, ports);
+  smartlist_free(ports);
+  ports = NULL;
+  retval = 0;
+
+ err:
+  if (*have_low_ports_out < 0)
+    *have_low_ports_out = (n_low_ports > 0);
+  if (ports) {
+    SMARTLIST_FOREACH(ports, port_cfg_t *, p, port_cfg_free(p));
+    smartlist_free(ports);
+  }
+  return retval;
+}
+
+/** Update the relay *Port_set values in <b>options</b> from <b>ports</b>. */
+void
+port_update_port_set_relay(or_options_t *options,
+                      const smartlist_t *ports)
+{
+  if (BUG(!options))
+    return;
+
+  if (BUG(!ports))
+    return;
+
+  if (options->ClientOnly)
+    return;
+
+  /* Update the relay *Port_set options.  The !! here is to force a boolean
+   * out of an integer. */
+  options->ORPort_set =
+    !! port_count_real_listeners(ports, CONN_TYPE_OR_LISTENER, 0);
+  options->DirPort_set =
+    !! port_count_real_listeners(ports, CONN_TYPE_DIR_LISTENER, 0);
+  options->ExtORPort_set =
+    !! port_count_real_listeners(ports, CONN_TYPE_EXT_OR_LISTENER, 0);
+}
+
+/**
+ * Legacy validation function, which checks that the current OS is usable in
+ * relay mode, if options is set to a relay mode.
+ *
+ * Warns about OSes with potential issues. Does not set *<b>msg</b>.
+ * Always returns 0.
+ */
+int
+options_validate_relay_os(const or_options_t *old_options,
+                          or_options_t *options,
+                          char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!server_mode(options))
+    return 0;
+
+  const char *uname = get_uname();
+
+  if (!strcmpstart(uname, "Windows 95") ||
+      !strcmpstart(uname, "Windows 98") ||
+      !strcmpstart(uname, "Windows Me")) {
+    log_warn(LD_CONFIG, "Tor is running as a server, but you are "
+        "running %s; this probably won't work. See "
+        "https://www.torproject.org/docs/faq.html#BestOSForRelay "
+        "for details.", uname);
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay info options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_info(const or_options_t *old_options,
+                            or_options_t *options,
+                            char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (options->Nickname == NULL) {
+    if (server_mode(options)) {
+      options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
+    }
+  } else {
+    if (!is_legal_nickname(options->Nickname)) {
+      tor_asprintf(msg,
+          "Nickname '%s', nicknames must be between 1 and 19 characters "
+          "inclusive, and must contain only the characters [a-zA-Z0-9].",
+          options->Nickname);
+      return -1;
+    }
+  }
+
+  if (server_mode(options) && !options->ContactInfo) {
+    log_warn(LD_CONFIG,
+             "Your ContactInfo config option is not set. Please strongly "
+             "consider setting it, so we can contact you if your relay is "
+             "misconfigured, end-of-life, or something else goes wrong. "
+             "It is also possible that your relay might get rejected from "
+             "the network due to a missing valid contact address.");
+  }
+
+  const char *ContactInfo = options->ContactInfo;
+  if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo)))
+    REJECT("ContactInfo config option must be UTF-8.");
+
+  return 0;
+}
+
+/** Parse an authority type from <b>options</b>-\>PublishServerDescriptor
+ * and write it to <b>options</b>-\>PublishServerDescriptor_. Treat "1"
+ * as "v3" unless BridgeRelay is 1, in which case treat it as "bridge".
+ * Treat "0" as "".
+ * Return 0 on success or -1 if not a recognized authority type (in which
+ * case the value of PublishServerDescriptor_ is undefined). */
+static int
+compute_publishserverdescriptor(or_options_t *options)
+{
+  smartlist_t *list = options->PublishServerDescriptor;
+  dirinfo_type_t *auth = &options->PublishServerDescriptor_;
+  *auth = NO_DIRINFO;
+  if (!list) /* empty list, answer is none */
+    return 0;
+  SMARTLIST_FOREACH_BEGIN(list, const char *, string) {
+    if (!strcasecmp(string, "v1"))
+      log_warn(LD_CONFIG, "PublishServerDescriptor v1 has no effect, because "
+                          "there are no v1 directory authorities anymore.");
+    else if (!strcmp(string, "1"))
+      if (options->BridgeRelay)
+        *auth |= BRIDGE_DIRINFO;
+      else
+        *auth |= V3_DIRINFO;
+    else if (!strcasecmp(string, "v2"))
+      log_warn(LD_CONFIG, "PublishServerDescriptor v2 has no effect, because "
+                          "there are no v2 directory authorities anymore.");
+    else if (!strcasecmp(string, "v3"))
+      *auth |= V3_DIRINFO;
+    else if (!strcasecmp(string, "bridge"))
+      *auth |= BRIDGE_DIRINFO;
+    else if (!strcasecmp(string, "hidserv"))
+      log_warn(LD_CONFIG,
+               "PublishServerDescriptor hidserv is invalid. See "
+               "PublishHidServDescriptors.");
+    else if (!strcasecmp(string, "") || !strcmp(string, "0"))
+      /* no authority */;
+    else
+      return -1;
+  } SMARTLIST_FOREACH_END(string);
+  return 0;
+}
+
+/**
+ * Validate the configured bridge distribution method from a BridgeDistribution
+ * config line.
+ *
+ * The input <b>bd</b>, is a string taken from the BridgeDistribution config
+ * line (if present).  If the option wasn't set, return 0 immediately.  The
+ * BridgeDistribution option is then validated.  Currently valid, recognised
+ * options are:
+ *
+ * - "none"
+ * - "any"
+ * - "https"
+ * - "email"
+ * - "moat"
+ *
+ * If the option string is unrecognised, a warning will be logged and 0 is
+ * returned.  If the option string contains an invalid character, -1 is
+ * returned.
+ **/
+STATIC int
+check_bridge_distribution_setting(const char *bd)
+{
+  if (bd == NULL)
+    return 0;
+
+  const char *RECOGNIZED[] = {
+    "none", "any", "https", "email", "moat"
+  };
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(RECOGNIZED); ++i) {
+    if (!strcasecmp(bd, RECOGNIZED[i]))
+      return 0;
+  }
+
+  const char *cp = bd;
+  //  Method = (KeywordChar | "_") +
+  while (TOR_ISALNUM(*cp) || *cp == '-' || *cp == '_')
+    ++cp;
+
+  if (*cp == 0) {
+    log_warn(LD_CONFIG, "Unrecognized BridgeDistribution value %s. I'll "
+           "assume you know what you are doing...", escaped(bd));
+    return 0; // we reached the end of the string; all is well
+  } else {
+    return -1; // we found a bad character in the string.
+  }
+}
+
+/**
+ * Legacy validation/normalization function for the bridge relay options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_publish_server(const or_options_t *old_options,
+                                or_options_t *options,
+                                char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (compute_publishserverdescriptor(options) < 0) {
+    tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor");
+    return -1;
+  }
+
+  if ((options->BridgeRelay
+        || options->PublishServerDescriptor_ & BRIDGE_DIRINFO)
+      && (options->PublishServerDescriptor_ & V3_DIRINFO)) {
+    REJECT("Bridges are not supposed to publish router descriptors to the "
+           "directory authorities. Please correct your "
+           "PublishServerDescriptor line.");
+  }
+
+  if (options->BridgeDistribution) {
+    if (!options->BridgeRelay) {
+      REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
+    }
+    if (check_bridge_distribution_setting(options->BridgeDistribution) < 0) {
+      REJECT("Invalid BridgeDistribution value.");
+    }
+  }
+
+  if (options->PublishServerDescriptor)
+    SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, {
+      if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0"))
+        if (smartlist_len(options->PublishServerDescriptor) > 1) {
+          COMPLAIN("You have passed a list of multiple arguments to the "
+                   "PublishServerDescriptor option that includes 0 or 1. "
+                   "0 or 1 should only be used as the sole argument. "
+                   "This configuration will be rejected in a future release.");
+          break;
+        }
+    });
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay padding options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_padding(const or_options_t *old_options,
+                               or_options_t *options,
+                               char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!server_mode(options))
+    return 0;
+
+  if (options->ConnectionPadding != -1) {
+    REJECT("Relays must use 'auto' for the ConnectionPadding setting.");
+  }
+
+  if (options->ReducedConnectionPadding != 0) {
+    REJECT("Relays cannot set ReducedConnectionPadding. ");
+  }
+
+  if (options->CircuitPadding == 0) {
+    REJECT("Relays cannot set CircuitPadding to 0. ");
+  }
+
+  if (options->ReducedCircuitPadding == 1) {
+    REJECT("Relays cannot set ReducedCircuitPadding. ");
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay bandwidth options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_bandwidth(const or_options_t *old_options,
+                                 or_options_t *options,
+                                 char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  /* 31851: the tests expect us to validate bandwidths, even when we are not
+  * in relay mode. */
+  if (config_ensure_bandwidth_cap(&options->MaxAdvertisedBandwidth,
+                           "MaxAdvertisedBandwidth", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->RelayBandwidthRate,
+                           "RelayBandwidthRate", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->RelayBandwidthBurst,
+                           "RelayBandwidthBurst", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->PerConnBWRate,
+                           "PerConnBWRate", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->PerConnBWBurst,
+                           "PerConnBWBurst", msg) < 0)
+    return -1;
+
+  if (options->RelayBandwidthRate && !options->RelayBandwidthBurst)
+    options->RelayBandwidthBurst = options->RelayBandwidthRate;
+  if (options->RelayBandwidthBurst && !options->RelayBandwidthRate)
+    options->RelayBandwidthRate = options->RelayBandwidthBurst;
+
+  if (server_mode(options)) {
+    const unsigned required_min_bw =
+      public_server_mode(options) ?
+       RELAY_REQUIRED_MIN_BANDWIDTH : BRIDGE_REQUIRED_MIN_BANDWIDTH;
+    const char * const optbridge =
+      public_server_mode(options) ? "" : "bridge ";
+    if (options->BandwidthRate < required_min_bw) {
+      tor_asprintf(msg,
+                       "BandwidthRate is set to %d bytes/second. "
+                       "For %sservers, it must be at least %u.",
+                       (int)options->BandwidthRate, optbridge,
+                       required_min_bw);
+      return -1;
+    } else if (options->MaxAdvertisedBandwidth <
+               required_min_bw/2) {
+      tor_asprintf(msg,
+                       "MaxAdvertisedBandwidth is set to %d bytes/second. "
+                       "For %sservers, it must be at least %u.",
+                       (int)options->MaxAdvertisedBandwidth, optbridge,
+                       required_min_bw/2);
+      return -1;
+    }
+    if (options->RelayBandwidthRate &&
+      options->RelayBandwidthRate < required_min_bw) {
+      tor_asprintf(msg,
+                       "RelayBandwidthRate is set to %d bytes/second. "
+                       "For %sservers, it must be at least %u.",
+                       (int)options->RelayBandwidthRate, optbridge,
+                       required_min_bw);
+      return -1;
+    }
+  }
+
+  /* 31851: the tests expect us to validate bandwidths, even when we are not
+   * in relay mode. */
+  if (options->RelayBandwidthRate > options->RelayBandwidthBurst)
+    REJECT("RelayBandwidthBurst must be at least equal "
+           "to RelayBandwidthRate.");
+
+  /* if they set relaybandwidth* really high but left bandwidth*
+   * at the default, raise the defaults. */
+  if (options->RelayBandwidthRate > options->BandwidthRate)
+    options->BandwidthRate = options->RelayBandwidthRate;
+  if (options->RelayBandwidthBurst > options->BandwidthBurst)
+    options->BandwidthBurst = options->RelayBandwidthBurst;
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay bandwidth accounting
+ * options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_accounting(const or_options_t *old_options,
+                                  or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  /* 31851: the tests expect us to validate accounting, even when we are not
+   * in relay mode. */
+  if (accounting_parse_options(options, 1)<0)
+    REJECT("Failed to parse accounting options. See logs for details.");
+
+  if (options->AccountingMax) {
+    if (options->RendConfigLines && server_mode(options)) {
+      log_warn(LD_CONFIG, "Using accounting with a hidden service and an "
+               "ORPort is risky: your hidden service(s) and your public "
+               "address will all turn off at the same time, which may alert "
+               "observers that they are being run by the same party.");
+    } else if (config_count_key(options->RendConfigLines,
+                                "HiddenServiceDir") > 1) {
+      log_warn(LD_CONFIG, "Using accounting with multiple hidden services is "
+               "risky: they will all turn off at the same time, which may "
+               "alert observers that they are being run by the same party.");
+    }
+  }
+
+  options->AccountingRule = ACCT_MAX;
+  if (options->AccountingRule_option) {
+    if (!strcmp(options->AccountingRule_option, "sum"))
+      options->AccountingRule = ACCT_SUM;
+    else if (!strcmp(options->AccountingRule_option, "max"))
+      options->AccountingRule = ACCT_MAX;
+    else if (!strcmp(options->AccountingRule_option, "in"))
+      options->AccountingRule = ACCT_IN;
+    else if (!strcmp(options->AccountingRule_option, "out"))
+      options->AccountingRule = ACCT_OUT;
+    else
+      REJECT("AccountingRule must be 'sum', 'max', 'in', or 'out'");
+  }
+
+  return 0;
+}
+
+/** Verify whether lst is a list of strings containing valid-looking
+ * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$'
+ * to any nickname or fingerprint that needs it. Also splits comma-separated
+ * list elements into multiple elements. Return 0 on success.
+ * Warn and return -1 on failure.
+ */
+static int
+normalize_nickname_list(config_line_t **normalized_out,
+                        const config_line_t *lst, const char *name,
+                        char **msg)
+{
+  if (!lst)
+    return 0;
+
+  config_line_t *new_nicknames = NULL;
+  config_line_t **new_nicknames_next = &new_nicknames;
+
+  const config_line_t *cl;
+  for (cl = lst; cl; cl = cl->next) {
+    const char *line = cl->value;
+    if (!line)
+      continue;
+
+    int valid_line = 1;
+    smartlist_t *sl = smartlist_new();
+    smartlist_split_string(sl, line, ",",
+      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+    SMARTLIST_FOREACH_BEGIN(sl, char *, s)
+    {
+      char *normalized = NULL;
+      if (!is_legal_nickname_or_hexdigest(s)) {
+        // check if first char is dollar
+        if (s[0] != '$') {
+          // Try again but with a dollar symbol prepended
+          char *prepended;
+          tor_asprintf(&prepended, "$%s", s);
+
+          if (is_legal_nickname_or_hexdigest(prepended)) {
+            // The nickname is valid when it's prepended, set it as the
+            // normalized version
+            normalized = prepended;
+          } else {
+            // Still not valid, free and fallback to error message
+            tor_free(prepended);
+          }
+        }
+
+        if (!normalized) {
+          tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
+          valid_line = 0;
+          break;
+        }
+      } else {
+        normalized = tor_strdup(s);
+      }
+
+      config_line_t *next = tor_malloc_zero(sizeof(*next));
+      next->key = tor_strdup(cl->key);
+      next->value = normalized;
+      next->next = NULL;
+
+      *new_nicknames_next = next;
+      new_nicknames_next = &next->next;
+    } SMARTLIST_FOREACH_END(s);
+
+    SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
+    smartlist_free(sl);
+
+    if (!valid_line) {
+      config_free_lines(new_nicknames);
+      return -1;
+    }
+  }
+
+  *normalized_out = new_nicknames;
+
+  return 0;
+}
+
+#define ONE_MEGABYTE (UINT64_C(1) << 20)
+
+/* If we have less than 300 MB suggest disabling dircache */
+#define DIRCACHE_MIN_MEM_MB 300
+#define DIRCACHE_MIN_MEM_BYTES (DIRCACHE_MIN_MEM_MB*ONE_MEGABYTE)
+#define STRINGIFY(val) #val
+
+/** Create a warning message for emitting if we are a dircache but may not have
+ * enough system memory, or if we are not a dircache but probably should be.
+ * Return -1 when a message is returned in *msg*, else return 0. */
+STATIC int
+have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
+                             char **msg)
+{
+  *msg = NULL;
+  /* XXX We should possibly be looking at MaxMemInQueues here
+   * unconditionally.  Or we should believe total_mem unconditionally. */
+  if (total_mem == 0) {
+    if (get_total_system_memory(&total_mem) < 0) {
+      total_mem = options->MaxMemInQueues >= SIZE_MAX ?
+        SIZE_MAX : (size_t)options->MaxMemInQueues;
+    }
+  }
+  if (options->DirCache) {
+    if (total_mem < DIRCACHE_MIN_MEM_BYTES) {
+      if (options->BridgeRelay) {
+        tor_asprintf(msg, "Running a Bridge with less than %d MB of memory "
+                       "is not recommended.", DIRCACHE_MIN_MEM_MB);
+      } else {
+        tor_asprintf(msg, "Being a directory cache (default) with less than "
+                       "%d MB of memory is not recommended and may consume "
+                       "most of the available resources. Consider disabling "
+                       "this functionality by setting the DirCache option "
+                       "to 0.", DIRCACHE_MIN_MEM_MB);
+      }
+    }
+  } else {
+    if (total_mem >= DIRCACHE_MIN_MEM_BYTES) {
+      *msg = tor_strdup("DirCache is disabled and we are configured as a "
+               "relay. We will not become a Guard.");
+    }
+  }
+  return *msg == NULL ? 0 : -1;
+}
+#undef STRINGIFY
+
+/**
+ * Legacy validation/normalization function for the relay mode options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_mode(const or_options_t *old_options,
+                            or_options_t *options,
+                            char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (server_mode(options) && options->RendConfigLines)
+    log_warn(LD_CONFIG,
+        "Tor is currently configured as a relay and a hidden service. "
+        "That's not very secure: you should probably run your hidden service "
+        "in a separate Tor process, at least -- see "
+        "https://trac.torproject.org/8742");
+
+  if (options->BridgeRelay && options->DirPort_set) {
+    log_warn(LD_CONFIG, "Can't set a DirPort on a bridge relay; disabling "
+             "DirPort");
+    config_free_lines(options->DirPort_lines);
+    options->DirPort_lines = NULL;
+    options->DirPort_set = 0;
+  }
+
+  if (options->DirPort_set && !options->DirCache) {
+    REJECT("DirPort configured but DirCache disabled. DirPort requires "
+           "DirCache.");
+  }
+
+  if (options->BridgeRelay && !options->DirCache) {
+    REJECT("We're a bridge but DirCache is disabled. BridgeRelay requires "
+           "DirCache.");
+  }
+
+  if (options->BridgeRelay == 1 && ! options->ORPort_set)
+    REJECT("BridgeRelay is 1, ORPort is not set. This is an invalid "
+           "combination.");
+
+  if (server_mode(options)) {
+    char *dircache_msg = NULL;
+    if (have_enough_mem_for_dircache(options, 0, &dircache_msg)) {
+      log_warn(LD_CONFIG, "%s", dircache_msg);
+      tor_free(dircache_msg);
+    }
+  }
+
+  if (options->MyFamily_lines && options->BridgeRelay) {
+    log_warn(LD_CONFIG, "Listing a family for a bridge relay is not "
+             "supported: it can reveal bridge fingerprints to censors. "
+             "You should also make sure you aren't listing this bridge's "
+             "fingerprint in any other MyFamily.");
+  }
+  if (options->MyFamily_lines && !options->ContactInfo) {
+    log_warn(LD_CONFIG, "MyFamily is set but ContactInfo is not configured. "
+             "ContactInfo should always be set when MyFamily option is too.");
+  }
+  if (normalize_nickname_list(&options->MyFamily,
+                              options->MyFamily_lines, "MyFamily", msg))
+    return -1;
+
+  if (options->ConstrainedSockets) {
+    if (options->DirPort_set) {
+      /* Providing cached directory entries while system TCP buffers are scarce
+       * will exacerbate the socket errors.  Suggest that this be disabled. */
+      COMPLAIN("You have requested constrained socket buffers while also "
+               "serving directory entries via DirPort.  It is strongly "
+               "suggested that you disable serving directory requests when "
+               "system TCP buffer resources are scarce.");
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay testing options
+ * in options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_testing(const or_options_t *old_options,
+                               or_options_t *options,
+                               char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (options->SigningKeyLifetime < options->TestingSigningKeySlop*2)
+    REJECT("SigningKeyLifetime is too short.");
+  if (options->TestingLinkCertLifetime < options->TestingAuthKeySlop*2)
+    REJECT("LinkCertLifetime is too short.");
+  if (options->TestingAuthKeyLifetime < options->TestingLinkKeySlop*2)
+    REJECT("TestingAuthKeyLifetime is too short.");
+
+  return 0;
+}
+
+/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
+ * will require us to rotate the CPU and DNS workers; else return 0. */
+static int
+options_transition_affects_workers(const or_options_t *old_options,
+                                   const or_options_t *new_options)
+{
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_INT(NumCPUs);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_BOOL(ServerDNSSearchDomains);
+  YES_IF_CHANGED_BOOL(SafeLogging_);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(LogMessageDomains);
+  YES_IF_CHANGED_LINELIST(Logs);
+
+  if (server_mode(old_options) != server_mode(new_options) ||
+      public_server_mode(old_options) != public_server_mode(new_options) ||
+      dir_server_mode(old_options) != dir_server_mode(new_options))
+    return 1;
+
+  /* Nothing that changed matters. */
+  return 0;
+}
+
+/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
+ * will require us to generate a new descriptor; else return 0. */
+static int
+options_transition_affects_descriptor(const or_options_t *old_options,
+                                      const or_options_t *new_options)
+{
+  /* XXX We can be smarter here. If your DirPort isn't being
+   * published and you just turned it off, no need to republish. Etc. */
+
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_STRING(Nickname);
+  YES_IF_CHANGED_STRING(Address);
+  YES_IF_CHANGED_LINELIST(ExitPolicy);
+  YES_IF_CHANGED_BOOL(ExitRelay);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectPrivate);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectLocalInterfaces);
+  YES_IF_CHANGED_BOOL(IPv6Exit);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(DisableNetwork);
+  YES_IF_CHANGED_BOOL(PublishServerDescriptor_);
+  YES_IF_CHANGED_STRING(ContactInfo);
+  YES_IF_CHANGED_STRING(BridgeDistribution);
+  YES_IF_CHANGED_LINELIST(MyFamily);
+  YES_IF_CHANGED_STRING(AccountingStart);
+  YES_IF_CHANGED_INT(AccountingMax);
+  YES_IF_CHANGED_INT(AccountingRule);
+  YES_IF_CHANGED_BOOL(DirCache);
+  YES_IF_CHANGED_BOOL(AssumeReachable);
+
+  if (relay_get_effective_bwrate(old_options) !=
+        relay_get_effective_bwrate(new_options) ||
+      relay_get_effective_bwburst(old_options) !=
+        relay_get_effective_bwburst(new_options) ||
+      public_server_mode(old_options) != public_server_mode(new_options))
+    return 1;
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay actions based on it. All of
+ * the things we do should survive being done repeatedly.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  const int transition_affects_workers =
+    old_options && options_transition_affects_workers(old_options, options);
+
+  /* We want to reinit keys as needed before we do much of anything else:
+     keys are important, and other things can depend on them. */
+  if (transition_affects_workers ||
+      (authdir_mode_v3(options) && (!old_options ||
+                                    !authdir_mode_v3(old_options)))) {
+    if (init_keys() < 0) {
+      log_warn(LD_BUG,"Error initializing keys; exiting");
+      return -1;
+    }
+  }
+
+  if (server_mode(options)) {
+    static int cdm_initialized = 0;
+    if (cdm_initialized == 0) {
+      cdm_initialized = 1;
+      consdiffmgr_configure(NULL);
+      consdiffmgr_validate();
+    }
+  }
+
+  /* Check for transitions that need action. */
+  if (old_options) {
+    if (transition_affects_workers) {
+      log_info(LD_GENERAL,
+               "Worker-related options changed. Rotating workers.");
+      const int server_mode_turned_on =
+        server_mode(options) && !server_mode(old_options);
+      const int dir_server_mode_turned_on =
+        dir_server_mode(options) && !dir_server_mode(old_options);
+
+      if (server_mode_turned_on || dir_server_mode_turned_on) {
+        cpu_init();
+      }
+
+      if (server_mode_turned_on) {
+        ip_address_changed(0);
+        if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL)))
+          inform_testing_reachability();
+      }
+      cpuworkers_rotate_keyinfo();
+    }
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay accounting actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_accounting(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  const or_options_t *options = get_options();
+
+  /* Set up accounting */
+  if (accounting_parse_options(options, 0)<0) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG,"Error in previously validated accounting options");
+    return -1;
+    // LCOV_EXCL_STOP
+  }
+  if (accounting_is_enabled(options))
+    configure_accounting(time(NULL));
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay bandwidth actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_bandwidth(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* Check for transitions that need action. */
+  if (old_options) {
+    if (options->PerConnBWRate != old_options->PerConnBWRate ||
+        options->PerConnBWBurst != old_options->PerConnBWBurst)
+      connection_or_update_token_buckets(get_connection_array(), options);
+
+    if (options->RelayBandwidthRate != old_options->RelayBandwidthRate ||
+        options->RelayBandwidthBurst != old_options->RelayBandwidthBurst)
+      connection_bucket_adjust(options);
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take bridge statistics actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_bridge_stats(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+/* How long should we delay counting bridge stats after becoming a bridge?
+ * We use this so we don't count clients who used our bridge thinking it is
+ * a relay. If you change this, don't forget to change the log message
+ * below. It's 4 hours (the time it takes to stop being used by clients)
+ * plus some extra time for clock skew. */
+#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60)
+
+  /* Check for transitions that need action. */
+  if (old_options) {
+    if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) {
+      int was_relay = 0;
+      if (options->BridgeRelay) {
+        time_t int_start = time(NULL);
+        if (config_lines_eq(old_options->ORPort_lines,options->ORPort_lines)) {
+          int_start += RELAY_BRIDGE_STATS_DELAY;
+          was_relay = 1;
+        }
+        geoip_bridge_stats_init(int_start);
+        log_info(LD_CONFIG, "We are acting as a bridge now.  Starting new "
+                 "GeoIP stats interval%s.", was_relay ? " in 6 "
+                 "hours from now" : "");
+      } else {
+        geoip_bridge_stats_term();
+        log_info(LD_GENERAL, "We are no longer acting as a bridge.  "
+                 "Forgetting GeoIP stats.");
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay statistics actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Sets <b>*print_notice_out</b> if we enabled stats, and need to print
+ * a stats log using options_act_relay_stats_msg().
+ *
+ * If loading the GeoIP file failed, sets DirReqStatistics and
+ * EntryStatistics to 0. This breaks the normalization/act ordering
+ * introduced in 29211.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_stats(const or_options_t *old_options,
+                        bool *print_notice_out)
+{
+  if (BUG(!print_notice_out))
+    return -1;
+
+  or_options_t *options = get_options_mutable();
+
+  if (options->CellStatistics || options->DirReqStatistics ||
+      options->EntryStatistics || options->ExitPortStatistics ||
+      options->ConnDirectionStatistics ||
+      options->HiddenServiceStatistics) {
+    time_t now = time(NULL);
+    int print_notice = 0;
+
+    if ((!old_options || !old_options->CellStatistics) &&
+        options->CellStatistics) {
+      rep_hist_buffer_stats_init(now);
+      print_notice = 1;
+    }
+    if ((!old_options || !old_options->DirReqStatistics) &&
+        options->DirReqStatistics) {
+      if (geoip_is_loaded(AF_INET)) {
+        geoip_dirreq_stats_init(now);
+        print_notice = 1;
+      } else {
+        /* disable statistics collection since we have no geoip file */
+        /* 29211: refactor to avoid the normalisation/act inversion */
+        options->DirReqStatistics = 0;
+        if (options->ORPort_set)
+          log_notice(LD_CONFIG, "Configured to measure directory request "
+                                "statistics, but no GeoIP database found. "
+                                "Please specify a GeoIP database using the "
+                                "GeoIPFile option.");
+      }
+    }
+    if ((!old_options || !old_options->EntryStatistics) &&
+        options->EntryStatistics && !should_record_bridge_info(options)) {
+      /* If we get here, we've started recording bridge info when we didn't
+       * do so before.  Note that "should_record_bridge_info()" will
+       * always be false at this point, because of the earlier block
+       * that cleared EntryStatistics when public_server_mode() was false.
+       * We're leaving it in as defensive programming. */
+      if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) {
+        geoip_entry_stats_init(now);
+        print_notice = 1;
+      } else {
+        options->EntryStatistics = 0;
+        log_notice(LD_CONFIG, "Configured to measure entry node "
+                              "statistics, but no GeoIP database found. "
+                              "Please specify a GeoIP database using the "
+                              "GeoIPFile option.");
+      }
+    }
+    if ((!old_options || !old_options->ExitPortStatistics) &&
+        options->ExitPortStatistics) {
+      rep_hist_exit_stats_init(now);
+      print_notice = 1;
+    }
+    if ((!old_options || !old_options->ConnDirectionStatistics) &&
+        options->ConnDirectionStatistics) {
+      rep_hist_conn_stats_init(now);
+    }
+    if ((!old_options || !old_options->HiddenServiceStatistics) &&
+        options->HiddenServiceStatistics) {
+      log_info(LD_CONFIG, "Configured to measure hidden service statistics.");
+      rep_hist_hs_stats_init(now);
+    }
+    if (print_notice)
+      *print_notice_out = 1;
+  }
+
+  /* If we used to have statistics enabled but we just disabled them,
+     stop gathering them.  */
+  if (old_options && old_options->CellStatistics &&
+      !options->CellStatistics)
+    rep_hist_buffer_stats_term();
+  if (old_options && old_options->DirReqStatistics &&
+      !options->DirReqStatistics)
+    geoip_dirreq_stats_term();
+  if (old_options && old_options->EntryStatistics &&
+      !options->EntryStatistics)
+    geoip_entry_stats_term();
+  if (old_options && old_options->HiddenServiceStatistics &&
+      !options->HiddenServiceStatistics)
+    rep_hist_hs_stats_term();
+  if (old_options && old_options->ExitPortStatistics &&
+      !options->ExitPortStatistics)
+    rep_hist_exit_stats_term();
+  if (old_options && old_options->ConnDirectionStatistics &&
+      !options->ConnDirectionStatistics)
+    rep_hist_conn_stats_term();
+
+  return 0;
+}
+
+/** Print a notice about relay/dirauth stats being enabled. */
+void
+options_act_relay_stats_msg(void)
+{
+  log_notice(LD_CONFIG, "Configured to measure statistics. Look for "
+             "the *-stats files that will first be written to the "
+             "data directory in 24 hours from now.");
+}
+
+/** Fetch the active option list, and take relay descriptor actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_desc(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* Since our options changed, we might need to regenerate and upload our
+   * server descriptor.
+   */
+  if (!old_options ||
+      options_transition_affects_descriptor(old_options, options))
+    mark_my_descriptor_dirty("config change");
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay DoS actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_dos(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* DoS mitigation subsystem only applies to public relay. */
+  if (public_server_mode(options)) {
+    /* If we are configured as a relay, initialize the subsystem. Even on HUP,
+     * this is safe to call as it will load data from the current options
+     * or/and the consensus. */
+    dos_init();
+  } else if (old_options && public_server_mode(old_options)) {
+    /* Going from relay to non relay, clean it up. */
+    dos_free_all();
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirport actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_dir(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  const or_options_t *options = get_options();
+
+  if (!public_server_mode(options))
+    return 0;
+
+  /* Load the webpage we're going to serve every time someone asks for '/' on
+     our DirPort. */
+  tor_free(global_dirfrontpagecontents);
+  if (options->DirPortFrontPage) {
+    global_dirfrontpagecontents =
+      read_file_to_str(options->DirPortFrontPage, 0, NULL);
+    if (!global_dirfrontpagecontents) {
+      log_warn(LD_CONFIG,
+               "DirPortFrontPage file '%s' not found. Continuing anyway.",
+               options->DirPortFrontPage);
+    }
+  }
+
+  return 0;
+}
diff --git a/src/feature/relay/relay_config.h b/src/feature/relay/relay_config.h
new file mode 100644
index 0000000000..c70c322d88
--- /dev/null
+++ b/src/feature/relay/relay_config.h
@@ -0,0 +1,196 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_config.h
+ * @brief Header for feature/relay/relay_config.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_CONFIG_H
+#define TOR_FEATURE_RELAY_RELAY_CONFIG_H
+
+struct or_options_t;
+
+#ifdef HAVE_MODULE_RELAY
+
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+struct smartlist_t;
+
+int options_validate_relay_mode(const struct or_options_t *old_options,
+                                struct or_options_t *options,
+                                char **msg);
+
+MOCK_DECL(const char*, relay_get_dirportfrontpage, (void));
+void relay_config_free_all(void);
+
+uint32_t relay_get_effective_bwrate(const struct or_options_t *options);
+uint32_t relay_get_effective_bwburst(const struct or_options_t *options);
+
+void port_warn_nonlocal_ext_orports(const struct smartlist_t *ports,
+                               const char *portname);
+
+int port_parse_ports_relay(struct or_options_t *options,
+                      char **msg,
+                      struct smartlist_t *ports_out,
+                      int *have_low_ports_out);
+void port_update_port_set_relay(struct or_options_t *options,
+                           const struct smartlist_t *ports);
+
+int options_validate_relay_os(const struct or_options_t *old_options,
+                              struct or_options_t *options,
+                              char **msg);
+
+int options_validate_relay_info(const struct or_options_t *old_options,
+                                struct or_options_t *options,
+                                char **msg);
+
+int options_validate_publish_server(const struct or_options_t *old_options,
+                                    struct or_options_t *options,
+                                    char **msg);
+
+int options_validate_relay_padding(const struct or_options_t *old_options,
+                                   struct or_options_t *options,
+                                   char **msg);
+
+int options_validate_relay_bandwidth(const struct or_options_t *old_options,
+                                     struct or_options_t *options,
+                                     char **msg);
+
+int options_validate_relay_accounting(const struct or_options_t *old_options,
+                                      struct or_options_t *options,
+                                      char **msg);
+
+int options_validate_relay_testing(const struct or_options_t *old_options,
+                                   struct or_options_t *options,
+                                   char **msg);
+
+int options_act_relay(const struct or_options_t *old_options);
+int options_act_relay_accounting(const struct or_options_t *old_options);
+int options_act_relay_bandwidth(const struct or_options_t *old_options);
+int options_act_bridge_stats(const struct or_options_t *old_options);
+
+int options_act_relay_stats(const struct or_options_t *old_options,
+                            bool *print_notice_out);
+void options_act_relay_stats_msg(void);
+
+int options_act_relay_desc(const struct or_options_t *old_options);
+int options_act_relay_dos(const struct or_options_t *old_options);
+int options_act_relay_dir(const struct or_options_t *old_options);
+
+#ifdef RELAY_CONFIG_PRIVATE
+
+STATIC int check_bridge_distribution_setting(const char *bd);
+STATIC int have_enough_mem_for_dircache(const struct or_options_t *options,
+                                        size_t total_mem, char **msg);
+
+#endif /* defined(RELAY_CONFIG_PRIVATE) */
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#include "lib/cc/compat_compiler.h"
+
+/** When tor is compiled with the relay module disabled, it can't be
+ * configured as a relay or bridge.
+ *
+ * Always sets ClientOnly to 1.
+ *
+ * Returns -1 and sets msg to a newly allocated string, if ORPort, DirPort,
+ * DirCache, or BridgeRelay are set in options. Otherwise returns 0. */
+static inline int
+options_validate_relay_mode(const struct or_options_t *old_options,
+                            struct or_options_t *options,
+                            char **msg)
+{
+  (void)old_options;
+
+  /* Only check the primary options for now, #29211 will disable more
+   * options. These ORPort and DirPort checks are too strict, and will
+   * reject valid configs that disable ports, like "ORPort 0". */
+  if (options->DirCache ||
+      options->BridgeRelay ||
+      options->ORPort_lines ||
+      options->DirPort_lines) {
+    /* REJECT() this configuration */
+    *msg = tor_strdup("This tor was built with relay mode disabled. "
+                      "It can not be configured with an ORPort, a DirPort, "
+                      "DirCache 1, or BridgeRelay 1.");
+    return -1;
+  }
+
+  return 0;
+}
+
+static inline int
+port_parse_ports_relay(or_options_t *options,
+                       char **msg,
+                       smartlist_t *ports_out,
+                       int *have_low_ports_out)
+{
+  (void)options;
+  (void)msg;
+  (void)ports_out;
+  if (*have_low_ports_out < 0)
+    *have_low_ports_out = 0;
+  return 0;
+}
+
+#define relay_get_dirportfrontpage() \
+  (NULL)
+#define relay_config_free_all() \
+  STMT_BEGIN STMT_END
+
+#define relay_get_effective_bwrate(options) \
+  (((void)(options)),0)
+#define relay_get_effective_bwburst(options) \
+  (((void)(options)),0)
+
+#define port_warn_nonlocal_ext_orports(ports, portname) \
+  (((void)(ports)),((void)(portname)))
+
+#define port_update_port_set_relay(options, ports) \
+  (((void)(options)),((void)(ports)))
+
+#define options_validate_relay_os(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_info(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_publish_server(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_padding(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_bandwidth(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_accounting(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_testing(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+
+#define options_act_relay(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_accounting(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_bandwidth(old_options) \
+  (((void)(old_options)),0)
+#define options_act_bridge_stats(old_options) \
+  (((void)(old_options)),0)
+
+#define options_act_relay_stats(old_options, print_notice_out) \
+  (((void)(old_options)),((void)(print_notice_out)),0)
+#define options_act_relay_stats_msg() \
+  STMT_BEGIN STMT_END
+
+#define options_act_relay_desc(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_dos(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_dir(old_options) \
+  (((void)(old_options)),0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_CONFIG_H) */
diff --git a/src/feature/relay/relay_find_addr.c b/src/feature/relay/relay_find_addr.c
new file mode 100644
index 0000000000..86cd799d42
--- /dev/null
+++ b/src/feature/relay/relay_find_addr.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2001-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file relay_find_addr.c
+ * \brief Implement mechanism for a relay to find its address.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/resolve_addr.h"
+
+#include "core/mainloop/mainloop.h"
+
+#include "feature/control/control_events.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/relay/relay_find_addr.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+
+/** The most recently guessed value of our IP address, based on directory
+ * headers. */
+static tor_addr_t last_guessed_ip = TOR_ADDR_NULL;
+
+/** We failed to resolve our address locally, but we'd like to build
+ * a descriptor and publish / test reachability. If we have a guess
+ * about our address based on directory headers, answer it and return
+ * 0; else return -1. */
+static int
+router_guess_address_from_dir_headers(uint32_t *guess)
+{
+  if (!tor_addr_is_null(&last_guessed_ip)) {
+    *guess = tor_addr_to_ipv4h(&last_guessed_ip);
+    return 0;
+  }
+  return -1;
+}
+
+/** A directory server <b>d_conn</b> told us our IP address is
+ * <b>suggestion</b>.
+ * If this address is different from the one we think we are now, and
+ * if our computer doesn't actually know its IP address, then switch. */
+void
+router_new_address_suggestion(const char *suggestion,
+                              const dir_connection_t *d_conn)
+{
+  tor_addr_t addr;
+  uint32_t cur = 0;             /* Current IPv4 address.  */
+  const or_options_t *options = get_options();
+
+  /* first, learn what the IP address actually is */
+  if (tor_addr_parse(&addr, suggestion) == -1) {
+    log_debug(LD_DIR, "Malformed X-Your-Address-Is header %s. Ignoring.",
+              escaped(suggestion));
+    return;
+  }
+
+  log_debug(LD_DIR, "Got X-Your-Address-Is: %s.", suggestion);
+
+  if (!server_mode(options)) {
+    tor_addr_copy(&last_guessed_ip, &addr);
+    return;
+  }
+
+  /* XXXX ipv6 */
+  cur = get_last_resolved_addr();
+  if (cur ||
+      resolve_my_address(LOG_INFO, options, &cur, NULL, NULL) >= 0) {
+    /* We're all set -- we already know our address. Great. */
+    tor_addr_from_ipv4h(&last_guessed_ip, cur); /* store it in case we
+                                                   need it later */
+    return;
+  }
+  if (tor_addr_is_internal(&addr, 0)) {
+    /* Don't believe anybody who says our IP is, say, 127.0.0.1. */
+    return;
+  }
+  if (tor_addr_eq(&d_conn->base_.addr, &addr)) {
+    /* Don't believe anybody who says our IP is their IP. */
+    log_debug(LD_DIR, "A directory server told us our IP address is %s, "
+              "but they are just reporting their own IP address. Ignoring.",
+              suggestion);
+    return;
+  }
+
+  /* Okay.  We can't resolve our own address, and X-Your-Address-Is is giving
+   * us an answer different from what we had the last time we managed to
+   * resolve it. */
+  if (!tor_addr_eq(&last_guessed_ip, &addr)) {
+    control_event_server_status(LOG_NOTICE,
+                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=DIRSERV",
+                                suggestion);
+    log_addr_has_changed(LOG_NOTICE, &last_guessed_ip, &addr,
+                         d_conn->base_.address);
+    ip_address_changed(0);
+    tor_addr_copy(&last_guessed_ip, &addr); /* router_rebuild_descriptor()
+                                               will fetch it */
+  }
+}
+
+/** Make a current best guess at our address, either because
+ * it's configured in torrc, or because we've learned it from
+ * dirserver headers. Place the answer in *<b>addr</b> and return
+ * 0 on success, else return -1 if we have no guess.
+ *
+ * If <b>cache_only</b> is true, just return any cached answers, and
+ * don't try to get any new answers.
+ */
+MOCK_IMPL(int,
+router_pick_published_address, (const or_options_t *options, uint32_t *addr,
+                                int cache_only))
+{
+  /* First, check the cached output from resolve_my_address(). */
+  *addr = get_last_resolved_addr();
+  if (*addr)
+    return 0;
+
+  /* Second, consider doing a resolve attempt right here. */
+  if (!cache_only) {
+    if (resolve_my_address(LOG_INFO, options, addr, NULL, NULL) >= 0) {
+      log_info(LD_CONFIG,"Success: chose address '%s'.", fmt_addr32(*addr));
+      return 0;
+    }
+  }
+
+  /* Third, check the cached output from router_new_address_suggestion(). */
+  if (router_guess_address_from_dir_headers(addr) >= 0)
+    return 0;
+
+  /* We have no useful cached answers. Return failure. */
+  return -1;
+}
diff --git a/src/feature/relay/relay_find_addr.h b/src/feature/relay/relay_find_addr.h
new file mode 100644
index 0000000000..ac51a977e6
--- /dev/null
+++ b/src/feature/relay/relay_find_addr.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file relay_find_addr.h
+ * \brief Header file for relay_find_addr.c.
+ **/
+
+#ifndef TOR_RELAY_FIND_ADDR_H
+#define TOR_RELAY_FIND_ADDR_H
+
+MOCK_DECL(int, router_pick_published_address,
+          (const or_options_t *options, uint32_t *addr, int cache_only));
+
+void router_new_address_suggestion(const char *suggestion,
+                                   const dir_connection_t *d_conn);
+
+#ifdef RELAY_FIND_ADDR_PRIVATE
+
+#endif /* RELAY_FIND_ADDR_PRIVATE */
+
+#endif /* TOR_RELAY_FIND_ADDR_H */
+
diff --git a/src/feature/relay/relay_handshake.c b/src/feature/relay/relay_handshake.c
new file mode 100644
index 0000000000..030dc94956
--- /dev/null
+++ b/src/feature/relay/relay_handshake.c
@@ -0,0 +1,565 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_handshake.c
+ * @brief Functions to implement the relay-only parts of our
+ *    connection handshake.
+ *
+ * Some parts of our TLS link handshake are only done by relays (including
+ * bridges).  Specifically, only relays need to send CERTS cells; only
+ * relays need to send or receive AUTHCHALLENGE cells, and only relays need to
+ * send or receive AUTHENTICATE cells.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "feature/relay/relay_handshake.h"
+
+#include "app/config/config.h"
+#include "core/or/connection_or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "trunnel/link_handshake.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/torcert.h"
+
+#include "core/or/or_connection_st.h"
+#include "core/or/or_handshake_certs_st.h"
+#include "core/or/or_handshake_state_st.h"
+#include "core/or/var_cell_st.h"
+
+#include "lib/tls/tortls.h"
+#include "lib/tls/x509.h"
+
+/** Helper used to add an encoded certs to a cert cell */
+static void
+add_certs_cell_cert_helper(certs_cell_t *certs_cell,
+                           uint8_t cert_type,
+                           const uint8_t *cert_encoded,
+                           size_t cert_len)
+{
+  tor_assert(cert_len <= UINT16_MAX);
+  certs_cell_cert_t *ccc = certs_cell_cert_new();
+  ccc->cert_type = cert_type;
+  ccc->cert_len = cert_len;
+  certs_cell_cert_setlen_body(ccc, cert_len);
+  memcpy(certs_cell_cert_getarray_body(ccc), cert_encoded, cert_len);
+
+  certs_cell_add_certs(certs_cell, ccc);
+}
+
+/** Add an encoded X509 cert (stored as <b>cert_len</b> bytes at
+ * <b>cert_encoded</b>) to the trunnel certs_cell_t object that we are
+ * building in <b>certs_cell</b>.  Set its type field to <b>cert_type</b>.
+ * (If <b>cert</b> is NULL, take no action.) */
+static void
+add_x509_cert(certs_cell_t *certs_cell,
+              uint8_t cert_type,
+              const tor_x509_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  const uint8_t *cert_encoded = NULL;
+  size_t cert_len;
+  tor_x509_cert_get_der(cert, &cert_encoded, &cert_len);
+
+  add_certs_cell_cert_helper(certs_cell, cert_type, cert_encoded, cert_len);
+}
+
+/** Add an Ed25519 cert from <b>cert</b> to the trunnel certs_cell_t object
+ * that we are building in <b>certs_cell</b>.  Set its type field to
+ * <b>cert_type</b>. (If <b>cert</b> is NULL, take no action.) */
+static void
+add_ed25519_cert(certs_cell_t *certs_cell,
+                 uint8_t cert_type,
+                 const tor_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  add_certs_cell_cert_helper(certs_cell, cert_type,
+                             cert->encoded, cert->encoded_len);
+}
+
+#ifdef TOR_UNIT_TESTS
+int certs_cell_ed25519_disabled_for_testing = 0;
+#else
+#define certs_cell_ed25519_disabled_for_testing 0
+#endif
+
+/** Send a CERTS cell on the connection <b>conn</b>.  Return 0 on success, -1
+ * on failure. */
+int
+connection_or_send_certs_cell(or_connection_t *conn)
+{
+  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL;
+  tor_x509_cert_t *own_link_cert = NULL;
+  var_cell_t *cell;
+
+  certs_cell_t *certs_cell = NULL;
+
+  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
+
+  if (! conn->handshake_state)
+    return -1;
+
+  const int conn_in_server_mode = ! conn->handshake_state->started_here;
+
+  /* Get the encoded values of the X509 certificates */
+  if (tor_tls_get_my_certs(conn_in_server_mode,
+                           &global_link_cert, &id_cert) < 0)
+    return -1;
+
+  if (conn_in_server_mode) {
+    own_link_cert = tor_tls_get_own_cert(conn->tls);
+  }
+  tor_assert(id_cert);
+
+  certs_cell = certs_cell_new();
+
+  /* Start adding certs.  First the link cert or auth1024 cert. */
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(own_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_TLS_LINK, own_link_cert);
+  } else {
+    tor_assert(global_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_AUTH_1024, global_link_cert);
+  }
+
+  /* Next the RSA->RSA ID cert */
+  add_x509_cert(certs_cell,
+                OR_CERT_TYPE_ID_1024, id_cert);
+
+  /* Next the Ed25519 certs */
+  add_ed25519_cert(certs_cell,
+                   CERTTYPE_ED_ID_SIGN,
+                   get_master_signing_key_cert());
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(conn->handshake_state->own_link_cert ||
+                        certs_cell_ed25519_disabled_for_testing);
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_LINK,
+                     conn->handshake_state->own_link_cert);
+  } else {
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_AUTH,
+                     get_current_auth_key_cert());
+  }
+
+  /* And finally the crosscert. */
+  {
+    const uint8_t *crosscert=NULL;
+    size_t crosscert_len;
+    get_master_rsa_crosscert(&crosscert, &crosscert_len);
+    if (crosscert) {
+      add_certs_cell_cert_helper(certs_cell,
+                               CERTTYPE_RSA1024_ID_EDID,
+                               crosscert, crosscert_len);
+    }
+  }
+
+  /* We've added all the certs; make the cell. */
+  certs_cell->n_certs = certs_cell_getlen_certs(certs_cell);
+
+  ssize_t alloc_len = certs_cell_encoded_len(certs_cell);
+  tor_assert(alloc_len >= 0 && alloc_len <= UINT16_MAX);
+  cell = var_cell_new(alloc_len);
+  cell->command = CELL_CERTS;
+  ssize_t enc_len = certs_cell_encode(cell->payload, alloc_len, certs_cell);
+  tor_assert(enc_len > 0 && enc_len <= alloc_len);
+  cell->payload_len = enc_len;
+
+  connection_or_write_var_cell_to_buf(cell, conn);
+  var_cell_free(cell);
+  certs_cell_free(certs_cell);
+  tor_x509_cert_free(own_link_cert);
+
+  return 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+int testing__connection_or_pretend_TLSSECRET_is_supported = 0;
+#else
+#define testing__connection_or_pretend_TLSSECRET_is_supported 0
+#endif
+
+/** Return true iff <b>challenge_type</b> is an AUTHCHALLENGE type that
+ * we can send and receive. */
+int
+authchallenge_type_is_supported(uint16_t challenge_type)
+{
+  switch (challenge_type) {
+     case AUTHTYPE_RSA_SHA256_TLSSECRET:
+#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+       return 1;
+#else
+       return testing__connection_or_pretend_TLSSECRET_is_supported;
+#endif
+     case AUTHTYPE_ED25519_SHA256_RFC5705:
+       return 1;
+     case AUTHTYPE_RSA_SHA256_RFC5705:
+     default:
+       return 0;
+  }
+}
+
+/** Return true iff <b>challenge_type_a</b> is one that we would rather
+ * use than <b>challenge_type_b</b>. */
+int
+authchallenge_type_is_better(uint16_t challenge_type_a,
+                             uint16_t challenge_type_b)
+{
+  /* Any supported type is better than an unsupported one;
+   * all unsupported types are equally bad. */
+  if (!authchallenge_type_is_supported(challenge_type_a))
+    return 0;
+  if (!authchallenge_type_is_supported(challenge_type_b))
+    return 1;
+  /* It happens that types are superior in numerically ascending order.
+   * If that ever changes, this must change too. */
+  return (challenge_type_a > challenge_type_b);
+}
+
+/** Send an AUTH_CHALLENGE cell on the connection <b>conn</b>. Return 0
+ * on success, -1 on failure. */
+int
+connection_or_send_auth_challenge_cell(or_connection_t *conn)
+{
+  var_cell_t *cell = NULL;
+  int r = -1;
+  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
+
+  if (! conn->handshake_state)
+    return -1;
+
+  auth_challenge_cell_t *ac = auth_challenge_cell_new();
+
+  tor_assert(sizeof(ac->challenge) == 32);
+  crypto_rand((char*)ac->challenge, sizeof(ac->challenge));
+
+  if (authchallenge_type_is_supported(AUTHTYPE_RSA_SHA256_TLSSECRET))
+    auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
+  /* Disabled, because everything that supports this method also supports
+   * the much-superior ED25519_SHA256_RFC5705 */
+  /* auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_RFC5705); */
+  if (authchallenge_type_is_supported(AUTHTYPE_ED25519_SHA256_RFC5705))
+    auth_challenge_cell_add_methods(ac, AUTHTYPE_ED25519_SHA256_RFC5705);
+  auth_challenge_cell_set_n_methods(ac,
+                                    auth_challenge_cell_getlen_methods(ac));
+
+  cell = var_cell_new(auth_challenge_cell_encoded_len(ac));
+  ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len,
+                                           ac);
+  if (len != cell->payload_len) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Encoded auth challenge cell length not as expected");
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+  cell->command = CELL_AUTH_CHALLENGE;
+
+  connection_or_write_var_cell_to_buf(cell, conn);
+  r = 0;
+
+ done:
+  var_cell_free(cell);
+  auth_challenge_cell_free(ac);
+
+  return r;
+}
+
+/** Compute the main body of an AUTHENTICATE cell that a client can use
+ * to authenticate itself on a v3 handshake for <b>conn</b>.  Return it
+ * in a var_cell_t.
+ *
+ * If <b>server</b> is true, only calculate the first
+ * V3_AUTH_FIXED_PART_LEN bytes -- the part of the authenticator that's
+ * determined by the rest of the handshake, and which match the provided value
+ * exactly.
+ *
+ * If <b>server</b> is false and <b>signing_key</b> is NULL, calculate the
+ * first V3_AUTH_BODY_LEN bytes of the authenticator (that is, everything
+ * that should be signed), but don't actually sign it.
+ *
+ * If <b>server</b> is false and <b>signing_key</b> is provided, calculate the
+ * entire authenticator, signed with <b>signing_key</b>.
+ *
+ * Return the length of the cell body on success, and -1 on failure.
+ */
+var_cell_t *
+connection_or_compute_authenticate_cell_body(or_connection_t *conn,
+                                             const int authtype,
+                                             crypto_pk_t *signing_key,
+                                      const ed25519_keypair_t *ed_signing_key,
+                                      int server)
+{
+  auth1_t *auth = NULL;
+  auth_ctx_t *ctx = auth_ctx_new();
+  var_cell_t *result = NULL;
+  int old_tlssecrets_algorithm = 0;
+  const char *authtype_str = NULL;
+
+  int is_ed = 0;
+
+  /* assert state is reasonable XXXX */
+  switch (authtype) {
+  case AUTHTYPE_RSA_SHA256_TLSSECRET:
+    authtype_str = "AUTH0001";
+    old_tlssecrets_algorithm = 1;
+    break;
+  case AUTHTYPE_RSA_SHA256_RFC5705:
+    authtype_str = "AUTH0002";
+    break;
+  case AUTHTYPE_ED25519_SHA256_RFC5705:
+    authtype_str = "AUTH0003";
+    is_ed = 1;
+    break;
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  auth = auth1_new();
+  ctx->is_ed = is_ed;
+
+  /* Type: 8 bytes. */
+  memcpy(auth1_getarray_type(auth), authtype_str, 8);
+
+  {
+    const tor_x509_cert_t *id_cert=NULL;
+    const common_digests_t *my_digests, *their_digests;
+    const uint8_t *my_id, *their_id, *client_id, *server_id;
+    if (tor_tls_get_my_certs(server, NULL, &id_cert))
+      goto err;
+    my_digests = tor_x509_cert_get_id_digests(id_cert);
+    their_digests =
+      tor_x509_cert_get_id_digests(conn->handshake_state->certs->id_cert);
+    tor_assert(my_digests);
+    tor_assert(their_digests);
+    my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
+    their_id = (uint8_t*)their_digests->d[DIGEST_SHA256];
+
+    client_id = server ? their_id : my_id;
+    server_id = server ? my_id : their_id;
+
+    /* Client ID digest: 32 octets. */
+    memcpy(auth->cid, client_id, 32);
+
+    /* Server ID digest: 32 octets. */
+    memcpy(auth->sid, server_id, 32);
+  }
+
+  if (is_ed) {
+    const ed25519_public_key_t *my_ed_id, *their_ed_id;
+    if (!conn->handshake_state->certs->ed_id_sign) {
+      log_warn(LD_OR, "Ed authenticate without Ed ID cert from peer.");
+      goto err;
+    }
+    my_ed_id = get_master_identity_key();
+    their_ed_id = &conn->handshake_state->certs->ed_id_sign->signing_key;
+
+    const uint8_t *cid_ed = (server ? their_ed_id : my_ed_id)->pubkey;
+    const uint8_t *sid_ed = (server ? my_ed_id : their_ed_id)->pubkey;
+
+    memcpy(auth->u1_cid_ed, cid_ed, ED25519_PUBKEY_LEN);
+    memcpy(auth->u1_sid_ed, sid_ed, ED25519_PUBKEY_LEN);
+  }
+
+  {
+    crypto_digest_t *server_d, *client_d;
+    if (server) {
+      server_d = conn->handshake_state->digest_sent;
+      client_d = conn->handshake_state->digest_received;
+    } else {
+      client_d = conn->handshake_state->digest_sent;
+      server_d = conn->handshake_state->digest_received;
+    }
+
+    /* Server log digest : 32 octets */
+    crypto_digest_get_digest(server_d, (char*)auth->slog, 32);
+
+    /* Client log digest : 32 octets */
+    crypto_digest_get_digest(client_d, (char*)auth->clog, 32);
+  }
+
+  {
+    /* Digest of cert used on TLS link : 32 octets. */
+    tor_x509_cert_t *cert = NULL;
+    if (server) {
+      cert = tor_tls_get_own_cert(conn->tls);
+    } else {
+      cert = tor_tls_get_peer_cert(conn->tls);
+    }
+    if (!cert) {
+      log_warn(LD_OR, "Unable to find cert when making %s data.",
+               authtype_str);
+      goto err;
+    }
+
+    memcpy(auth->scert,
+           tor_x509_cert_get_cert_digests(cert)->d[DIGEST_SHA256], 32);
+
+    tor_x509_cert_free(cert);
+  }
+
+  /* HMAC of clientrandom and serverrandom using master key : 32 octets */
+  if (old_tlssecrets_algorithm) {
+    if (tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_OR, "Somebody asked us for an older TLS "
+         "authentication method (AUTHTYPE_RSA_SHA256_TLSSECRET) "
+         "which we don't support.");
+    }
+  } else {
+    char label[128];
+    tor_snprintf(label, sizeof(label),
+                 "EXPORTER FOR TOR TLS CLIENT BINDING %s", authtype_str);
+    int r = tor_tls_export_key_material(conn->tls, auth->tlssecrets,
+                                        auth->cid, sizeof(auth->cid),
+                                        label);
+    if (r < 0) {
+      if (r != -2)
+        log_warn(LD_BUG, "TLS key export failed for unknown reason.");
+      // If r == -2, this was openssl bug 7712.
+      goto err;
+    }
+  }
+
+  /* 8 octets were reserved for the current time, but we're trying to get out
+   * of the habit of sending time around willynilly.  Fortunately, nothing
+   * checks it.  That's followed by 16 bytes of nonce. */
+  crypto_rand((char*)auth->rand, 24);
+
+  ssize_t maxlen = auth1_encoded_len(auth, ctx);
+  if (ed_signing_key && is_ed) {
+    maxlen += ED25519_SIG_LEN;
+  } else if (signing_key && !is_ed) {
+    maxlen += crypto_pk_keysize(signing_key);
+  }
+
+  const int AUTH_CELL_HEADER_LEN = 4; /* 2 bytes of type, 2 bytes of length */
+  result = var_cell_new(AUTH_CELL_HEADER_LEN + maxlen);
+  uint8_t *const out = result->payload + AUTH_CELL_HEADER_LEN;
+  const size_t outlen = maxlen;
+  ssize_t len;
+
+  result->command = CELL_AUTHENTICATE;
+  set_uint16(result->payload, htons(authtype));
+
+  if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed part of AUTH1 data.");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (server) {
+    auth1_t *tmp = NULL;
+    ssize_t len2 = auth1_parse(&tmp, out, len, ctx);
+    if (!tmp) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to parse signed part of AUTH1 data that "
+               "we just encoded");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    result->payload_len = (tmp->end_of_signed - result->payload);
+
+    auth1_free(tmp);
+    if (len2 != len) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Mismatched length when re-parsing AUTH1 data.");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    goto done;
+  }
+
+  if (ed_signing_key && is_ed) {
+    ed25519_signature_t sig;
+    if (ed25519_sign(&sig, out, len, ed_signing_key) < 0) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to sign ed25519 authentication data");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    auth1_setlen_sig(auth, ED25519_SIG_LEN);
+    memcpy(auth1_getarray_sig(auth), sig.sig, ED25519_SIG_LEN);
+
+  } else if (signing_key && !is_ed) {
+    auth1_setlen_sig(auth, crypto_pk_keysize(signing_key));
+
+    char d[32];
+    crypto_digest256(d, (char*)out, len, DIGEST_SHA256);
+    int siglen = crypto_pk_private_sign(signing_key,
+                                    (char*)auth1_getarray_sig(auth),
+                                    auth1_getlen_sig(auth),
+                                    d, 32);
+    if (siglen < 0) {
+      log_warn(LD_OR, "Unable to sign AUTH1 data.");
+      goto err;
+    }
+
+    auth1_setlen_sig(auth, siglen);
+  }
+
+  len = auth1_encode(out, outlen, auth, ctx);
+  if (len < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed AUTH1 data.");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+  tor_assert(len + AUTH_CELL_HEADER_LEN <= result->payload_len);
+  result->payload_len = len + AUTH_CELL_HEADER_LEN;
+  set_uint16(result->payload+2, htons(len));
+
+  goto done;
+
+ err:
+  var_cell_free(result);
+  result = NULL;
+ done:
+  auth1_free(auth);
+  auth_ctx_free(ctx);
+  return result;
+}
+
+/** Send an AUTHENTICATE cell on the connection <b>conn</b>.  Return 0 on
+ * success, -1 on failure */
+MOCK_IMPL(int,
+connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype))
+{
+  var_cell_t *cell;
+  crypto_pk_t *pk = tor_tls_get_my_client_auth_key();
+  /* XXXX make sure we're actually supposed to send this! */
+
+  if (!pk) {
+    log_warn(LD_BUG, "Can't compute authenticate cell: no client auth key");
+    return -1;
+  }
+  if (! authchallenge_type_is_supported(authtype)) {
+    log_warn(LD_BUG, "Tried to send authenticate cell with unknown "
+             "authentication type %d", authtype);
+    return -1;
+  }
+
+  cell = connection_or_compute_authenticate_cell_body(conn,
+                                                 authtype,
+                                                 pk,
+                                                 get_current_auth_keypair(),
+                                                 0 /* not server */);
+  if (! cell) {
+    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unable to compute authenticate cell!");
+    return -1;
+  }
+  connection_or_write_var_cell_to_buf(cell, conn);
+  var_cell_free(cell);
+
+  return 0;
+}
diff --git a/src/feature/relay/relay_handshake.h b/src/feature/relay/relay_handshake.h
new file mode 100644
index 0000000000..99a658cbcc
--- /dev/null
+++ b/src/feature/relay/relay_handshake.h
@@ -0,0 +1,90 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_handshake.h
+ * @brief Header for feature/relay/relay_handshake.c
+ **/
+
+#ifndef TOR_CORE_OR_RELAY_HANDSHAKE_H
+#define TOR_CORE_OR_RELAY_HANDSHAKE_H
+
+#ifdef HAVE_MODULE_RELAY
+struct ed25519_keypair_t;
+
+int connection_or_send_certs_cell(or_connection_t *conn);
+int connection_or_send_auth_challenge_cell(or_connection_t *conn);
+
+var_cell_t *connection_or_compute_authenticate_cell_body(
+                              or_connection_t *conn,
+                              const int authtype,
+                              crypto_pk_t *signing_key,
+                              const struct ed25519_keypair_t *ed_signing_key,
+                              int server);
+
+int authchallenge_type_is_supported(uint16_t challenge_type);
+int authchallenge_type_is_better(uint16_t challenge_type_a,
+                                 uint16_t challenge_type_b);
+
+MOCK_DECL(int,connection_or_send_authenticate_cell,
+          (or_connection_t *conn, int type));
+
+#ifdef TOR_UNIT_TESTS
+extern int certs_cell_ed25519_disabled_for_testing;
+#endif
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+static inline int
+connection_or_send_certs_cell(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+static inline int
+connection_or_send_auth_challenge_cell(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+static inline var_cell_t *
+connection_or_compute_authenticate_cell_body(
+                              or_connection_t *conn,
+                              const int authtype,
+                              crypto_pk_t *signing_key,
+                              const struct ed25519_keypair_t *ed_signing_key,
+                              int server)
+{
+  (void)conn;
+  (void)authtype;
+  (void)signing_key;
+  (void)ed_signing_key;
+  (void)server;
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+
+#define authchallenge_type_is_supported(t) (0)
+#define authchallenge_type_is_better(a, b) (0)
+
+static inline int
+connection_or_send_authenticate_cell(or_connection_t *conn, int type)
+{
+  (void)conn;
+  (void)type;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+#ifdef TOR_UNIT_TESTS
+extern int certs_cell_ed25519_disabled_for_testing;
+#endif
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_CORE_OR_RELAY_HANDSHAKE_H) */
diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c
new file mode 100644
index 0000000000..bfe12cd0b0
--- /dev/null
+++ b/src/feature/relay/relay_periodic.c
@@ -0,0 +1,318 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.c
+ * @brief Periodic functions for the relay subsytem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+#include "core/mainloop/cpuworker.h" // XXXX use a pubsub event.
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/circuituse.h" // XXXX move have_performed_bandwidth_test
+
+#include "feature/relay/dns.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/stats/predict_ports.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/control/control_events.h"
+
+#ifndef COCCI
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+#endif /* !defined(COCCI) */
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: If we're a server and initializing dns failed, retry.
+ */
+static int
+retry_dns_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+#define RETRY_DNS_INTERVAL (10*60)
+  if (server_mode(options) && has_dns_init_failed())
+    dns_init();
+  return RETRY_DNS_INTERVAL;
+}
+
+DECLARE_EVENT(retry_dns, ROUTER, 0);
+
+static int dns_honesty_first_time = 1;
+
+/**
+ * Periodic event: if we're an exit, see if our DNS server is telling us
+ * obvious lies.
+ */
+static int
+check_dns_honesty_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  /* 9. and if we're an exit node, check whether our DNS is telling stories
+   * to us. */
+  if (net_is_disabled() ||
+      ! public_server_mode(options) ||
+      router_my_exit_policy_is_reject_star())
+    return PERIODIC_EVENT_NO_UPDATE;
+
+  if (dns_honesty_first_time) {
+    /* Don't launch right when we start */
+    dns_honesty_first_time = 0;
+    return crypto_rand_int_range(60, 180);
+  }
+
+  dns_launch_correctness_checks();
+  return 12*3600 + crypto_rand_int(12*3600);
+}
+
+DECLARE_EVENT(check_dns_honesty, RELAY, FL(NEED_NET));
+
+/* Periodic callback: rotate the onion keys after the period defined by the
+ * "onion-key-rotation-days" consensus parameter, shut down and restart all
+ * cpuworkers, and update our descriptor if necessary.
+ */
+static int
+rotate_onion_key_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_lifetime = get_onion_key_lifetime();
+    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
+    if (rotation_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL,"Rotating onion key.");
+    rotate_onion_key();
+    cpuworkers_rotate_keyinfo();
+    if (router_rebuild_descriptor(1)<0) {
+      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
+    }
+    if (advertised_server_mode() && !net_is_disabled())
+      router_upload_dir_desc_to_dirservers(0);
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(rotate_onion_key, ROUTER, 0);
+
+/** Periodic callback: consider rebuilding or and re-uploading our descriptor
+ * (if we've passed our internal checks). */
+static int
+check_descriptor_callback(time_t now, const or_options_t *options)
+{
+/** How often do we check whether part of our router info has changed in a
+ * way that would require an upload? That includes checking whether our IP
+ * address has changed. */
+#define CHECK_DESCRIPTOR_INTERVAL (60)
+
+  (void)options;
+
+  /* 2b. Once per minute, regenerate and upload the descriptor if the old
+   * one is inaccurate. */
+  if (!net_is_disabled()) {
+    check_descriptor_bandwidth_changed(now);
+    check_descriptor_ipaddress_changed(now);
+    mark_my_descriptor_dirty_if_too_old(now);
+    consider_publishable_server(0);
+  }
+
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_descriptor, ROUTER, FL(NEED_NET));
+
+static int dirport_reachability_count = 0;
+
+/**
+ * Periodic callback: check whether we're reachable (as a relay), and
+ * whether our bandwidth has changed enough that we need to
+ * publish a new descriptor.
+ */
+static int
+check_for_reachability_bw_callback(time_t now, const or_options_t *options)
+{
+  /* XXXX This whole thing was stuck in the middle of what is now
+   * XXXX check_descriptor_callback.  I'm not sure it's right. */
+  /** How often should we consider launching reachability tests in our first
+   * TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT seconds? */
+#define EARLY_CHECK_REACHABILITY_INTERVAL (60)
+
+  /* also, check religiously for reachability, if it's within the first
+   * 20 minutes of our uptime. */
+  if (server_mode(options) &&
+      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
+      !net_is_disabled()) {
+    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+      router_do_reachability_checks(1, dirport_reachability_count==0);
+      if (++dirport_reachability_count > 5)
+        dirport_reachability_count = 0;
+      return EARLY_CHECK_REACHABILITY_INTERVAL;
+    } else {
+      /* If we haven't checked for 12 hours and our bandwidth estimate is
+       * low, do another bandwidth test. This is especially important for
+       * bridges, since they might go long periods without much use. */
+      const routerinfo_t *me = router_get_my_routerinfo();
+      static int first_time = 1;
+      if (!first_time && me &&
+          me->bandwidthcapacity < me->bandwidthrate &&
+          me->bandwidthcapacity < 51200) {
+        reset_bandwidth_test();
+      }
+      first_time = 0;
+#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
+      return BANDWIDTH_RECHECK_INTERVAL;
+    }
+  }
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_for_reachability_bw, ROUTER, FL(NEED_NET));
+
+/**
+ * Callback: Send warnings if Tor doesn't find its ports reachable.
+ */
+static int
+reachability_warnings_callback(time_t now, const or_options_t *options)
+{
+  (void) now;
+
+  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
+  }
+
+  if (server_mode(options) &&
+      !net_is_disabled() &&
+      have_completed_a_circuit()) {
+    /* every 20 minutes, check and complain if necessary */
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me && !check_whether_orport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      if (address) {
+        log_warn(LD_CONFIG,
+                 "Your server (%s:%d) has not managed to confirm that "
+                 "its ORPort is reachable. Relays do not publish descriptors "
+                 "until their ORPort and DirPort are reachable. Please check "
+                 "your firewalls, ports, address, /etc/hosts file, etc.",
+                 address, me->or_port);
+        control_event_server_status(LOG_WARN,
+                                    "REACHABILITY_FAILED ORADDRESS=%s:%d",
+                                    address, me->or_port);
+        tor_free(address);
+      }
+    }
+
+    if (me && !check_whether_dirport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      if (address) {
+        log_warn(LD_CONFIG,
+                 "Your server (%s:%d) has not managed to confirm that its "
+                 "DirPort is reachable. Relays do not publish descriptors "
+                 "until their ORPort and DirPort are reachable. Please check "
+                 "your firewalls, ports, address, /etc/hosts file, etc.",
+                 address, me->dir_port);
+        control_event_server_status(LOG_WARN,
+                                    "REACHABILITY_FAILED DIRADDRESS=%s:%d",
+                                    address, me->dir_port);
+        tor_free(address);
+      }
+    }
+  }
+
+  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
+}
+
+DECLARE_EVENT(reachability_warnings, ROUTER, FL(NEED_NET));
+
+/* Periodic callback: Every 30 seconds, check whether it's time to make new
+ * Ed25519 subkeys.
+ */
+static int
+check_ed_keys_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    if (should_make_new_ed_keys(options, now)) {
+      int new_signing_key = load_ed_keys(options, now);
+      if (new_signing_key < 0 ||
+          generate_ed_link_cert(options, now, new_signing_key > 0)) {
+        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
+        tor_shutdown_event_loop_and_exit(1);
+      }
+    }
+    return 30;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_ed_keys, ROUTER, 0);
+
+/* Period callback: Check if our old onion keys are still valid after the
+ * period of time defined by the consensus parameter
+ * "onion-key-grace-period-days", otherwise expire them by setting them to
+ * NULL.
+ */
+static int
+check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_grace_period = get_onion_key_grace_period();
+    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
+    if (expiry_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL, "Expiring old onion keys.");
+    expire_old_onion_keys();
+    cpuworkers_rotate_keyinfo();
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_onion_keys_expiry_time, ROUTER, 0);
+
+void
+relay_register_periodic_events(void)
+{
+  periodic_events_register(&retry_dns_event);
+  periodic_events_register(&check_dns_honesty_event);
+  periodic_events_register(&rotate_onion_key_event);
+  periodic_events_register(&check_descriptor_event);
+  periodic_events_register(&check_for_reachability_bw_event);
+  periodic_events_register(&reachability_warnings_event);
+  periodic_events_register(&check_ed_keys_event);
+  periodic_events_register(&check_onion_keys_expiry_time_event);
+
+  dns_honesty_first_time = 1;
+  dirport_reachability_count = 0;
+}
+
+/**
+ * Update our schedule so that we'll check whether we need to update our
+ * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
+ * seconds.
+ */
+void
+reschedule_descriptor_update_check(void)
+{
+  periodic_event_reschedule(&check_descriptor_event);
+}
diff --git a/src/feature/relay/relay_periodic.h b/src/feature/relay/relay_periodic.h
new file mode 100644
index 0000000000..ccda9a440b
--- /dev/null
+++ b/src/feature/relay/relay_periodic.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.h
+ * @brief Header for feature/relay/relay_periodic.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+#define TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+
+#ifdef HAVE_MODULE_RELAY
+
+void relay_register_periodic_events(void);
+void reschedule_descriptor_update_check(void);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#include "lib/cc/compat_compiler.h"
+
+#define relay_register_periodic_events() \
+  STMT_NIL
+#define reschedule_descriptor_update_check() \
+  STMT_NIL
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_PERIODIC_H) */
diff --git a/src/feature/relay/relay_stub.c b/src/feature/relay/relay_stub.c
new file mode 100644
index 0000000000..283aaf6e49
--- /dev/null
+++ b/src/feature/relay/relay_stub.c
@@ -0,0 +1,21 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_stub.c
+ * @brief Stub declarations for use when relay module is disabled.
+ **/
+
+#include "orconfig.h"
+#include "feature/relay/relay_sys.h"
+#include "lib/subsys/subsys.h"
+
+const struct subsys_fns_t sys_relay = {
+  .name = "relay",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = false,
+  .level = RELAY_SUBSYS_LEVEL,
+};
diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c
new file mode 100644
index 0000000000..2e90740925
--- /dev/null
+++ b/src/feature/relay/relay_sys.c
@@ -0,0 +1,49 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.c
+ * @brief Subsystem definitions for the relay module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/onion_queue.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/relay_sys.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/router.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_relay_initialize(void)
+{
+  relay_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_relay_shutdown(void)
+{
+  dns_free_all();
+  ext_orport_free_all();
+  clear_pending_onions();
+  routerkeys_free_all();
+  router_free_all();
+}
+
+const struct subsys_fns_t sys_relay = {
+  .name = "relay",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = RELAY_SUBSYS_LEVEL,
+  .initialize = subsys_relay_initialize,
+  .shutdown = subsys_relay_shutdown,
+};
diff --git a/src/feature/relay/relay_sys.h b/src/feature/relay/relay_sys.h
new file mode 100644
index 0000000000..9bad93a6c9
--- /dev/null
+++ b/src/feature/relay/relay_sys.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.h
+ * @brief Header for feature/relay/relay_sys.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_SYS_H
+#define TOR_FEATURE_RELAY_RELAY_SYS_H
+
+extern const struct subsys_fns_t sys_relay;
+
+/**
+ * Subsystem level for the relay system.
+ *
+ * Defined here so that it can be shared between the real and stub
+ * definitions.
+ **/
+#define RELAY_SUBSYS_LEVEL 50
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_SYS_H) */
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index 283f7c326b..75a3d2f35c 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -1,13 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ROUTER_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "app/config/statefile.h"
 #include "app/main/main.h"
 #include "core/mainloop/connection.h"
@@ -16,7 +17,7 @@
 #include "core/or/policies.h"
 #include "core/or/protover.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -30,10 +31,13 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
 #include "feature/relay/dns.h"
+#include "feature/relay/relay_config.h"
+#include "feature/relay/relay_find_addr.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
 #include "feature/relay/routermode.h"
@@ -49,6 +53,7 @@
 #include "lib/encoding/confline.h"
 #include "lib/osinfo/uname.h"
 #include "lib/tls/tortls.h"
+#include "lib/version/torversion.h"
 
 #include "feature/dirauth/authmode.h"
 
@@ -58,6 +63,7 @@
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
@@ -149,6 +155,8 @@ routerinfo_err_to_string(int err)
       return "Cannot generate descriptor";
     case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
       return "Descriptor still rebuilding - not ready yet";
+    case TOR_ROUTERINFO_ERROR_INTERNAL_BUG:
+      return "Internal bug, see logs for details";
   }
 
   log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
@@ -191,8 +199,8 @@ set_onion_key(crypto_pk_t *k)
 
 /** Return the current onion key.  Requires that the onion key has been
  * loaded or generated. */
-crypto_pk_t *
-get_onion_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_onion_key,(void))
 {
   tor_assert(onionkey);
   return onionkey;
@@ -239,7 +247,7 @@ expire_old_onion_keys(void)
     lastonionkey = NULL;
   }
 
-  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+  /* We zero out the keypair. See the fast_mem_is_zero() check made in
    * construct_ntor_key_map() below. */
   memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
@@ -266,11 +274,12 @@ expire_old_onion_keys(void)
 
 /** Return the current secret onion key for the ntor handshake. Must only
  * be called from the main thread. */
-static const curve25519_keypair_t *
-get_current_curve25519_keypair(void)
+MOCK_IMPL(STATIC const struct curve25519_keypair_t *,
+get_current_curve25519_keypair,(void))
 {
   return &curve25519_onion_key;
 }
+
 /** Return a map from KEYID (the key itself) to keypairs for use in the ntor
  * handshake. Must only be called from the main thread. */
 di_digest256_map_t *
@@ -281,12 +290,12 @@ construct_ntor_key_map(void)
   const uint8_t *cur_pk = curve25519_onion_key.pubkey.public_key;
   const uint8_t *last_pk = last_curve25519_onion_key.pubkey.public_key;
 
-  if (!tor_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
+  if (!fast_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, cur_pk,
                     tor_memdup(&curve25519_onion_key,
                                sizeof(curve25519_keypair_t)));
   }
-  if (!tor_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
+  if (!fast_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
       tor_memneq(cur_pk, last_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, last_pk,
                     tor_memdup(&last_curve25519_onion_key,
@@ -337,6 +346,16 @@ set_server_identity_key(crypto_pk_t *k)
   }
 }
 
+#ifdef TOR_UNIT_TESTS
+/** Testing only -- set the server's RSA identity digest to
+ * be <b>digest</b> */
+void
+set_server_identity_key_digest_testing(const uint8_t *digest)
+{
+  memcpy(server_identitykey_digest, digest, DIGEST_LEN);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Make sure that we have set up our identity keys to match or not match as
  * appropriate, and die with an assertion if we have not. */
 static void
@@ -356,11 +375,13 @@ assert_identity_keys_ok(void)
   }
 }
 
+#ifdef HAVE_MODULE_RELAY
+
 /** Returns the current server identity key; requires that the key has
  * been set, and that we are running as a Tor server.
  */
-crypto_pk_t *
-get_server_identity_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_server_identity_key,(void))
 {
   tor_assert(server_identitykey);
   tor_assert(server_mode(get_options()));
@@ -368,6 +389,8 @@ get_server_identity_key(void)
   return server_identitykey;
 }
 
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 /** Return true iff we are a server and the server identity key
  * has been set. */
 int
@@ -634,7 +657,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out,
                fname);
     goto done;
   }
-  parsed = authority_cert_parse_from_string(cert, &eos);
+  parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos);
   if (!parsed) {
     log_warn(LD_DIR, "Unable to parse certificate in %s", fname);
     goto done;
@@ -727,8 +750,8 @@ v3_authority_check_key_expiry(void)
 }
 
 /** Get the lifetime of an onion key in days. This value is defined by the
- * network consesus parameter "onion-key-rotation-days". Always returns a value
- * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
+ * network consensus parameter "onion-key-rotation-days". Always returns a
+ * value between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
  * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>.
  */
 static int
@@ -742,7 +765,7 @@ get_onion_key_rotation_days_(void)
 }
 
 /** Get the current lifetime of an onion key in seconds. This value is defined
- * by the network consesus parameter "onion-key-rotation-days", but the value
+ * by the network consensus parameter "onion-key-rotation-days", but the value
  * is converted to seconds.
  */
 int
@@ -752,7 +775,7 @@ get_onion_key_lifetime(void)
 }
 
 /** Get the grace period of an onion key in seconds. This value is defined by
- * the network consesus parameter "onion-key-grace-period-days", but the value
+ * the network consensus parameter "onion-key-grace-period-days", but the value
  * is converted to seconds.
  */
 int
@@ -866,15 +889,6 @@ init_keys_common(void)
   if (!key_lock)
     key_lock = tor_mutex_new();
 
-  /* There are a couple of paths that put us here before we've asked
-   * openssl to initialize itself. */
-  if (crypto_global_init(get_options()->HardwareAccel,
-                         get_options()->AccelName,
-                         get_options()->AccelDir)) {
-    log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
-    return -1;
-  }
-
   return 0;
 }
 
@@ -1031,7 +1045,7 @@ init_keys(void)
       return -1;
 
     keydir = get_keydir_fname("secret_onion_key_ntor.old");
-    if (tor_mem_is_zero((const char *)
+    if (fast_mem_is_zero((const char *)
                            last_curve25519_onion_key.pubkey.public_key,
                         CURVE25519_PUBKEY_LEN) &&
         file_status(keydir) == FN_FILE) {
@@ -1062,8 +1076,10 @@ init_keys(void)
   if (authdir_mode_v3(options)) {
     const char *m = NULL;
     routerinfo_t *ri;
-    /* We need to add our own fingerprint so it gets recognized. */
-    if (dirserv_add_own_fingerprint(get_server_identity_key())) {
+    /* We need to add our own fingerprint and ed25519 key so it gets
+     * recognized. */
+    if (dirserv_add_own_fingerprint(get_server_identity_key(),
+                                    get_master_identity_key())) {
       log_err(LD_GENERAL,"Error adding own fingerprint to set of relays");
       return -1;
     }
@@ -1202,7 +1218,7 @@ router_should_be_dirserver(const or_options_t *options, int dir_port)
      * much larger effect on output than input so there is no reason to turn it
      * off if using AccountingRule in. */
     int interval_length = accounting_get_interval_length();
-    uint32_t effective_bw = get_effective_bwrate(options);
+    uint32_t effective_bw = relay_get_effective_bwrate(options);
     uint64_t acc_bytes;
     if (!interval_length) {
       log_warn(LD_BUG, "An accounting interval is not allowed to be zero "
@@ -1384,7 +1400,7 @@ consider_publishable_server(int force)
 }
 
 /** Return the port of the first active listener of type
- *  <b>listener_type</b>. */
+ *  <b>listener_type</b>. Returns 0 if no port is found. */
 /** XXX not a very good interface. it's not reliable when there are
     multiple listeners. */
 uint16_t
@@ -1406,8 +1422,7 @@ router_get_active_listener_port_by_type_af(int listener_type,
 
 /** Return the port that we should advertise as our ORPort; this is either
  * the one configured in the ORPort option, or the one we actually bound to
- * if ORPort is "auto".
- */
+ * if ORPort is "auto". Returns 0 if no port is found. */
 uint16_t
 router_get_advertised_or_port(const or_options_t *options)
 {
@@ -1456,7 +1471,7 @@ router_get_advertised_ipv6_or_ap(const or_options_t *options,
                                                       AF_INET6);
 
   if (!addr || port == 0) {
-    log_info(LD_CONFIG, "There is no advertised IPv6 ORPort.");
+    log_debug(LD_CONFIG, "There is no advertised IPv6 ORPort.");
     return;
   }
 
@@ -1477,6 +1492,24 @@ router_get_advertised_ipv6_or_ap(const or_options_t *options,
   ipv6_ap_out->port = port;
 }
 
+/** Returns true if this router has an advertised IPv6 ORPort. */
+bool
+router_has_advertised_ipv6_orport(const or_options_t *options)
+{
+  tor_addr_port_t ipv6_ap;
+  router_get_advertised_ipv6_or_ap(options, &ipv6_ap);
+  return tor_addr_port_is_valid_ap(&ipv6_ap, 0);
+}
+
+/** Returns true if this router has an advertised IPv6 ORPort. */
+MOCK_IMPL(bool,
+router_can_extend_over_ipv6,(const or_options_t *options))
+{
+  /* We might add some extra checks here, such as ExtendAllowIPv6Addresses
+  * from ticket 33818. */
+  return router_has_advertised_ipv6_orport(options);
+}
+
 /** Return the port that we should advertise as our DirPort;
  * this is one of three possibilities:
  * The one that is passed as <b>dirport</b> if the DirPort option is 0, or
@@ -1511,9 +1544,9 @@ static extrainfo_t *desc_extrainfo = NULL;
 static const char *desc_gen_reason = "uninitialized reason";
 /** Since when has our descriptor been "clean"?  0 if we need to regenerate it
  * now. */
-static time_t desc_clean_since = 0;
+STATIC time_t desc_clean_since = 0;
 /** Why did we mark the descriptor dirty? */
-static const char *desc_dirty_reason = "Tor just started";
+STATIC const char *desc_dirty_reason = "Tor just started";
 /** Boolean: do we need to regenerate the above? */
 static int desc_needs_upload = 0;
 
@@ -1732,45 +1765,6 @@ router_get_descriptor_gen_reason(void)
   return desc_gen_reason;
 }
 
-/** A list of nicknames that we've warned about including in our family
- * declaration verbatim rather than as digests. */
-static smartlist_t *warned_nonexistent_family = NULL;
-
-static int router_guess_address_from_dir_headers(uint32_t *guess);
-
-/** Make a current best guess at our address, either because
- * it's configured in torrc, or because we've learned it from
- * dirserver headers. Place the answer in *<b>addr</b> and return
- * 0 on success, else return -1 if we have no guess.
- *
- * If <b>cache_only</b> is true, just return any cached answers, and
- * don't try to get any new answers.
- */
-MOCK_IMPL(int,
-router_pick_published_address,(const or_options_t *options, uint32_t *addr,
-                               int cache_only))
-{
-  /* First, check the cached output from resolve_my_address(). */
-  *addr = get_last_resolved_addr();
-  if (*addr)
-    return 0;
-
-  /* Second, consider doing a resolve attempt right here. */
-  if (!cache_only) {
-    if (resolve_my_address(LOG_INFO, options, addr, NULL, NULL) >= 0) {
-      log_info(LD_CONFIG,"Success: chose address '%s'.", fmt_addr32(*addr));
-      return 0;
-    }
-  }
-
-  /* Third, check the cached output from router_new_address_suggestion(). */
-  if (router_guess_address_from_dir_headers(addr) >= 0)
-    return 0;
-
-  /* We have no useful cached answers. Return failure. */
-  return -1;
-}
-
 /* Like router_check_descriptor_address_consistency, but specifically for the
  * ORPort or DirPort.
  * listener_type is either CONN_TYPE_OR_LISTENER or CONN_TYPE_DIR_LISTENER. */
@@ -1848,26 +1842,159 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr)
                                                    CONN_TYPE_DIR_LISTENER);
 }
 
-/** Build a fresh routerinfo, signed server descriptor, and extra-info document
- * for this OR. Set r to the generated routerinfo, e to the generated
- * extra-info document. Return 0 on success, -1 on temporary error. Failure to
- * generate an extra-info document is not an error and is indicated by setting
- * e to NULL. Caller is responsible for freeing generated documents if 0 is
- * returned.
+/** A list of nicknames that we've warned about including in our family,
+ * for one reason or another. */
+static smartlist_t *warned_family = NULL;
+
+/**
+ * Return a new smartlist containing the family members configured in
+ * <b>options</b>.  Warn about invalid or missing entries.  Return NULL
+ * if this relay should not declare a family.
+ **/
+STATIC smartlist_t *
+get_my_declared_family(const or_options_t *options)
+{
+  if (!options->MyFamily)
+    return NULL;
+
+  if (options->BridgeRelay)
+    return NULL;
+
+  if (!warned_family)
+    warned_family = smartlist_new();
+
+  smartlist_t *declared_family = smartlist_new();
+  config_line_t *family;
+
+  /* First we try to get the whole family in the form of hexdigests. */
+  for (family = options->MyFamily; family; family = family->next) {
+    char *name = family->value;
+    const node_t *member;
+    if (options->Nickname && !strcasecmp(name, options->Nickname))
+      continue; /* Don't list ourself by nickname, that's redundant */
+    else
+      member = node_get_by_nickname(name, 0);
+
+    if (!member) {
+      /* This node doesn't seem to exist, so warn about it if it is not
+       * a hexdigest. */
+      int is_legal = is_legal_nickname_or_hexdigest(name);
+      if (!smartlist_contains_string(warned_family, name) &&
+          !is_legal_hexdigest(name)) {
+        if (is_legal)
+          log_warn(LD_CONFIG,
+                   "There is a router named %s in my declared family, but "
+                   "I have no descriptor for it. I'll use the nickname "
+                   "as is, but this may confuse clients. Please list it "
+                   "by identity digest instead.", escaped(name));
+        else
+          log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                   "family, but that isn't a legal digest or nickname. "
+                   "Skipping it.", escaped(name));
+        smartlist_add_strdup(warned_family, name);
+      }
+      if (is_legal) {
+        smartlist_add_strdup(declared_family, name);
+      }
+    } else {
+      /* List the node by digest. */
+      char *fp = tor_malloc(HEX_DIGEST_LEN+2);
+      fp[0] = '$';
+      base16_encode(fp+1,HEX_DIGEST_LEN+1,
+                    member->identity, DIGEST_LEN);
+      smartlist_add(declared_family, fp);
+
+      if (! is_legal_hexdigest(name) &&
+          !smartlist_contains_string(warned_family, name)) {
+        /* Warn if this node was not specified by hexdigest. */
+        log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                 "family, but it wasn't listed by digest. Please consider "
+                 "saying %s instead, if that's what you meant.",
+                 escaped(name), fp);
+        smartlist_add_strdup(warned_family, name);
+      }
+    }
+  }
+
+  /* Now declared_family should have the closest we can come to the
+   * identities that the user wanted.
+   *
+   * Unlike older versions of Tor, we _do_ include our own identity: this
+   * helps microdescriptor compression, and helps in-memory compression
+   * on clients. */
+  nodefamily_t *nf = nodefamily_from_members(declared_family,
+                                     router_get_my_id_digest(),
+                                     NF_WARN_MALFORMED,
+                                     NULL);
+  SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s));
+  smartlist_free(declared_family);
+  if (!nf) {
+    return NULL;
+  }
+
+  char *s = nodefamily_format(nf);
+  nodefamily_free(nf);
+
+  smartlist_t *result = smartlist_new();
+  smartlist_split_string(result, s, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  tor_free(s);
+
+  if (smartlist_len(result) == 1) {
+    /* This is a one-element list containing only ourself; instead return
+     * nothing */
+    const char *singleton = smartlist_get(result, 0);
+    bool is_me = false;
+    if (singleton[0] == '$') {
+      char d[DIGEST_LEN];
+      int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1));
+      if (n == DIGEST_LEN &&
+          fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) {
+        is_me = true;
+      }
+    }
+    if (!is_me) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Found a singleton family list with an element "
+               "that wasn't us! Element was %s", escaped(singleton));
+      // LCOV_EXCL_STOP
+    } else {
+      SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+      smartlist_free(result);
+      return NULL;
+    }
+  }
+
+  return result;
+}
+
+/** Allocate a fresh, unsigned routerinfo for this OR, without any of the
+ * fields that depend on the corresponding extrainfo.
+ *
+ * On success, set ri_out to the new routerinfo, and return 0.
+ * Caller is responsible for freeing the generated routerinfo.
+ *
+ * Returns a negative value and sets ri_out to NULL on temporary error.
  */
-int
-router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+MOCK_IMPL(STATIC int,
+router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out))
 {
-  routerinfo_t *ri;
-  extrainfo_t *ei;
+  routerinfo_t *ri = NULL;
   uint32_t addr;
   char platform[256];
   int hibernating = we_are_hibernating();
   const or_options_t *options = get_options();
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
+  if (BUG(!ri_out)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
 
   if (router_pick_published_address(options, &addr, 0) < 0) {
     log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
-    return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    result = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    goto err;
   }
 
   /* Log a message if the address in the descriptor doesn't match the ORPort
@@ -1894,15 +2021,15 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   /* For now, at most one IPv6 or-address is being advertised. */
   tor_addr_port_t ipv6_orport;
   router_get_advertised_ipv6_or_ap(options, &ipv6_orport);
-  /* If there is no valud IPv6 ORPort, the address and port are null. */
+  /* If there is no valid IPv6 ORPort, the address and port are null. */
   tor_addr_copy(&ri->ipv6_addr, &ipv6_orport.addr);
   ri->ipv6_orport = ipv6_orport.port;
 
   ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key());
   if (BUG(crypto_pk_get_digest(ri->identity_pkey,
                            ri->cache_info.identity_digest) < 0)) {
-    routerinfo_free(ri);
-    return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    result = TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    goto err;
   }
   ri->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
@@ -1913,10 +2040,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   ri->protocol_list = tor_strdup(protover_get_supported_protocols());
 
   /* compute ri->bandwidthrate as the min of various options */
-  ri->bandwidthrate = get_effective_bwrate(options);
+  ri->bandwidthrate = relay_get_effective_bwrate(options);
 
   /* and compute ri->bandwidthburst similarly */
-  ri->bandwidthburst = get_effective_bwburst(options);
+  ri->bandwidthburst = relay_get_effective_bwburst(options);
 
   /* Report bandwidth, unless we're hibernating or shutting down */
   ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess();
@@ -1939,134 +2066,260 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
     tor_free(p_tmp);
   }
 
-  if (options->MyFamily && ! options->BridgeRelay) {
-    if (!warned_nonexistent_family)
-      warned_nonexistent_family = smartlist_new();
-    ri->declared_family = smartlist_new();
-    config_line_t *family;
-    for (family = options->MyFamily; family; family = family->next) {
-       char *name = family->value;
-       const node_t *member;
-       if (!strcasecmp(name, options->Nickname))
-         continue; /* Don't list ourself, that's redundant */
-       else
-         member = node_get_by_nickname(name, 0);
-       if (!member) {
-         int is_legal = is_legal_nickname_or_hexdigest(name);
-         if (!smartlist_contains_string(warned_nonexistent_family, name) &&
-             !is_legal_hexdigest(name)) {
-           if (is_legal)
-             log_warn(LD_CONFIG,
-                      "I have no descriptor for the router named \"%s\" in my "
-                      "declared family; I'll use the nickname as is, but "
-                      "this may confuse clients.", name);
-           else
-             log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
-                      "declared family, but that isn't a legal nickname. "
-                      "Skipping it.", escaped(name));
-           smartlist_add_strdup(warned_nonexistent_family, name);
-         }
-         if (is_legal) {
-           smartlist_add_strdup(ri->declared_family, name);
-         }
-       } else if (router_digest_is_me(member->identity)) {
-         /* Don't list ourself in our own family; that's redundant */
-         /* XXX shouldn't be possible */
-       } else {
-         char *fp = tor_malloc(HEX_DIGEST_LEN+2);
-         fp[0] = '$';
-         base16_encode(fp+1,HEX_DIGEST_LEN+1,
-                       member->identity, DIGEST_LEN);
-         smartlist_add(ri->declared_family, fp);
-         if (smartlist_contains_string(warned_nonexistent_family, name))
-           smartlist_string_remove(warned_nonexistent_family, name);
-       }
-    }
+  ri->declared_family = get_my_declared_family(options);
 
-    /* remove duplicates from the list */
-    smartlist_sort_strings(ri->declared_family);
-    smartlist_uniq_strings(ri->declared_family);
+  if (options->BridgeRelay) {
+    ri->purpose = ROUTER_PURPOSE_BRIDGE;
+    /* Bridges shouldn't be able to send their descriptors unencrypted,
+     anyway, since they don't have a DirPort, and always connect to the
+     bridge authority anonymously.  But just in case they somehow think of
+     sending them on an unencrypted connection, don't allow them to try. */
+    ri->cache_info.send_unencrypted = 0;
+  } else {
+    ri->purpose = ROUTER_PURPOSE_GENERAL;
+    ri->cache_info.send_unencrypted = 1;
   }
 
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  *ri_out = NULL;
+  return result;
+
+ done:
+  *ri_out = ri;
+  return 0;
+}
+
+/** Allocate and return a fresh, unsigned extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * Uses options->Nickname to set the nickname, and options->BridgeRelay to set
+ * ei->cache_info.send_unencrypted.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+static extrainfo_t *
+router_build_fresh_unsigned_extrainfo(const routerinfo_t *ri)
+{
+  extrainfo_t *ei = NULL;
+  const or_options_t *options = get_options();
+
+  if (BUG(!ri))
+    return NULL;
+
   /* Now generate the extrainfo. */
   ei = tor_malloc_zero(sizeof(extrainfo_t));
   ei->cache_info.is_extrainfo = 1;
-  strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname));
+  strlcpy(ei->nickname, options->Nickname, sizeof(ei->nickname));
   ei->cache_info.published_on = ri->cache_info.published_on;
   ei->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
 
   memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest,
          DIGEST_LEN);
+
+  if (options->BridgeRelay) {
+    /* See note in router_build_fresh_routerinfo(). */
+    ei->cache_info.send_unencrypted = 0;
+  } else {
+    ei->cache_info.send_unencrypted = 1;
+  }
+
+  return ei;
+}
+
+/** Dump the extrainfo descriptor body for ei, sign it, and add the body and
+ * signature to ei->cache_info. Note that the extrainfo body is determined by
+ * ei, and some additional config and statistics state: see
+ * extrainfo_dump_to_string() for details.
+ *
+ * Return 0 on success, -1 on temporary error.
+ * If ei is NULL, logs a BUG() warning and returns -1.
+ * On error, ei->cache_info is not modified.
+ */
+static int
+router_dump_and_sign_extrainfo_descriptor_body(extrainfo_t *ei)
+{
+  if (BUG(!ei))
+    return -1;
+
   if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body,
                                ei, get_server_identity_key(),
                                get_master_signing_keypair()) < 0) {
     log_warn(LD_BUG, "Couldn't generate extra-info descriptor.");
-    extrainfo_free(ei);
-    ei = NULL;
-  } else {
-    ei->cache_info.signed_descriptor_len =
-      strlen(ei->cache_info.signed_descriptor_body);
-    router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
-                              ei->cache_info.signed_descriptor_len,
-                              ei->cache_info.signed_descriptor_digest);
-    crypto_digest256((char*) ei->digest256,
-                     ei->cache_info.signed_descriptor_body,
-                     ei->cache_info.signed_descriptor_len,
-                     DIGEST_SHA256);
+    return -1;
   }
 
-  /* Now finish the router descriptor. */
-  if (ei) {
-    memcpy(ri->cache_info.extra_info_digest,
-           ei->cache_info.signed_descriptor_digest,
-           DIGEST_LEN);
-    memcpy(ri->cache_info.extra_info_digest256,
-           ei->digest256,
-           DIGEST256_LEN);
-  } else {
-    /* ri was allocated with tor_malloc_zero, so there is no need to
-     * zero ri->cache_info.extra_info_digest here. */
+  ei->cache_info.signed_descriptor_len =
+    strlen(ei->cache_info.signed_descriptor_body);
+
+  router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
+                            ei->cache_info.signed_descriptor_len,
+                            ei->cache_info.signed_descriptor_digest);
+  crypto_digest256((char*) ei->digest256,
+                   ei->cache_info.signed_descriptor_body,
+                   ei->cache_info.signed_descriptor_len,
+                   DIGEST_SHA256);
+
+  return 0;
+}
+
+/** Allocate and return a fresh, signed extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+STATIC extrainfo_t *
+router_build_fresh_signed_extrainfo(const routerinfo_t *ri)
+{
+  int result = -1;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!ri))
+    return NULL;
+
+  ei = router_build_fresh_unsigned_extrainfo(ri);
+  /* router_build_fresh_unsigned_extrainfo() should not fail. */
+  if (BUG(!ei))
+    goto err;
+
+  result = router_dump_and_sign_extrainfo_descriptor_body(ei);
+  if (result < 0)
+    goto err;
+
+  goto done;
+
+ err:
+  extrainfo_free(ei);
+  return NULL;
+
+ done:
+  return ei;
+}
+
+/** Set the fields in ri that depend on ei.
+ *
+ * If ei is NULL, logs a BUG() warning and zeroes the relevant fields.
+ */
+STATIC void
+router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                        const extrainfo_t *ei)
+{
+  if (BUG(!ei)) {
+    /* Just to be safe, zero ri->cache_info.extra_info_digest here. */
+    memset(ri->cache_info.extra_info_digest, 0, DIGEST_LEN);
+    memset(ri->cache_info.extra_info_digest256, 0, DIGEST256_LEN);
+    return;
   }
+
+  /* Now finish the router descriptor. */
+  memcpy(ri->cache_info.extra_info_digest,
+         ei->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  memcpy(ri->cache_info.extra_info_digest256,
+         ei->digest256,
+         DIGEST256_LEN);
+}
+
+/** Dump the descriptor body for ri, sign it, and add the body and signature to
+ * ri->cache_info. Note that the descriptor body is determined by ri, and some
+ * additional config and state: see router_dump_router_to_string() for details.
+ *
+ * Return 0 on success, and a negative value on temporary error.
+ * If ri is NULL, logs a BUG() warning and returns a negative value.
+ * On error, ri->cache_info is not modified.
+ */
+STATIC int
+router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri)
+{
+  if (BUG(!ri))
+    return TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
   if (! (ri->cache_info.signed_descriptor_body =
           router_dump_router_to_string(ri, get_server_identity_key(),
                                        get_onion_key(),
                                        get_current_curve25519_keypair(),
                                        get_master_signing_keypair())) ) {
     log_warn(LD_BUG, "Couldn't generate router descriptor.");
-    routerinfo_free(ri);
-    extrainfo_free(ei);
     return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
   }
+
   ri->cache_info.signed_descriptor_len =
     strlen(ri->cache_info.signed_descriptor_body);
 
-  ri->purpose =
-    options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-  if (options->BridgeRelay) {
-    /* Bridges shouldn't be able to send their descriptors unencrypted,
-       anyway, since they don't have a DirPort, and always connect to the
-       bridge authority anonymously.  But just in case they somehow think of
-       sending them on an unencrypted connection, don't allow them to try. */
-    ri->cache_info.send_unencrypted = 0;
-    if (ei)
-      ei->cache_info.send_unencrypted = 0;
-  } else {
-    ri->cache_info.send_unencrypted = 1;
-    if (ei)
-      ei->cache_info.send_unencrypted = 1;
-  }
-
   router_get_router_hash(ri->cache_info.signed_descriptor_body,
                          strlen(ri->cache_info.signed_descriptor_body),
                          ri->cache_info.signed_descriptor_digest);
 
+  return 0;
+}
+
+/** Build a fresh routerinfo, signed server descriptor, and signed extrainfo
+ * document for this OR.
+ *
+ * Set r to the generated routerinfo, e to the generated extrainfo document.
+ * Failure to generate an extra-info document is not an error and is indicated
+ * by setting e to NULL.
+ * Return 0 on success, and a negative value on temporary error.
+ * Caller is responsible for freeing generated documents on success.
+ */
+int
+router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+{
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+  routerinfo_t *ri = NULL;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!r))
+    goto err;
+
+  if (BUG(!e))
+    goto err;
+
+  result = router_build_fresh_unsigned_routerinfo(&ri);
+  if (result < 0) {
+    goto err;
+  }
+  /* If ri is NULL, then result should be negative. So this check should be
+   * unreachable. */
+  if (BUG(!ri)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
+
+  ei = router_build_fresh_signed_extrainfo(ri);
+
+  /* Failing to create an ei is not an error. */
   if (ei) {
-    tor_assert(!
-          routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
-                                                 &ri->cache_info, NULL));
+    router_update_routerinfo_from_extrainfo(ri, ei);
   }
 
+  result = router_dump_and_sign_routerinfo_descriptor_body(ri);
+  if (result < 0)
+    goto err;
+
+  if (ei) {
+     if (BUG(routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                    &ri->cache_info, NULL))) {
+       result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+       goto err;
+     }
+  }
+
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  extrainfo_free(ei);
+  *r = NULL;
+  *e = NULL;
+  return result;
+
+ done:
   *r = ri;
   *e = ei;
   return 0;
@@ -2152,7 +2405,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
   /* Now we see whether we want to be retrying frequently or no.  The
    * rule here is that we'll retry frequently if we aren't listed in the
    * live consensus we have, or if the publication time of the
-   * descriptor listed for us in the consensus is very old. */
+   * descriptor listed for us in the consensus is very old, or if the
+   * consensus lists us as "stale" and we haven't regenerated since the
+   * consensus was published. */
   ns = networkstatus_get_live_consensus(now);
   if (ns) {
     rs = networkstatus_vote_find_entry(ns, server_identitykey_digest);
@@ -2160,6 +2415,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
       retry_fast_reason = "not listed in consensus";
     else if (rs->published_on < slow_cutoff)
       retry_fast_reason = "version listed in consensus is quite old";
+    else if (rs->is_staledesc && ns->valid_after > desc_clean_since)
+      retry_fast_reason = "listed as stale in consensus";
   }
 
   if (retry_fast_reason && desc_clean_since < fast_cutoff)
@@ -2231,9 +2488,13 @@ check_descriptor_bandwidth_changed(time_t now)
   }
 }
 
+// This function can be "noreturn" if relay mode is disabled and
+// ALL_BUGS_ARE_FATAL is set.
+DISABLE_GCC_WARNING("-Wmissing-noreturn")
+
 /** Note at log level severity that our best guess of address has changed from
  * <b>prev</b> to <b>cur</b>. */
-static void
+void
 log_addr_has_changed(int severity,
                      const tor_addr_t *prev,
                      const tor_addr_t *cur,
@@ -2260,6 +2521,7 @@ log_addr_has_changed(int severity,
              "Guessed our IP address as %s (source: %s).",
              addrbuf_cur, source);
 }
+ENABLE_GCC_WARNING("-Wmissing-noreturn")
 
 /** Check whether our own address as defined by the Address configuration
  * has changed. This is for routers that get their address from a service
@@ -2305,86 +2567,6 @@ check_descriptor_ipaddress_changed(time_t now)
   tor_free(hostname);
 }
 
-/** The most recently guessed value of our IP address, based on directory
- * headers. */
-static tor_addr_t last_guessed_ip = TOR_ADDR_NULL;
-
-/** A directory server <b>d_conn</b> told us our IP address is
- * <b>suggestion</b>.
- * If this address is different from the one we think we are now, and
- * if our computer doesn't actually know its IP address, then switch. */
-void
-router_new_address_suggestion(const char *suggestion,
-                              const dir_connection_t *d_conn)
-{
-  tor_addr_t addr;
-  uint32_t cur = 0;             /* Current IPv4 address.  */
-  const or_options_t *options = get_options();
-
-  /* first, learn what the IP address actually is */
-  if (tor_addr_parse(&addr, suggestion) == -1) {
-    log_debug(LD_DIR, "Malformed X-Your-Address-Is header %s. Ignoring.",
-              escaped(suggestion));
-    return;
-  }
-
-  log_debug(LD_DIR, "Got X-Your-Address-Is: %s.", suggestion);
-
-  if (!server_mode(options)) {
-    tor_addr_copy(&last_guessed_ip, &addr);
-    return;
-  }
-
-  /* XXXX ipv6 */
-  cur = get_last_resolved_addr();
-  if (cur ||
-      resolve_my_address(LOG_INFO, options, &cur, NULL, NULL) >= 0) {
-    /* We're all set -- we already know our address. Great. */
-    tor_addr_from_ipv4h(&last_guessed_ip, cur); /* store it in case we
-                                                   need it later */
-    return;
-  }
-  if (tor_addr_is_internal(&addr, 0)) {
-    /* Don't believe anybody who says our IP is, say, 127.0.0.1. */
-    return;
-  }
-  if (tor_addr_eq(&d_conn->base_.addr, &addr)) {
-    /* Don't believe anybody who says our IP is their IP. */
-    log_debug(LD_DIR, "A directory server told us our IP address is %s, "
-              "but they are just reporting their own IP address. Ignoring.",
-              suggestion);
-    return;
-  }
-
-  /* Okay.  We can't resolve our own address, and X-Your-Address-Is is giving
-   * us an answer different from what we had the last time we managed to
-   * resolve it. */
-  if (!tor_addr_eq(&last_guessed_ip, &addr)) {
-    control_event_server_status(LOG_NOTICE,
-                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=DIRSERV",
-                                suggestion);
-    log_addr_has_changed(LOG_NOTICE, &last_guessed_ip, &addr,
-                         d_conn->base_.address);
-    ip_address_changed(0);
-    tor_addr_copy(&last_guessed_ip, &addr); /* router_rebuild_descriptor()
-                                               will fetch it */
-  }
-}
-
-/** We failed to resolve our address locally, but we'd like to build
- * a descriptor and publish / test reachability. If we have a guess
- * about our address based on directory headers, answer it and return
- * 0; else return -1. */
-static int
-router_guess_address_from_dir_headers(uint32_t *guess)
-{
-  if (!tor_addr_is_null(&last_guessed_ip)) {
-    *guess = tor_addr_to_ipv4h(&last_guessed_ip);
-    return 0;
-  }
-  return -1;
-}
-
 /** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short
  * string describing the version of Tor and the operating system we're
  * currently running on.
@@ -2405,6 +2587,10 @@ get_platform_str(char *platform, size_t len)
 /** OR only: Given a routerinfo for this router, and an identity key to sign
  * with, encode the routerinfo as a signed server descriptor and return a new
  * string encoding the result, or NULL on failure.
+ *
+ * In addition to the fields in router, this function calls
+ * onion_key_lifetime(), get_options(), and we_are_hibernating(), and uses the
+ * results to populate some fields in the descriptor.
  */
 char *
 router_dump_router_to_string(routerinfo_t *router,
@@ -2468,11 +2654,8 @@ router_dump_router_to_string(routerinfo_t *router,
       log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
       goto err;
     }
-    if (ed25519_public_to_base64(ed_fp_base64,
-                       &router->cache_info.signing_key_cert->signing_key)<0) {
-      log_err(LD_BUG,"Couldn't base64-encode identity key\n");
-      goto err;
-    }
+    ed25519_public_to_base64(ed_fp_base64,
+                            &router->cache_info.signing_key_cert->signing_key);
     tor_asprintf(&ed_cert_line, "identity-ed25519\n"
                  "-----BEGIN ED25519 CERT-----\n"
                  "%s"
@@ -2606,6 +2789,9 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   address = tor_dup_ip(router->addr);
+  if (!address)
+    goto err;
+
   chunks = smartlist_new();
 
   /* Generate the easy portion of the router descriptor. */
@@ -2722,8 +2908,7 @@ router_dump_router_to_string(routerinfo_t *router,
     if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN,
                      signing_keypair) < 0)
       goto err;
-    if (ed25519_signature_to_base64(buf, &sig) < 0)
-      goto err;
+    ed25519_signature_to_base64(buf, &sig);
 
     smartlist_add_asprintf(chunks, "%s\n", buf);
   }
@@ -2862,34 +3047,26 @@ load_stats_file(const char *filename, const char *end_line, time_t now,
   return r;
 }
 
-/** Write the contents of <b>extrainfo</b> and aggregated statistics to
- * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on
- * success, negative on failure. */
-int
-extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
-                         crypto_pk_t *ident_key,
-                         const ed25519_keypair_t *signing_keypair)
+/** Add header strings to chunks, based on the extrainfo object extrainfo,
+ * and ed25519 keypair signing_keypair, if emit_ed_sigs is true.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_header_helper(
+                                     smartlist_t *chunks,
+                                     const extrainfo_t *extrainfo,
+                                     const ed25519_keypair_t *signing_keypair,
+                                     int emit_ed_sigs)
 {
-  const or_options_t *options = get_options();
   char identity[HEX_DIGEST_LEN+1];
   char published[ISO_TIME_LEN+1];
-  char digest[DIGEST_LEN];
-  char *bandwidth_usage;
-  int result;
-  static int write_stats_to_extrainfo = 1;
-  char sig[DIROBJ_MAX_SIG_LEN+1];
-  char *s = NULL, *pre, *contents, *cp, *s_dup = NULL;
-  time_t now = time(NULL);
-  smartlist_t *chunks = smartlist_new();
-  extrainfo_t *ei_tmp = NULL;
-  const int emit_ed_sigs = signing_keypair &&
-    extrainfo->cache_info.signing_key_cert;
   char *ed_cert_line = NULL;
+  char *pre = NULL;
+  int rv = -1;
 
   base16_encode(identity, sizeof(identity),
                 extrainfo->cache_info.identity_digest, DIGEST_LEN);
   format_iso_time(published, extrainfo->cache_info.published_on);
-  bandwidth_usage = rep_hist_get_bandwidth_lines();
   if (emit_ed_sigs) {
     if (!extrainfo->cache_info.signing_key_cert->signing_key_included ||
         !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key,
@@ -2915,21 +3092,64 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     ed_cert_line = tor_strdup("");
   }
 
-  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s",
+  /* This is the first chunk in the file. If the file is too big, other chunks
+   * are removed. So we must only add one chunk here. */
+  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n",
                extrainfo->nickname, identity,
                ed_cert_line,
-               published, bandwidth_usage);
+               published);
   smartlist_add(chunks, pre);
 
-  if (geoip_is_loaded(AF_INET))
-    smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
-                           geoip_db_digest(AF_INET));
-  if (geoip_is_loaded(AF_INET6))
-    smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
-                           geoip_db_digest(AF_INET6));
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  tor_free(ed_cert_line);
+  return rv;
+}
+
+/** Add pluggable transport and statistics strings to chunks, skipping
+ * statistics if write_stats_to_extrainfo is false.
+ * Helper for extrainfo_dump_to_string().
+ * Can not fail. */
+static void
+extrainfo_dump_to_string_stats_helper(smartlist_t *chunks,
+                                      int write_stats_to_extrainfo)
+{
+  const or_options_t *options = get_options();
+  char *contents = NULL;
+  time_t now = time(NULL);
+
+  /* If the file is too big, these chunks are removed, starting with the last
+   * chunk. So each chunk must be a complete line, and the file must be valid
+   * after each chunk. */
+
+  /* Add information about the pluggable transports we support, even if we
+   * are not publishing statistics. This information is needed by BridgeDB
+   * to distribute bridges. */
+  if (options->ServerTransportPlugin) {
+    char *pluggable_transports = pt_get_extra_info_descriptor_string();
+    if (pluggable_transports)
+      smartlist_add(chunks, pluggable_transports);
+  }
 
   if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
     log_info(LD_GENERAL, "Adding stats to extra-info descriptor.");
+    /* Bandwidth usage stats don't have their own option */
+    {
+      contents = rep_hist_get_bandwidth_lines();
+      smartlist_add(chunks, contents);
+    }
+    /* geoip hashes aren't useful unless we are publishing other stats */
+    if (geoip_is_loaded(AF_INET))
+      smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
+                             geoip_db_digest(AF_INET));
+    if (geoip_is_loaded(AF_INET6))
+      smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
+                             geoip_db_digest(AF_INET6));
     if (options->DirReqStatistics &&
         load_stats_file("stats"PATH_SEPARATOR"dirreq-stats",
                         "dirreq-stats-end", now, &contents) > 0) {
@@ -2965,50 +3185,140 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
       if (contents)
         smartlist_add(chunks, contents);
     }
+    /* bridge statistics */
+    if (should_record_bridge_info(options)) {
+      const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
+      if (bridge_stats) {
+        smartlist_add_strdup(chunks, bridge_stats);
+      }
+    }
   }
+}
 
-  /* Add information about the pluggable transports we support. */
-  if (options->ServerTransportPlugin) {
-    char *pluggable_transports = pt_get_extra_info_descriptor_string();
-    if (pluggable_transports)
-      smartlist_add(chunks, pluggable_transports);
-  }
+/** Add an ed25519 signature of chunks to chunks, using the ed25519 keypair
+ * signing_keypair.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_ed_sig_helper(
+                                     smartlist_t *chunks,
+                                     const ed25519_keypair_t *signing_keypair)
+{
+  char sha256_digest[DIGEST256_LEN];
+  ed25519_signature_t ed_sig;
+  char buf[ED25519_SIG_BASE64_LEN+1];
+  int rv = -1;
+
+  /* These are two of the three final chunks in the file. If the file is too
+   * big, other chunks are removed. So we must only add two chunks here. */
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+  crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
+                                 ED_DESC_SIGNATURE_PREFIX,
+                                 chunks, "", DIGEST_SHA256);
+  if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
+                   signing_keypair) < 0)
+    goto err;
+  ed25519_signature_to_base64(buf, &ed_sig);
 
-  if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
-    const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
-    if (bridge_stats) {
-      smartlist_add_strdup(chunks, bridge_stats);
-    }
+  smartlist_add_asprintf(chunks, "%s\n", buf);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Add an RSA signature of extrainfo_string to chunks, using the RSA key
+ * ident_key.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_rsa_sig_helper(smartlist_t *chunks,
+                                        crypto_pk_t *ident_key,
+                                        const char *extrainfo_string)
+{
+  char sig[DIROBJ_MAX_SIG_LEN+1];
+  char digest[DIGEST_LEN];
+  int rv = -1;
+
+  memset(sig, 0, sizeof(sig));
+  if (router_get_extrainfo_hash(extrainfo_string, strlen(extrainfo_string),
+                                digest) < 0 ||
+      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
+                                     ident_key) < 0) {
+    log_warn(LD_BUG, "Could not append signature to extra-info "
+                     "descriptor.");
+    goto err;
   }
+  smartlist_add_strdup(chunks, sig);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Write the contents of <b>extrainfo</b>, to * *<b>s_out</b>, signing them
+ * with <b>ident_key</b>.
+ *
+ * If ExtraInfoStatistics is 1, also write aggregated statistics and related
+ * configuration data before signing. Most statistics also have an option that
+ * enables or disables that particular statistic.
+ *
+ * Always write pluggable transport lines.
+ *
+ * Return 0 on success, negative on failure. */
+int
+extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
+                         crypto_pk_t *ident_key,
+                         const ed25519_keypair_t *signing_keypair)
+{
+  int result;
+  static int write_stats_to_extrainfo = 1;
+  char *s = NULL, *cp, *s_dup = NULL;
+  smartlist_t *chunks = smartlist_new();
+  extrainfo_t *ei_tmp = NULL;
+  const int emit_ed_sigs = signing_keypair &&
+    extrainfo->cache_info.signing_key_cert;
+  int rv = 0;
+
+  rv = extrainfo_dump_to_string_header_helper(chunks, extrainfo,
+                                              signing_keypair,
+                                              emit_ed_sigs);
+  if (rv < 0)
+    goto err;
+
+  extrainfo_dump_to_string_stats_helper(chunks, write_stats_to_extrainfo);
 
   if (emit_ed_sigs) {
-    char sha256_digest[DIGEST256_LEN];
-    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
-    crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
-                                   ED_DESC_SIGNATURE_PREFIX,
-                                   chunks, "", DIGEST_SHA256);
-    ed25519_signature_t ed_sig;
-    char buf[ED25519_SIG_BASE64_LEN+1];
-    if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
-                     signing_keypair) < 0)
-      goto err;
-    if (ed25519_signature_to_base64(buf, &ed_sig) < 0)
+    rv = extrainfo_dump_to_string_ed_sig_helper(chunks, signing_keypair);
+    if (rv < 0)
       goto err;
-
-    smartlist_add_asprintf(chunks, "%s\n", buf);
   }
 
+  /* This is one of the three final chunks in the file. If the file is too big,
+   * other chunks are removed. So we must only add one chunk here. */
   smartlist_add_strdup(chunks, "router-signature\n");
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
   while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
     /* So long as there are at least two chunks (one for the initial
      * extra-info line and one for the router-signature), we can keep removing
-     * things. */
-    if (smartlist_len(chunks) > 2) {
-      /* We remove the next-to-last element (remember, len-1 is the last
-         element), since we need to keep the router-signature element. */
-      int idx = smartlist_len(chunks) - 2;
+     * things. If emit_ed_sigs is true, we also keep 2 additional chunks at the
+     * end for the ed25519 signature. */
+    const int required_chunks = emit_ed_sigs ? 4 : 2;
+    if (smartlist_len(chunks) > required_chunks) {
+      /* We remove the next-to-last or 4th-last element (remember, len-1 is the
+       * last element), since we need to keep the router-signature elements. */
+      int idx = smartlist_len(chunks) - required_chunks;
       char *e = smartlist_get(chunks, idx);
       smartlist_del_keeporder(chunks, idx);
       log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
@@ -3025,15 +3335,10 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     }
   }
 
-  memset(sig, 0, sizeof(sig));
-  if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 ||
-      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
-                                     ident_key) < 0) {
-    log_warn(LD_BUG, "Could not append signature to extra-info "
-                     "descriptor.");
+  rv = extrainfo_dump_to_string_rsa_sig_helper(chunks, ident_key, s);
+  if (rv < 0)
     goto err;
-  }
-  smartlist_add_strdup(chunks, sig);
+
   tor_free(s);
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -3069,9 +3374,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk));
   smartlist_free(chunks);
   tor_free(s_dup);
-  tor_free(ed_cert_line);
   extrainfo_free(ei_tmp);
-  tor_free(bandwidth_usage);
 
   return result;
 }
@@ -3081,9 +3384,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
 void
 router_reset_warnings(void)
 {
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_clear(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_clear(warned_family);
   }
 }
 
@@ -3096,6 +3399,10 @@ router_free_all(void)
   crypto_pk_free(server_identitykey);
   crypto_pk_free(client_identitykey);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
   tor_mutex_free(key_lock);
   routerinfo_free(desc_routerinfo);
   extrainfo_free(desc_extrainfo);
@@ -3107,11 +3414,12 @@ router_free_all(void)
   memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key));
   memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_free(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_free(warned_family);
   }
 }
+
 /* From the given RSA key object, convert it to ASN-1 encoded format and set
  * the newly allocated object in onion_pkey_out. The length of the key is set
  * in onion_pkey_len_out. */
diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h
index ab1f771017..50790a73dd 100644
--- a/src/feature/relay/router.h
+++ b/src/feature/relay/router.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,11 +23,18 @@ struct ed25519_keypair_t;
 #define TOR_ROUTERINFO_ERROR_DIGEST_FAILED   (-4)
 #define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5)
 #define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6)
+#define TOR_ROUTERINFO_ERROR_INTERNAL_BUG    (-7)
 
-crypto_pk_t *get_onion_key(void);
+MOCK_DECL(crypto_pk_t *,get_onion_key,(void));
 time_t get_onion_key_set_at(void);
 void set_server_identity_key(crypto_pk_t *k);
-crypto_pk_t *get_server_identity_key(void);
+/* Some compilers are clever enough to know that when relay mode is disabled,
+ * this function never returns. */
+#ifdef HAVE_MODULE_RELAY
+MOCK_DECL(crypto_pk_t *,get_server_identity_key,(void));
+#else
+#define get_server_identity_key() (tor_abort_(),NULL)
+#endif
 int server_identity_key_is_set(void);
 void set_client_identity_key(crypto_pk_t *k);
 crypto_pk_t *get_tlsclient_identity_key(void);
@@ -61,6 +68,8 @@ uint16_t router_get_active_listener_port_by_type_af(int listener_type,
 uint16_t router_get_advertised_or_port(const or_options_t *options);
 void router_get_advertised_ipv6_or_ap(const or_options_t *options,
                                       tor_addr_port_t *ipv6_ap_out);
+bool router_has_advertised_ipv6_orport(const or_options_t *options);
+MOCK_DECL(bool, router_can_extend_over_ipv6,(const or_options_t *options));
 uint16_t router_get_advertised_or_port_by_af(const or_options_t *options,
                                              sa_family_t family);
 uint16_t router_get_advertised_dir_port(const or_options_t *options,
@@ -78,8 +87,6 @@ void mark_my_descriptor_dirty(const char *reason);
 void check_descriptor_bandwidth_changed(time_t now);
 void check_descriptor_ipaddress_changed(time_t now);
 int router_has_bandwidth_to_be_dirserver(const or_options_t *options);
-void router_new_address_suggestion(const char *suggestion,
-                                   const dir_connection_t *d_conn);
 int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port);
 MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void));
 MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
@@ -91,9 +98,6 @@ int router_digest_is_me(const char *digest);
 const uint8_t *router_get_my_id_digest(void);
 int router_extrainfo_digest_is_me(const char *digest);
 int router_is_me(const routerinfo_t *router);
-MOCK_DECL(int,router_pick_published_address,(const or_options_t *options,
-                                             uint32_t *addr,
-                                             int cache_only));
 int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e);
 int router_rebuild_descriptor(int force);
 char *router_dump_router_to_string(routerinfo_t *router,
@@ -111,14 +115,34 @@ int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo,
 const char *routerinfo_err_to_string(int err);
 int routerinfo_err_is_transient(int err);
 
+void log_addr_has_changed(int severity, const tor_addr_t *prev,
+                          const tor_addr_t *cur, const char *source);
+
 void router_reset_warnings(void);
-void router_reset_reachability(void);
 void router_free_all(void);
 
 #ifdef ROUTER_PRIVATE
-/* Used only by router.c and test.c */
+/* Used only by router.c and the unit tests */
 STATIC void get_platform_str(char *platform, size_t len);
 STATIC int router_write_fingerprint(int hashed);
-#endif
+STATIC smartlist_t *get_my_declared_family(const or_options_t *options);
+
+#ifdef TOR_UNIT_TESTS
+extern time_t desc_clean_since;
+extern const char *desc_dirty_reason;
+void set_server_identity_key_digest_testing(const uint8_t *digest);
+MOCK_DECL(STATIC const struct curve25519_keypair_t *,
+                                       get_current_curve25519_keypair,(void));
+
+MOCK_DECL(STATIC int,
+              router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out));
+STATIC extrainfo_t *router_build_fresh_signed_extrainfo(
+                                                      const routerinfo_t *ri);
+STATIC void router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                                    const extrainfo_t *ei);
+STATIC int router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(ROUTER_PRIVATE) */
 
 #endif /* !defined(TOR_ROUTER_H) */
diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c
index f639fc91e7..d3de83cb86 100644
--- a/src/feature/relay/routerkeys.c
+++ b/src/feature/relay/routerkeys.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -226,7 +226,7 @@ load_ed_keys(const or_options_t *options, time_t now)
         tor_free(fname);
       }
     }
-    if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
+    if (safe_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
       sign_signing_key_with_id = NULL;
     else
       sign_signing_key_with_id = id;
@@ -631,14 +631,14 @@ get_master_identity_keypair(void)
 }
 #endif /* defined(TOR_UNIT_TESTS) */
 
-const ed25519_keypair_t *
-get_master_signing_keypair(void)
+MOCK_IMPL(const ed25519_keypair_t *,
+get_master_signing_keypair,(void))
 {
   return master_signing_key;
 }
 
-const struct tor_cert_st *
-get_master_signing_key_cert(void)
+MOCK_IMPL(const struct tor_cert_st *,
+get_master_signing_key_cert,(void))
 {
   return signing_key_cert;
 }
@@ -706,6 +706,8 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
 
   *len_out = 0;
   if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    log_info(LD_OR, "crypto_pk_get_digest failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
   }
   memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
@@ -713,8 +715,12 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
   int r = crypto_pk_private_sign(onion_key,
                                (char*)signature, sizeof(signature),
                                (const char*)signed_data, sizeof(signed_data));
-  if (r < 0)
+  if (r < 0) {
+    /* It's probably missing the private key */
+    log_info(LD_OR, "crypto_pk_private_sign failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
+  }
 
   *len_out = r;
 
diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h
index 0badd34191..1fb5d724e9 100644
--- a/src/feature/relay/routerkeys.h
+++ b/src/feature/relay/routerkeys.h
@@ -1,14 +1,21 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerkeys.h
+ * @brief Header for routerkeys.c
+ **/
+
 #ifndef TOR_ROUTERKEYS_H
 #define TOR_ROUTERKEYS_H
 
 #include "lib/crypt_ops/crypto_ed25519.h"
 
+#ifdef HAVE_MODULE_RELAY
+
 const ed25519_public_key_t *get_master_identity_key(void);
-const ed25519_keypair_t *get_master_signing_keypair(void);
-const struct tor_cert_st *get_master_signing_key_cert(void);
+MOCK_DECL(const ed25519_keypair_t *, get_master_signing_keypair,(void));
+MOCK_DECL(const struct tor_cert_st *, get_master_signing_key_cert,(void));
 
 const ed25519_keypair_t *get_current_auth_keypair(void);
 const struct tor_cert_st *get_current_link_cert_cert(void);
@@ -19,6 +26,7 @@ void get_master_rsa_crosscert(const uint8_t **cert_out,
 
 int router_ed25519_id_is_me(const ed25519_public_key_t *id);
 
+/* These are only used by router.c */
 struct tor_cert_st *make_ntor_onion_key_crosscert(
                                   const curve25519_keypair_t *onion_key,
                                   const ed25519_public_key_t *master_id_key,
@@ -37,6 +45,85 @@ int generate_ed_link_cert(const or_options_t *options, time_t now, int force);
 
 void routerkeys_free_all(void);
 
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define router_ed25519_id_is_me(id) \
+  ((void)(id), 0)
+
+static inline void *
+relay_key_is_unavailable_(void)
+{
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+#define relay_key_is_unavailable(type) \
+  ((type)(relay_key_is_unavailable_()))
+
+// Many of these can be removed once relay_handshake.c is relay-only.
+#define get_current_auth_keypair() \
+  relay_key_is_unavailable(const ed25519_keypair_t *)
+#define get_master_signing_keypair() \
+  relay_key_is_unavailable(const ed25519_keypair_t *)
+#define get_current_link_cert_cert() \
+  relay_key_is_unavailable(const struct tor_cert_st *)
+#define get_current_auth_key_cert() \
+  relay_key_is_unavailable(const struct tor_cert_st *)
+#define get_master_signing_key_cert() \
+  relay_key_is_unavailable(const struct tor_cert_st *)
+#define get_master_rsa_crosscert(cert_out, size_out) \
+  STMT_BEGIN                                         \
+  tor_assert_nonfatal_unreached();                   \
+  *(cert_out) = NULL;                                \
+  *(size_out) = 0;                                   \
+  STMT_END
+#define get_master_identity_key() \
+  relay_key_is_unavailable(const ed25519_public_key_t *)
+
+#define generate_ed_link_cert(options, now, force) \
+  ((void)(options), (void)(now), (void)(force), 0)
+#define should_make_new_ed_keys(options, now) \
+  ((void)(options), (void)(now), 0)
+
+// These can get removed once router.c becomes relay-only.
+static inline struct tor_cert_st *
+make_ntor_onion_key_crosscert(const curve25519_keypair_t *onion_key,
+                              const ed25519_public_key_t *master_id_key,
+                              time_t now, time_t lifetime,
+                              int *sign_out)
+{
+  (void)onion_key;
+  (void)master_id_key;
+  (void)now;
+  (void)lifetime;
+  *sign_out = 0;
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+static inline uint8_t *
+make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
+                             const ed25519_public_key_t *master_id_key,
+                             const crypto_pk_t *rsa_id_key,
+                             int *len_out)
+{
+  (void)onion_key;
+  (void)master_id_key;
+  (void)rsa_id_key;
+  *len_out = 0;
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+
+/* This calls is used outside of relay mode, but only to implement
+ * CMD_KEY_EXPIRATION */
+#define log_cert_expiration()                                           \
+  (puts("Not available: Tor has been compiled without relay support"), 0)
+/* This calls is used outside of relay mode, but only to implement
+ * CMD_KEYGEN. */
+#define load_ed_keys(x,y)                                                \
+  (puts("Not available: Tor has been compiled without relay support"), 0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #ifdef TOR_UNIT_TESTS
 const ed25519_keypair_t *get_master_identity_keypair(void);
 void init_mock_ed_keys(const crypto_pk_t *rsa_identity_key);
diff --git a/src/feature/relay/routermode.c b/src/feature/relay/routermode.c
index 2a9ddeac4d..c4d8792b5b 100644
--- a/src/feature/relay/routermode.c
+++ b/src/feature/relay/routermode.c
@@ -1,14 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routermode.c
+ * @brief Check if we're running as a relay/cache.
+ **/
+
 #include "core/or/or.h"
 
 #include "app/config/config.h"
-#include "core/mainloop/connection.h"
-#include "core/or/port_cfg_st.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
 
@@ -25,21 +28,6 @@ dir_server_mode(const or_options_t *options)
     (server_mode(options) && router_has_bandwidth_to_be_dirserver(options));
 }
 
-/** Return true iff we are trying to proxy client connections. */
-int
-proxy_mode(const or_options_t *options)
-{
-  (void)options;
-  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
-    if (p->type == CONN_TYPE_AP_LISTENER ||
-        p->type == CONN_TYPE_AP_TRANS_LISTENER ||
-        p->type == CONN_TYPE_AP_DNS_LISTENER ||
-        p->type == CONN_TYPE_AP_NATD_LISTENER)
-      return 1;
-  } SMARTLIST_FOREACH_END(p);
-  return 0;
-}
-
 /** Return true iff we are trying to be a server.
  */
 MOCK_IMPL(int,
diff --git a/src/feature/relay/routermode.h b/src/feature/relay/routermode.h
index be535af478..6d7404968d 100644
--- a/src/feature/relay/routermode.h
+++ b/src/feature/relay/routermode.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,13 +12,31 @@
 #ifndef TOR_ROUTERMODE_H
 #define TOR_ROUTERMODE_H
 
+#ifdef HAVE_MODULE_RELAY
+
 int dir_server_mode(const or_options_t *options);
 
 MOCK_DECL(int, server_mode, (const or_options_t *options));
 MOCK_DECL(int, public_server_mode, (const or_options_t *options));
 MOCK_DECL(int, advertised_server_mode, (void));
-int proxy_mode(const or_options_t *options);
 
 void set_server_advertised(int s);
 
+/** Is the relay module enabled? */
+#define have_module_relay() (1)
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define dir_server_mode(options) (((void)(options)),0)
+#define server_mode(options) (((void)(options)),0)
+#define public_server_mode(options) (((void)(options)),0)
+#define advertised_server_mode() (0)
+
+/* We shouldn't be publishing descriptors when relay mode is disabled. */
+#define set_server_advertised(s) tor_assert_nonfatal(!(s))
+
+#define have_module_relay() (0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #endif /* !defined(TOR_ROUTERMODE_H) */
diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c
index 064eea6c46..18fe25b989 100644
--- a/src/feature/relay/selftest.c
+++ b/src/feature/relay/selftest.c
@@ -1,19 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file selftest.c
  * \brief Relay self-testing
  *
- * Relays need to make sure that their own ports are reasonable, and estimate
+ * Relays need to make sure that their own ports are reachable, and estimate
  * their own bandwidth, before publishing.
  */
 
-#define SELFTEST_PRIVATE
-
 #include "core/or/or.h"
 
 #include "app/config/config.h"
@@ -26,7 +24,7 @@
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/authority_cert_st.h"
@@ -35,6 +33,7 @@
 #include "feature/nodelist/routerlist.h" // but...
 #include "feature/nodelist/routerset.h"
 #include "feature/nodelist/torcert.h"
+#include "feature/relay/relay_periodic.h"
 #include "feature/relay/router.h"
 #include "feature/relay/selftest.h"
 
@@ -214,6 +213,44 @@ router_do_reachability_checks(int test_or, int test_dir)
   }
 }
 
+/** We've decided to start our reachability testing. If all
+ * is set, log this to the user. Return 1 if we did, or 0 if
+ * we chose not to log anything. */
+int
+inform_testing_reachability(void)
+{
+  char dirbuf[128];
+  char *address;
+  const routerinfo_t *me = router_get_my_routerinfo();
+  if (!me)
+    return 0;
+
+  address = tor_dup_ip(me->addr);
+  if (!address)
+    return 0;
+
+  control_event_server_status(LOG_NOTICE,
+                              "CHECKING_REACHABILITY ORADDRESS=%s:%d",
+                              address, me->or_port);
+  if (me->dir_port) {
+    tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d",
+                 address, me->dir_port);
+    control_event_server_status(LOG_NOTICE,
+                                "CHECKING_REACHABILITY DIRADDRESS=%s:%d",
+                                address, me->dir_port);
+  }
+  log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... "
+                         "(this may take up to %d minutes -- look for log "
+                         "messages indicating success)",
+      address, me->or_port,
+      me->dir_port ? dirbuf : "",
+      me->dir_port ? "are" : "is",
+      TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60);
+
+  tor_free(address);
+  return 1;
+}
+
 /** Annotate that we found our ORPort reachable. */
 void
 router_orport_found_reachable(void)
@@ -222,6 +259,10 @@ router_orport_found_reachable(void)
   const or_options_t *options = get_options();
   if (!can_reach_or_port && me) {
     char *address = tor_dup_ip(me->addr);
+
+    if (!address)
+      return;
+
     log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from "
                "the outside. Excellent.%s",
                options->PublishServerDescriptor_ != NO_DIRINFO
@@ -249,6 +290,10 @@ router_dirport_found_reachable(void)
   const or_options_t *options = get_options();
   if (!can_reach_dir_port && me) {
     char *address = tor_dup_ip(me->addr);
+
+    if (!address)
+      return;
+
     log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable "
                "from the outside. Excellent.%s",
                options->PublishServerDescriptor_ != NO_DIRINFO
diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h
index a80ec8936e..f5babc95da 100644
--- a/src/feature/relay/selftest.h
+++ b/src/feature/relay/selftest.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,13 +12,57 @@
 #ifndef TOR_SELFTEST_H
 #define TOR_SELFTEST_H
 
+#ifdef HAVE_MODULE_RELAY
+
 struct or_options_t;
 int check_whether_orport_reachable(const struct or_options_t *options);
 int check_whether_dirport_reachable(const struct or_options_t *options);
 
 void router_do_reachability_checks(int test_or, int test_dir);
+void router_perform_bandwidth_test(int num_circs, time_t now);
+int inform_testing_reachability(void);
+
 void router_orport_found_reachable(void);
 void router_dirport_found_reachable(void);
-void router_perform_bandwidth_test(int num_circs, time_t now);
 
-#endif
+void router_reset_reachability(void);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define check_whether_orport_reachable(opts) \
+  ((void)(opts), 0)
+#define check_whether_dirport_reachable(opts) \
+  ((void)(opts), 0)
+
+static inline void
+router_do_reachability_checks(int test_or, int test_dir)
+{
+  (void)test_or;
+  (void)test_dir;
+  tor_assert_nonfatal_unreached();
+}
+static inline void
+router_perform_bandwidth_test(int num_circs, time_t now)
+{
+  (void)num_circs;
+  (void)now;
+  tor_assert_nonfatal_unreached();
+}
+static inline int
+inform_testing_reachability(void)
+{
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+#define router_orport_found_reachable() \
+  STMT_NIL
+#define router_dirport_found_reachable() \
+  STMT_NIL
+
+#define router_reset_reachability() \
+  STMT_NIL
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_SELFTEST_H) */
diff --git a/src/feature/relay/transport_config.c b/src/feature/relay/transport_config.c
new file mode 100644
index 0000000000..7dcce70e30
--- /dev/null
+++ b/src/feature/relay/transport_config.c
@@ -0,0 +1,307 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file transport_config.c
+ * @brief Code to interpret the user's configuration of Tor's server
+ *        pluggable transports.
+ **/
+
+#include "orconfig.h"
+#define RELAY_TRANSPORT_CONFIG_PRIVATE
+#include "feature/relay/transport_config.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/encoding/keyval.h"
+
+#include "lib/container/smartlist.h"
+
+/* Required for dirinfo_type_t in or_options_t */
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/routermode.h"
+
+/* Copied from config.c, we will refactor later in 29211. */
+#define REJECT(arg) \
+  STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
+
+/** Given a ServerTransportListenAddr <b>line</b>, return its
+ *  <address:port> string. Return NULL if the line was not
+ *  well-formed.
+ *
+ *  If <b>transport</b> is set, return NULL if the line is not
+ *  referring to <b>transport</b>.
+ *
+ *  The returned string is allocated on the heap and it's the
+ *  responsibility of the caller to free it. */
+static char *
+get_bindaddr_from_transport_listen_line(const char *line,
+                                        const char *transport)
+{
+  smartlist_t *items = NULL;
+  const char *parsed_transport = NULL;
+  char *addrport = NULL;
+  tor_addr_t addr;
+  uint16_t port = 0;
+
+  items = smartlist_new();
+  smartlist_split_string(items, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+
+  if (smartlist_len(items) < 2) {
+    log_warn(LD_CONFIG,"Too few arguments on ServerTransportListenAddr line.");
+    goto err;
+  }
+
+  parsed_transport = smartlist_get(items, 0);
+  addrport = tor_strdup(smartlist_get(items, 1));
+
+  /* If 'transport' is given, check if it matches the one on the line */
+  if (transport && strcmp(transport, parsed_transport))
+    goto err;
+
+  /* Validate addrport */
+  if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port, -1)<0) {
+    log_warn(LD_CONFIG, "Error parsing ServerTransportListenAddr "
+             "address '%s'", addrport);
+    goto err;
+  }
+
+  goto done;
+
+ err:
+  tor_free(addrport);
+  addrport = NULL;
+
+ done:
+  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
+  smartlist_free(items);
+
+  return addrport;
+}
+
+/** Given the name of a pluggable transport in <b>transport</b>, check
+ *  the configuration file to see if the user has explicitly asked for
+ *  it to listen on a specific port. Return a <address:port> string if
+ *  so, otherwise NULL. */
+char *
+pt_get_bindaddr_from_config(const char *transport)
+{
+  config_line_t *cl;
+  const or_options_t *options = get_options();
+
+  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
+    char *bindaddr =
+      get_bindaddr_from_transport_listen_line(cl->value, transport);
+    if (bindaddr)
+      return bindaddr;
+  }
+
+  return NULL;
+}
+
+/** Given a ServerTransportOptions <b>line</b>, return a smartlist
+ *  with the options. Return NULL if the line was not well-formed.
+ *
+ *  If <b>transport</b> is set, return NULL if the line is not
+ *  referring to <b>transport</b>.
+ *
+ *  The returned smartlist and its strings are allocated on the heap
+ *  and it's the responsibility of the caller to free it. */
+STATIC smartlist_t *
+get_options_from_transport_options_line(const char *line,
+                                        const char *transport)
+{
+  smartlist_t *items = smartlist_new();
+  smartlist_t *pt_options = smartlist_new();
+  const char *parsed_transport = NULL;
+
+  smartlist_split_string(items, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+
+  if (smartlist_len(items) < 2) {
+    log_warn(LD_CONFIG,"Too few arguments on ServerTransportOptions line.");
+    goto err;
+  }
+
+  parsed_transport = smartlist_get(items, 0);
+  /* If 'transport' is given, check if it matches the one on the line */
+  if (transport && strcmp(transport, parsed_transport))
+    goto err;
+
+  SMARTLIST_FOREACH_BEGIN(items, const char *, option) {
+    if (option_sl_idx == 0) /* skip the transport field (first field)*/
+      continue;
+
+    /* validate that it's a k=v value */
+    if (!string_is_key_value(LOG_WARN, option)) {
+      log_warn(LD_CONFIG, "%s is not a k=v value.", escaped(option));
+      goto err;
+    }
+
+    /* add it to the options smartlist */
+    smartlist_add_strdup(pt_options, option);
+    log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option));
+  } SMARTLIST_FOREACH_END(option);
+
+  goto done;
+
+ err:
+  SMARTLIST_FOREACH(pt_options, char*, s, tor_free(s));
+  smartlist_free(pt_options);
+  pt_options = NULL;
+
+ done:
+  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
+  smartlist_free(items);
+
+  return pt_options;
+}
+
+/** Given the name of a pluggable transport in <b>transport</b>, check
+ *  the configuration file to see if the user has asked us to pass any
+ *  parameters to the pluggable transport. Return a smartlist
+ *  containing the parameters, otherwise NULL. */
+smartlist_t *
+pt_get_options_for_server_transport(const char *transport)
+{
+  config_line_t *cl;
+  const or_options_t *options = get_options();
+
+  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
+    smartlist_t *options_sl =
+      get_options_from_transport_options_line(cl->value, transport);
+    if (options_sl)
+      return options_sl;
+  }
+
+  return NULL;
+}
+
+/**
+ * Legacy validation/normalization function for the server transport options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_server_transport(const or_options_t *old_options,
+                                  or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  config_line_t *cl;
+
+  if (options->ServerTransportPlugin && !server_mode(options)) {
+    log_notice(LD_GENERAL, "Tor is not configured as a relay but you specified"
+               " a ServerTransportPlugin line (%s). The ServerTransportPlugin "
+               "line will be ignored.",
+               escaped(options->ServerTransportPlugin->value));
+  }
+
+  if (options->ServerTransportListenAddr && !options->ServerTransportPlugin) {
+    log_notice(LD_GENERAL, "You need at least a single managed-proxy to "
+               "specify a transport listen address. The "
+               "ServerTransportListenAddr line will be ignored.");
+  }
+
+  for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
+    if (pt_parse_transport_line(options, cl->value, 1, 1) < 0)
+      REJECT("Invalid server transport line. See logs for details.");
+  }
+
+  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
+    /** If get_bindaddr_from_transport_listen_line() fails with
+        'transport' being NULL, it means that something went wrong
+        while parsing the ServerTransportListenAddr line. */
+    char *bindaddr = get_bindaddr_from_transport_listen_line(cl->value, NULL);
+    if (!bindaddr)
+      REJECT("ServerTransportListenAddr did not parse. See logs for details.");
+    tor_free(bindaddr);
+  }
+
+  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
+    /** If get_options_from_transport_options_line() fails with
+        'transport' being NULL, it means that something went wrong
+        while parsing the ServerTransportOptions line. */
+    smartlist_t *options_sl =
+      get_options_from_transport_options_line(cl->value, NULL);
+    if (!options_sl)
+      REJECT("ServerTransportOptions did not parse. See logs for details.");
+
+    SMARTLIST_FOREACH(options_sl, char *, cp, tor_free(cp));
+    smartlist_free(options_sl);
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take server pluggable transport actions
+ * based on it. All of the things we do should survive being done repeatedly.
+ * If present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_server_transport(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  config_line_t *cl;
+  const or_options_t *options = get_options();
+  int running_tor = options->command == CMD_RUN_TOR;
+
+  /* If we are a bridge with a pluggable transport proxy but no
+     Extended ORPort, inform the user that they are missing out. */
+  if (options->ServerTransportPlugin &&
+      !options->ExtORPort_lines) {
+    log_notice(LD_CONFIG, "We use pluggable transports but the Extended "
+               "ORPort is disabled. Tor and your pluggable transports proxy "
+               "communicate with each other via the Extended ORPort so it "
+               "is suggested you enable it: it will also allow your Bridge "
+               "to collect statistics about its clients that use pluggable "
+               "transports. Please enable it using the ExtORPort torrc option "
+               "(e.g. set 'ExtORPort auto').");
+  }
+
+  /* If we have an ExtORPort, initialize its auth cookie. */
+  if (running_tor &&
+      init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) {
+    log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file.");
+    return -1;
+  }
+
+  if (!options->DisableNetwork) {
+    if (options->ServerTransportPlugin) {
+      for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
+        if (pt_parse_transport_line(options, cl->value, 0, 1) < 0) {
+          // LCOV_EXCL_START
+          log_warn(LD_BUG,
+                   "Previously validated ServerTransportPlugin line "
+                   "could not be added!");
+          return -1;
+          // LCOV_EXCL_STOP
+        }
+      }
+    }
+  }
+
+  return 0;
+}
diff --git a/src/feature/relay/transport_config.h b/src/feature/relay/transport_config.h
new file mode 100644
index 0000000000..6d956d9af1
--- /dev/null
+++ b/src/feature/relay/transport_config.h
@@ -0,0 +1,85 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file transport_config.h
+ * @brief Header for feature/relay/transport_config.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_TRANSPORT_CONFIG_H
+#define TOR_FEATURE_RELAY_TRANSPORT_CONFIG_H
+
+#ifdef HAVE_MODULE_RELAY
+
+#include "lib/testsupport/testsupport.h"
+
+struct or_options_t;
+struct smartlist_t;
+
+int options_validate_server_transport(const struct or_options_t *old_options,
+                                      struct or_options_t *options,
+                                      char **msg);
+
+char *pt_get_bindaddr_from_config(const char *transport);
+struct smartlist_t *pt_get_options_for_server_transport(const char *transport);
+
+int options_act_server_transport(const struct or_options_t *old_options);
+
+#ifdef RELAY_TRANSPORT_CONFIG_PRIVATE
+
+STATIC struct smartlist_t *get_options_from_transport_options_line(
+                      const char *line,
+                      const char *transport);
+
+#endif /* defined(RELAY_TRANSPORT_CONFIG_PRIVATE) */
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+/** When tor is compiled with the relay module disabled, it can't be
+ * configured with server pluggable transports.
+ *
+ * Returns -1 and sets msg to a newly allocated string, if ExtORPort,
+ * ServerTransportPlugin, ServerTransportListenAddr, or
+ * ServerTransportOptions are set in options. Otherwise returns 0. */
+static inline int
+options_validate_server_transport(const struct or_options_t *old_options,
+                                  struct or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  /* These ExtORPort checks are too strict, and will reject valid configs
+   * that disable ports, like "ExtORPort 0". */
+  if (options->ServerTransportPlugin ||
+      options->ServerTransportListenAddr ||
+      options->ServerTransportOptions ||
+      options->ExtORPort_lines) {
+    /* REJECT() this configuration */
+    *msg = tor_strdup("This tor was built with relay mode disabled. "
+                      "It can not be configured with an ExtORPort, "
+                      "a ServerTransportPlugin, a ServerTransportListenAddr, "
+                      "or ServerTransportOptions.");
+    return -1;
+  }
+
+  return 0;
+}
+
+#define pt_get_bindaddr_from_config(transport) \
+  (((void)(transport)),NULL)
+
+/* 31851: called from client/transports.c, but only from server code */
+#define pt_get_options_for_server_transport(transport) \
+  (((void)(transport)),NULL)
+
+#define options_validate_server_transport(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_act_server_transport(old_options) \
+  (((void)(old_options)),0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_TRANSPORT_CONFIG_H) */
diff --git a/src/feature/rend/.may_include b/src/feature/rend/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/rend/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/rend/feature_rend.md b/src/feature/rend/feature_rend.md
new file mode 100644
index 0000000000..bfd8ae3dbc
--- /dev/null
+++ b/src/feature/rend/feature_rend.md
@@ -0,0 +1,7 @@
+@dir /feature/rend
+@brief feature/rend: version 2 (old) hidden services
+
+This directory implements the v2 onion service protocol,
+as specified in
+[rend-spec-v2.txt](https://gitweb.torproject.org/torspec.git/tree/rend-spec-v2.txt).
+
diff --git a/src/feature/rend/include.am b/src/feature/rend/include.am
new file mode 100644
index 0000000000..fb12439a90
--- /dev/null
+++ b/src/feature/rend/include.am
@@ -0,0 +1,22 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/rend/rendcache.c		\
+	src/feature/rend/rendclient.c		\
+	src/feature/rend/rendcommon.c		\
+	src/feature/rend/rendmid.c		\
+	src/feature/rend/rendparse.c		\
+	src/feature/rend/rendservice.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/rend/rend_authorized_client_st.h	\
+	src/feature/rend/rend_encoded_v2_service_descriptor_st.h	\
+	src/feature/rend/rend_intro_point_st.h		\
+	src/feature/rend/rend_service_descriptor_st.h	\
+	src/feature/rend/rendcache.h			\
+	src/feature/rend/rendclient.h			\
+	src/feature/rend/rendcommon.h			\
+	src/feature/rend/rendmid.h			\
+	src/feature/rend/rendparse.h			\
+	src/feature/rend/rendservice.h
diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h
index 7bd4f2fe8c..c6a6676da9 100644
--- a/src/feature/rend/rend_authorized_client_st.h
+++ b/src/feature/rend/rend_authorized_client_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_authorized_client_st.h
+ * @brief Hidden-service authorized client structure.
+ **/
+
 #ifndef REND_AUTHORIZED_CLIENT_ST_H
 #define REND_AUTHORIZED_CLIENT_ST_H
 
@@ -14,5 +19,4 @@ struct rend_authorized_client_t {
   crypto_pk_t *client_key;
 };
 
-#endif
-
+#endif /* !defined(REND_AUTHORIZED_CLIENT_ST_H) */
diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
index 05ff145d53..fea91b876a 100644
--- a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
+++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_encoded_v2_service_descriptor_st.h
+ * @brief Encoded v2 HS descriptor structure.
+ **/
+
 #ifndef REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
 #define REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
 
@@ -13,5 +18,4 @@ struct rend_encoded_v2_service_descriptor_t {
   char *desc_str; /**< Descriptor string. */
 };
 
-#endif
-
+#endif /* !defined(REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H) */
diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h
index de6987e569..4f0aa01523 100644
--- a/src/feature/rend/rend_intro_point_st.h
+++ b/src/feature/rend/rend_intro_point_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_intro_point_st.h
+ * @brief v2 hidden service introduction point structure.
+ **/
+
 #ifndef REND_INTRO_POINT_ST_H
 #define REND_INTRO_POINT_ST_H
 
@@ -73,4 +78,4 @@ struct rend_intro_point_t {
   unsigned int circuit_established:1;
 };
 
-#endif
+#endif /* !defined(REND_INTRO_POINT_ST_H) */
diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h
index aeb3178064..80c8034f46 100644
--- a/src/feature/rend/rend_service_descriptor_st.h
+++ b/src/feature/rend/rend_service_descriptor_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_service_descriptor_st.h
+ * @brief Parsed v2 HS descriptor structure.
+ **/
+
 #ifndef REND_SERVICE_DESCRIPTOR_ST_H
 #define REND_SERVICE_DESCRIPTOR_ST_H
 
@@ -30,5 +35,4 @@ struct rend_service_descriptor_t {
   smartlist_t *successful_uploads;
 };
 
-#endif
-
+#endif /* !defined(REND_SERVICE_DESCRIPTOR_ST_H) */
diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c
index 1c3badaff3..0890a81d8f 100644
--- a/src/feature/rend/rendcache.c
+++ b/src/feature/rend/rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,6 +19,8 @@
 #include "feature/rend/rend_intro_point_st.h"
 #include "feature/rend/rend_service_descriptor_st.h"
 
+#include "lib/ctime/di_ops.h"
+
 /** Map from service id (as generated by rend_get_service_id) to
  * rend_cache_entry_t. */
 STATIC strmap_t *rend_cache = NULL;
@@ -45,7 +47,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL;
  * looked up in this cache and if present, it is discarded from the fetched
  * descriptor. At the end, all IP(s) in the cache, for a specific service
  * ID, that were NOT present in the descriptor are removed from this cache.
- * Which means that if at least one IP was not in this cache, thus usuable,
+ * Which means that if at least one IP was not in this cache, thus usable,
  * it's considered a new descriptor so we keep it. Else, if all IPs were in
  * this cache, we discard the descriptor as it's considered unusable.
  *
@@ -226,6 +228,17 @@ rend_cache_entry_free_void(void *p)
   rend_cache_entry_free_(p);
 }
 
+/** Check if a failure cache entry exists for the given intro point. */
+bool
+rend_cache_intro_failure_exists(const char *service_id,
+                                const uint8_t *intro_identity)
+{
+  tor_assert(service_id);
+  tor_assert(intro_identity);
+
+  return cache_failure_intro_lookup(intro_identity, service_id, NULL);
+}
+
 /** Free all storage held by the service descriptor cache. */
 void
 rend_cache_free_all(void)
@@ -513,9 +526,16 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e)
   rend_cache_entry_t *entry = NULL;
   static const int default_version = 2;
 
-  tor_assert(rend_cache);
   tor_assert(query);
 
+  /* This is possible if we are in the shutdown process and the cache was
+   * freed while some other subsystem might do a lookup to the cache for
+   * cleanup reasons such HS circuit cleanup for instance. */
+  if (!rend_cache) {
+    ret = -ENOENT;
+    goto end;
+  }
+
   if (!rend_valid_v2_service_id(query)) {
     ret = -EINVAL;
     goto end;
@@ -593,10 +613,10 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc)
   char desc_id_digest[DIGEST_LEN];
   tor_assert(rend_cache_v2_dir);
   if (base32_decode(desc_id_digest, DIGEST_LEN,
-                    desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) {
+                    desc_id, REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Rejecting v2 rendezvous descriptor request -- descriptor ID "
-           "contains illegal characters: %s",
+           "has wrong length or illegal characters: %s",
            safe_str(desc_id));
     return -1;
   }
@@ -854,7 +874,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     *entry = NULL;
   }
   if (base32_decode(want_desc_id, sizeof(want_desc_id),
-                    desc_id_base32, strlen(desc_id_base32)) != 0) {
+                    desc_id_base32, strlen(desc_id_base32)) !=
+      sizeof(want_desc_id)) {
     log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.",
              escaped_safe_str_client(desc_id_base32));
     goto err;
@@ -888,8 +909,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   if (intro_content && intro_size > 0) {
     int n_intro_points;
     if (rend_data->auth_type != REND_NO_AUTH &&
-        !tor_mem_is_zero(rend_data->descriptor_cookie,
-                         sizeof(rend_data->descriptor_cookie))) {
+        !safe_mem_is_zero(rend_data->descriptor_cookie,
+                          sizeof(rend_data->descriptor_cookie))) {
       char *ipos_decrypted = NULL;
       size_t ipos_decrypted_size;
       if (rend_decrypt_introduction_points(&ipos_decrypted,
@@ -1005,4 +1026,3 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   tor_free(intro_content);
   return retval;
 }
-
diff --git a/src/feature/rend/rendcache.h b/src/feature/rend/rendcache.h
index aec97eabb8..45410610b4 100644
--- a/src/feature/rend/rendcache.h
+++ b/src/feature/rend/rendcache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -80,6 +80,8 @@ int rend_cache_store_v2_desc_as_client(const char *desc,
                                        rend_cache_entry_t **entry);
 size_t rend_cache_get_total_allocation(void);
 
+bool rend_cache_intro_failure_exists(const char *service_id,
+                                     const uint8_t *intro_identity);
 void rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
                                    const uint8_t *identity,
                                    const char *service_id);
diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c
index 6e95142c0b..427491e3a8 100644
--- a/src/feature/rend/rendclient.c
+++ b/src/feature/rend/rendclient.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,7 +17,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/relay.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
@@ -119,7 +119,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   char tmp[RELAY_PAYLOAD_SIZE];
   rend_cache_entry_t *entry = NULL;
   crypt_path_t *cpath;
-  off_t dh_offset;
+  ptrdiff_t dh_offset;
   crypto_pk_t *intro_key = NULL;
   int status = 0;
   const char *onion_address;
@@ -403,14 +403,23 @@ rend_client_introduction_acked(origin_circuit_t *circ,
     } else {
       log_info(LD_REND,"...Found no rend circ. Dropping on the floor.");
     }
+    /* Save the rend data digest to a temporary object so that we don't access
+     * it after we mark the circuit for close. */
+    const uint8_t *rend_digest_tmp = NULL;
+    size_t digest_len;
+    uint8_t *cached_rend_digest = NULL;
+    rend_digest_tmp = rend_data_get_pk_digest(circ->rend_data, &digest_len);
+    cached_rend_digest = tor_malloc_zero(digest_len);
+    memcpy(cached_rend_digest, rend_digest_tmp, digest_len);
+
     /* close the circuit: we won't need it anymore. */
     circuit_change_purpose(TO_CIRCUIT(circ),
                            CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
 
     /* close any other intros launched in parallel */
-    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
-                                                           NULL));
+    rend_client_close_other_intros(cached_rend_digest);
+    tor_free(cached_rend_digest); /* free the temporary digest */
   } else {
     /* It's a NAK; the introduction point didn't relay our request. */
     circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
@@ -469,16 +478,19 @@ directory_get_from_hs_dir(const char *desc_id,
 
   /* Automatically pick an hs dir if none given. */
   if (!rs_hsdir) {
+    bool rate_limited = false;
+
     /* Determine responsible dirs. Even if we can't get all we want, work with
      * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */
     smartlist_t *responsible_dirs = smartlist_new();
     hid_serv_get_responsible_directories(responsible_dirs, desc_id);
 
-    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32);
+    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32, &rate_limited);
     if (!hs_dir) {
       /* No suitable hs dir can be found, stop right now. */
-      control_event_hsv2_descriptor_failed(rend_query, NULL,
-                                           "QUERY_NO_HSDIR");
+      const char *query_response = (rate_limited) ? "QUERY_RATE_LIMITED" :
+                                                    "QUERY_NO_HSDIR";
+      control_event_hsv2_descriptor_failed(rend_query, NULL, query_response);
       control_event_hs_descriptor_content(rend_data_get_address(rend_query),
                                           desc_id_base32, NULL, NULL);
       return 0;
@@ -1036,18 +1048,30 @@ rend_client_get_random_intro_impl(const rend_cache_entry_t *entry,
   const or_options_t *options = get_options();
   smartlist_t *usable_nodes;
   int n_excluded = 0;
+  char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
 
   /* We'll keep a separate list of the usable nodes.  If this becomes empty,
    * no nodes are usable.  */
   usable_nodes = smartlist_new();
   smartlist_add_all(usable_nodes, entry->parsed->intro_nodes);
 
+  /* Get service ID so we can use it to query the failure cache. If we fail to
+   * parse it, this cache entry is no good. */
+  if (BUG(rend_get_service_id(entry->parsed->pk, service_id) < 0)) {
+    smartlist_free(usable_nodes);
+    return NULL;
+  }
+
   /* Remove the intro points that have timed out during this HS
    * connection attempt from our list of usable nodes. */
-  SMARTLIST_FOREACH(usable_nodes, rend_intro_point_t *, ip,
-                    if (ip->timed_out) {
-                      SMARTLIST_DEL_CURRENT(usable_nodes, ip);
-                    });
+  SMARTLIST_FOREACH_BEGIN(usable_nodes, const rend_intro_point_t *, ip) {
+    bool failed_intro =
+      rend_cache_intro_failure_exists(service_id,
+                       (const uint8_t *) ip->extend_info->identity_digest);
+    if (ip->timed_out || failed_intro) {
+      SMARTLIST_DEL_CURRENT(usable_nodes, ip);
+    };
+  } SMARTLIST_FOREACH_END(ip);
 
  again:
   if (smartlist_len(usable_nodes) == 0) {
@@ -1226,3 +1250,66 @@ rend_parse_service_authorization(const or_options_t *options,
   }
   return res;
 }
+
+/** The given circuit is being freed. Take appropriate action if it is of
+ * interest to the client subsystem. */
+void
+rend_client_circuit_cleanup_on_free(const circuit_t *circ)
+{
+  int reason, orig_reason;
+  bool has_timed_out, ip_is_redundant;
+  const origin_circuit_t *ocirc = NULL;
+
+  tor_assert(circ);
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+
+  reason = circ->marked_for_close_reason;
+  orig_reason = circ->marked_for_close_orig_reason;
+  ocirc = CONST_TO_ORIGIN_CIRCUIT(circ);
+  tor_assert(ocirc->rend_data);
+
+  has_timed_out = (reason == END_CIRC_REASON_TIMEOUT);
+  ip_is_redundant = (orig_reason == END_CIRC_REASON_IP_NOW_REDUNDANT);
+
+  switch (circ->purpose) {
+  case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
+  {
+    if (ip_is_redundant) {
+      break;
+    }
+    tor_assert(circ->state == CIRCUIT_STATE_OPEN);
+    tor_assert(ocirc->build_state->chosen_exit);
+    /* Treat this like getting a nack from it */
+    log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s",
+        safe_str_client(rend_data_get_address(ocirc->rend_data)),
+        safe_str_client(build_state_get_exit_nickname(ocirc->build_state)),
+        has_timed_out ? "Recording timeout." : "Removing from descriptor.");
+    rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
+                                           ocirc->rend_data,
+                                           has_timed_out ?
+                                           INTRO_POINT_FAILURE_TIMEOUT :
+                                           INTRO_POINT_FAILURE_GENERIC);
+    break;
+  }
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+  {
+    /* Ignore if we were introducing and it timed out, we didn't pick an exit
+     * point yet (IP) or the reason indicate that it was a redundant IP. */
+    if (has_timed_out || !ocirc->build_state->chosen_exit || ip_is_redundant) {
+      break;
+    }
+    log_info(LD_REND, "Failed intro circ %s to %s "
+             "(building circuit to intro point). "
+             "Marking intro point as possibly unreachable.",
+             safe_str_client(rend_data_get_address(ocirc->rend_data)),
+             safe_str_client(build_state_get_exit_nickname(
+                                                  ocirc->build_state)));
+    rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
+                                           ocirc->rend_data,
+                                           INTRO_POINT_FAILURE_UNREACHABLE);
+    break;
+  }
+  default:
+    break;
+  }
+}
diff --git a/src/feature/rend/rendclient.h b/src/feature/rend/rendclient.h
index e5f333238e..b7aa212487 100644
--- a/src/feature/rend/rendclient.h
+++ b/src/feature/rend/rendclient.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #ifndef TOR_RENDCLIENT_H
 #define TOR_RENDCLIENT_H
 
+#include "feature/hs/hs_circuit.h"
 #include "feature/rend/rendcache.h"
 
 void rend_client_purge_state(void);
@@ -47,5 +48,7 @@ rend_service_authorization_t *rend_client_lookup_service_authorization(
                                                 const char *onion_address);
 void rend_service_authorization_free_all(void);
 
+void rend_client_circuit_cleanup_on_free(const circuit_t *circ);
+
 #endif /* !defined(TOR_RENDCLIENT_H) */
 
diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c
index de48af795f..5d04755819 100644
--- a/src/feature/rend/rendcommon.c
+++ b/src/feature/rend/rendcommon.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/hs/hs_client.h"
@@ -171,9 +171,10 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id,
   }
   /* Convert service ID to binary. */
   if (base32_decode(service_id_binary, REND_SERVICE_ID_LEN,
-                    service_id, REND_SERVICE_ID_LEN_BASE32) < 0) {
+                    service_id, REND_SERVICE_ID_LEN_BASE32) !=
+      REND_SERVICE_ID_LEN) {
     log_warn(LD_REND, "Could not compute v2 descriptor ID: "
-                      "Illegal characters in service ID: %s",
+                      "Illegal characters or wrong length for service ID: %s",
              safe_str_client(service_id));
     return -1;
   }
@@ -785,39 +786,39 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
   switch (command) {
     case RELAY_COMMAND_ESTABLISH_INTRO:
       if (or_circ)
-        r = hs_intro_received_establish_intro(or_circ,payload,length);
+        r = hs_intro_received_establish_intro(or_circ, payload, length);
       break;
     case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
       if (or_circ)
-        r = rend_mid_establish_rendezvous(or_circ,payload,length);
+        r = rend_mid_establish_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE1:
       if (or_circ)
-        r = hs_intro_received_introduce1(or_circ,payload,length);
+        r = hs_intro_received_introduce1(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE2:
       if (origin_circ)
-        r = hs_service_receive_introduce2(origin_circ,payload,length);
+        r = hs_service_receive_introduce2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE_ACK:
       if (origin_circ)
-        r = hs_client_receive_introduce_ack(origin_circ,payload,length);
+        r = hs_client_receive_introduce_ack(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS1:
       if (or_circ)
-        r = rend_mid_rendezvous(or_circ,payload,length);
+        r = rend_mid_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS2:
       if (origin_circ)
-        r = hs_client_receive_rendezvous2(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRO_ESTABLISHED:
       if (origin_circ)
-        r = hs_service_receive_intro_established(origin_circ,payload,length);
+        r = hs_service_receive_intro_established(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       if (origin_circ)
-        r = hs_client_receive_rendezvous_acked(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous_acked(origin_circ, payload, length);
       break;
     default:
       tor_fragile_assert();
diff --git a/src/feature/rend/rendcommon.h b/src/feature/rend/rendcommon.h
index f136863c7a..d8281e0578 100644
--- a/src/feature/rend/rendcommon.h
+++ b/src/feature/rend/rendcommon.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c
index af02b34e6b..b497362857 100644
--- a/src/feature/rend/rendmid.c
+++ b/src/feature/rend/rendmid.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,7 @@
 #include "feature/rend/rendmid.h"
 #include "feature/stats/rephist.h"
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 
 #include "core/or/or_circuit_st.h"
@@ -58,7 +59,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
   pk = crypto_pk_asn1_decode((char*)(request+2), asn1len);
   if (!pk) {
     reason = END_CIRC_REASON_TORPROTOCOL;
-    log_warn(LD_PROTOCOL, "Couldn't decode public key.");
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Couldn't decode public key.");
     goto err;
   }
 
@@ -80,7 +81,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
                                        (char*)request, 2+asn1len+DIGEST_LEN,
                                        (char*)(request+2+DIGEST_LEN+asn1len),
                                        request_len-(2+DIGEST_LEN+asn1len))<0) {
-    log_warn(LD_PROTOCOL,
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
              "Incorrect signature on ESTABLISH_INTRO cell; rejecting.");
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
@@ -117,6 +118,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
   /* Now, set up this circuit. */
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
   hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
+  hs_dos_setup_default_intro2_defenses(circ);
 
   log_info(LD_REND,
            "Established introduction point on circuit %u for service %s",
@@ -160,9 +162,9 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
   if (request_len < (DIGEST_LEN+(MAX_NICKNAME_LEN+1)+REND_COOKIE_LEN+
                      DH1024_KEY_LEN+CIPHER_KEY_LEN+
                      PKCS1_OAEP_PADDING_OVERHEAD)) {
-    log_warn(LD_PROTOCOL, "Impossibly short INTRODUCE1 cell on circuit %u; "
-             "responding with nack.",
-             (unsigned)circ->p_circ_id);
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Impossibly short INTRODUCE1 cell on circuit %u; "
+           "responding with nack.", (unsigned)circ->p_circ_id);
     goto err;
   }
 
@@ -181,6 +183,14 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(intro_circ)) {
+    log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v2 cell due to DoS "
+                          "limitations. Sending NACK to client.");
+    goto err;
+  }
+
   log_info(LD_REND,
            "Sending introduction request for service %s "
            "from circ %u to circ %u",
@@ -237,8 +247,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
-  /* Check if we are configured to accept established rendezvous cells from
-   * client or in other words Tor2Web clients. */
+  /* Check if we are configured to defend ourselves from clients that
+   * attempt to establish rendezvous points directly to us. */
   if (channel_is_client(circ->p_chan) &&
       dos_should_refuse_single_hop_client()) {
     /* Note it down for the heartbeat log purposes. */
@@ -248,7 +258,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
   }
 
   if (circ->base_.n_chan) {
-    log_warn(LD_PROTOCOL,
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
              "Tried to establish rendezvous on non-edge circuit");
     goto err;
   }
@@ -260,8 +270,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
   }
 
   if (hs_circuitmap_get_rend_circ_relay_side(request)) {
-    log_warn(LD_PROTOCOL,
-             "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
     goto err;
   }
 
@@ -303,9 +313,9 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
   int reason = END_CIRC_REASON_INTERNAL;
 
   if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_info(LD_REND,
-             "Tried to complete rendezvous on non-OR or non-edge circuit %u.",
-             (unsigned)circ->p_circ_id);
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Tried to complete rendezvous on non-OR or non-edge circuit %u.",
+           (unsigned)circ->p_circ_id);
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
   }
diff --git a/src/feature/rend/rendmid.h b/src/feature/rend/rendmid.h
index 8ae1fa16b8..789596d855 100644
--- a/src/feature/rend/rendmid.h
+++ b/src/feature/rend/rendmid.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/rend/rendparse.c b/src/feature/rend/rendparse.c
index abd0feb448..0979d767a7 100644
--- a/src/feature/rend/rendparse.c
+++ b/src/feature/rend/rendparse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -143,8 +143,9 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     goto err;
   }
   if (base32_decode(desc_id_out, DIGEST_LEN,
-                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
-    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
+                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Descriptor ID has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -174,8 +175,10 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
     goto err;
   }
-  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
-    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
+  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) !=
+      DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Secret ID part has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -429,8 +432,10 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed,
     /* Parse identifier. */
     tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
     if (base32_decode(info->identity_digest, DIGEST_LEN,
-                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
-      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
+                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) !=
+        DIGEST_LEN) {
+      log_warn(LD_REND,
+               "Identity digest has wrong length or illegal characters: %s",
                tok->args[0]);
       rend_intro_point_free(intro);
       goto err;
diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h
index 0cef931e90..75109c204d 100644
--- a/src/feature/rend/rendparse.h
+++ b/src/feature/rend/rendparse.h
@@ -1,12 +1,12 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file rend_parse.h
- * \brief Header file for rend_parse.c.
+ * \file rendparse.h
+ * \brief Header file for rendparse.c.
  **/
 
 #ifndef TOR_REND_PARSE_H
@@ -29,4 +29,4 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
                                    size_t intro_points_encoded_size);
 int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
 
-#endif
+#endif /* !defined(TOR_REND_PARSE_H) */
diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c
index eaf0432a7d..83388a72eb 100644
--- a/src/feature/rend/rendservice.c
+++ b/src/feature/rend/rendservice.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,8 +18,9 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_common.h"
@@ -48,6 +49,7 @@
 #include "core/or/crypt_path_reference_st.h"
 #include "core/or/edge_connection_st.h"
 #include "core/or/extend_info_st.h"
+#include "feature/hs/hs_opts_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "feature/rend/rend_authorized_client_st.h"
@@ -729,22 +731,20 @@ service_config_shadow_copy(rend_service_t *service,
   config->ports = NULL;
 }
 
-/* Parse the hidden service configuration starting at <b>line_</b> using the
+/* Parse the hidden service configuration from <b>hs_opts</b> using the
  * already configured generic service configuration in <b>config</b>. This
  * function will translate the config object to a rend_service_t and add it to
  * the temporary list if valid. If <b>validate_only</b> is set, parse, warn
  * and return as normal but don't actually add the service to the list. */
 int
-rend_config_service(const config_line_t *line_,
+rend_config_service(const hs_opts_t *hs_opts,
                     const or_options_t *options,
                     hs_service_config_t *config)
 {
-  const config_line_t *line;
   rend_service_t *service = NULL;
 
-  /* line_ can be NULL which would mean that the service configuration only
-   * have one line that is the directory directive. */
   tor_assert(options);
+  tor_assert(hs_opts);
   tor_assert(config);
 
   /* We are about to configure a version 2 service. Warn of deprecation. */
@@ -764,126 +764,109 @@ rend_config_service(const config_line_t *line_,
    * options, we'll copy over the useful data to the rend_service_t object. */
   service_config_shadow_copy(service, config);
 
-  for (line = line_; line; line = line->next) {
-    if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* We just hit the next hidden service, stop right now. */
-      break;
+  /* Number of introduction points. */
+  if (hs_opts->HiddenServiceNumIntroductionPoints > NUM_INTRO_POINTS_MAX) {
+    log_warn(LD_CONFIG, "HiddenServiceNumIntroductionPoints must be "
+             "between 0 and %d, not %d.",
+             NUM_INTRO_POINTS_MAX,
+             hs_opts->HiddenServiceNumIntroductionPoints);
+    goto err;
+  }
+  service->n_intro_points_wanted = hs_opts->HiddenServiceNumIntroductionPoints;
+  log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
+           service->n_intro_points_wanted, escaped(service->directory));
+
+  /* Client authorization */
+  if (hs_opts->HiddenServiceAuthorizeClient) {
+    /* Parse auth type and comma-separated list of client names and add a
+     * rend_authorized_client_t for each client to the service's list
+     * of authorized clients. */
+    smartlist_t *type_names_split, *clients;
+    const char *authname;
+    type_names_split = smartlist_new();
+    smartlist_split_string(type_names_split,
+                           hs_opts->HiddenServiceAuthorizeClient, " ", 0, 2);
+    if (smartlist_len(type_names_split) < 1) {
+      log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This "
+               "should have been prevented when parsing the "
+               "configuration.");
+      smartlist_free(type_names_split);
+      goto err;
     }
-    /* Number of introduction points. */
-    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
-      int ok = 0;
-      /* Those are specific defaults for version 2. */
-      service->n_intro_points_wanted =
-        (unsigned int) tor_parse_long(line->value, 10,
-                                      0, NUM_INTRO_POINTS_MAX, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceNumIntroductionPoints "
-                 "should be between %d and %d, not %s",
-                 0, NUM_INTRO_POINTS_MAX, line->value);
-        goto err;
-      }
-      log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
-               service->n_intro_points_wanted, escaped(service->directory));
-      continue;
+    authname = smartlist_get(type_names_split, 0);
+    if (!strcasecmp(authname, "basic")) {
+      service->auth_type = REND_BASIC_AUTH;
+    } else if (!strcasecmp(authname, "stealth")) {
+      service->auth_type = REND_STEALTH_AUTH;
+    } else {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
+               "unrecognized auth-type '%s'. Only 'basic' or 'stealth' "
+               "are recognized.",
+               (char *) smartlist_get(type_names_split, 0));
+      SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
+      smartlist_free(type_names_split);
+      goto err;
     }
-    if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
-      /* Parse auth type and comma-separated list of client names and add a
-       * rend_authorized_client_t for each client to the service's list
-       * of authorized clients. */
-      smartlist_t *type_names_split, *clients;
-      const char *authname;
-      if (service->auth_type != REND_NO_AUTH) {
-        log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient "
-                 "lines for a single service.");
-        goto err;
-      }
-      type_names_split = smartlist_new();
-      smartlist_split_string(type_names_split, line->value, " ", 0, 2);
-      if (smartlist_len(type_names_split) < 1) {
-        log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This "
-                         "should have been prevented when parsing the "
-                         "configuration.");
-        smartlist_free(type_names_split);
-        goto err;
-      }
-      authname = smartlist_get(type_names_split, 0);
-      if (!strcasecmp(authname, "basic")) {
-        service->auth_type = REND_BASIC_AUTH;
-      } else if (!strcasecmp(authname, "stealth")) {
-        service->auth_type = REND_STEALTH_AUTH;
-      } else {
-        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
-                 "unrecognized auth-type '%s'. Only 'basic' or 'stealth' "
-                 "are recognized.",
-                 (char *) smartlist_get(type_names_split, 0));
-        SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
-        smartlist_free(type_names_split);
-        goto err;
-      }
-      service->clients = smartlist_new();
-      if (smartlist_len(type_names_split) < 2) {
-        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
-                            "auth-type '%s', but no client names.",
-                 service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
-        SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
-        smartlist_free(type_names_split);
-        continue;
-      }
-      clients = smartlist_new();
-      smartlist_split_string(clients, smartlist_get(type_names_split, 1),
-                             ",", SPLIT_SKIP_SPACE, 0);
+    service->clients = smartlist_new();
+    if (smartlist_len(type_names_split) < 2) {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
+               "auth-type '%s', but no client names.",
+               service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
       SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
       smartlist_free(type_names_split);
-      /* Remove duplicate client names. */
-      {
-        int num_clients = smartlist_len(clients);
-        smartlist_sort_strings(clients);
-        smartlist_uniq_strings(clients);
-        if (smartlist_len(clients) < num_clients) {
-          log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
-                   "duplicate client name(s); removing.",
-                   num_clients - smartlist_len(clients));
-        }
-      }
-      SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name)
-      {
-        rend_authorized_client_t *client;
-        if (!rend_valid_client_name(client_name)) {
-          log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an "
-                              "illegal client name: '%s'. Names must be "
-                              "between 1 and %d characters and contain "
-                              "only [A-Za-z0-9+_-].",
-                   client_name, REND_CLIENTNAME_MAX_LEN);
-          SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
-          smartlist_free(clients);
-          goto err;
-        }
-        client = tor_malloc_zero(sizeof(rend_authorized_client_t));
-        client->client_name = tor_strdup(client_name);
-        smartlist_add(service->clients, client);
-        log_debug(LD_REND, "Adding client name '%s'", client_name);
+      goto err;
+    }
+    clients = smartlist_new();
+    smartlist_split_string(clients, smartlist_get(type_names_split, 1),
+                             ",", SPLIT_SKIP_SPACE, 0);
+    SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
+    smartlist_free(type_names_split);
+    /* Remove duplicate client names. */
+    {
+      int num_clients = smartlist_len(clients);
+      smartlist_sort_strings(clients);
+      smartlist_uniq_strings(clients);
+      if (smartlist_len(clients) < num_clients) {
+        log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                 "duplicate client name(s); removing.",
+                 num_clients - smartlist_len(clients));
       }
-      SMARTLIST_FOREACH_END(client_name);
-      SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
-      smartlist_free(clients);
-      /* Ensure maximum number of clients. */
-      if ((service->auth_type == REND_BASIC_AUTH &&
-            smartlist_len(service->clients) > 512) ||
-          (service->auth_type == REND_STEALTH_AUTH &&
-            smartlist_len(service->clients) > 16)) {
-        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
-                            "client authorization entries, but only a "
-                            "maximum of %d entries is allowed for "
-                            "authorization type '%s'.",
-                 smartlist_len(service->clients),
-                 service->auth_type == REND_BASIC_AUTH ? 512 : 16,
-                 service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
+    }
+    SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name) {
+      rend_authorized_client_t *client;
+      if (!rend_valid_client_name(client_name)) {
+        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an "
+                            "illegal client name: '%s'. Names must be "
+                            "between 1 and %d characters and contain "
+                            "only [A-Za-z0-9+_-].",
+                 client_name, REND_CLIENTNAME_MAX_LEN);
+        SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
+        smartlist_free(clients);
         goto err;
       }
-      continue;
+      client = tor_malloc_zero(sizeof(rend_authorized_client_t));
+      client->client_name = tor_strdup(client_name);
+      smartlist_add(service->clients, client);
+      log_debug(LD_REND, "Adding client name '%s'", client_name);
+    } SMARTLIST_FOREACH_END(client_name);
+    SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
+    smartlist_free(clients);
+    /* Ensure maximum number of clients. */
+    if ((service->auth_type == REND_BASIC_AUTH &&
+         smartlist_len(service->clients) > 512) ||
+        (service->auth_type == REND_STEALTH_AUTH &&
+         smartlist_len(service->clients) > 16)) {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                          "client authorization entries, but only a "
+                          "maximum of %d entries is allowed for "
+                          "authorization type '%s'.",
+               smartlist_len(service->clients),
+               service->auth_type == REND_BASIC_AUTH ? 512 : 16,
+               service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
+      goto err;
     }
   }
+
   /* Validate the service just parsed. */
   if (rend_validate_service(rend_service_staging_list, service) < 0) {
     /* Service is in the staging list so don't try to free it. */
@@ -2145,7 +2128,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
      *
      * We only use a one-hop path on the first attempt. If the first attempt
      * fails, we use a 3-hop path for reachability / reliability.
-     * See the comment in rend_service_relauch_rendezvous() for details. */
+     * See the comment in rend_service_relaunch_rendezvous() for details. */
     if (rend_service_use_direct_connection(options, rp) && i == 0) {
       flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL;
     }
@@ -2186,7 +2169,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   cpath->rend_dh_handshake_state = dh;
   dh = NULL;
-  if (circuit_init_cpath_crypto(cpath,
+  if (cpath_init_circuit_crypto(cpath,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 1, 0)<0)
     goto err;
@@ -3035,6 +3018,10 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
 {
   origin_circuit_t *newcirc;
   cpath_build_state_t *newstate, *oldstate;
+  const char *rend_pk_digest;
+  rend_service_t *service = NULL;
+
+  int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
 
   tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
   oldstate = oldcirc->build_state;
@@ -3049,13 +3036,31 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
   log_info(LD_REND,"Reattempting rendezvous circuit to '%s'",
            safe_str(extend_info_describe(oldstate->chosen_exit)));
 
+  /* Look up the service. */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(oldcirc->rend_data, NULL);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+
+  if (!service) {
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                  rend_pk_digest, REND_SERVICE_ID_LEN);
+
+    log_warn(LD_BUG, "Internal error: Trying to relaunch a rendezvous circ "
+                     "for an unrecognized service %s.",
+                     safe_str_client(serviceid));
+    return;
+  }
+
+  if (hs_service_requires_uptime_circ(service->ports)) {
+    flags |= CIRCLAUNCH_NEED_UPTIME;
+  }
+
   /* You'd think Single Onion Services would want to retry the rendezvous
    * using a direct connection. But if it's blocked by a firewall, or the
    * service is IPv6-only, or the rend point avoiding becoming a one-hop
    * proxy, we need a 3-hop connection. */
   newcirc = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
-                            oldstate->chosen_exit,
-                            CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+                            oldstate->chosen_exit, flags);
 
   if (!newcirc) {
     log_warn(LD_REND,"Couldn't relaunch rendezvous circuit to '%s'.",
@@ -3555,7 +3560,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   hop->package_window = circuit_initial_package_window();
   hop->deliver_window = CIRCWINDOW_START;
 
-  onion_append_to_cpath(&circuit->cpath, hop);
+  cpath_extend_linked_list(&circuit->cpath, hop);
   circuit->build_state->pending_final_cpath = NULL; /* prevent double-free */
 
   /* Change the circuit purpose. */
@@ -3729,20 +3734,23 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
       base32_encode(desc_id_base32, sizeof(desc_id_base32),
                     desc->desc_id, DIGEST_LEN);
       hs_dir_ip = tor_dup_ip(hs_dir->addr);
-      log_info(LD_REND, "Launching upload for v2 descriptor for "
-                        "service '%s' with descriptor ID '%s' with validity "
-                        "of %d seconds to hidden service directory '%s' on "
-                        "%s:%d.",
-               safe_str_client(service_id),
-               safe_str_client(desc_id_base32),
-               seconds_valid,
-               hs_dir->nickname,
-               hs_dir_ip,
-               hs_dir->or_port);
+      if (hs_dir_ip) {
+        log_info(LD_REND, "Launching upload for v2 descriptor for "
+                          "service '%s' with descriptor ID '%s' with validity "
+                          "of %d seconds to hidden service directory '%s' on "
+                          "%s:%d.",
+                 safe_str_client(service_id),
+                 safe_str_client(desc_id_base32),
+                 seconds_valid,
+                 hs_dir->nickname,
+                 hs_dir_ip,
+                 hs_dir->or_port);
+        tor_free(hs_dir_ip);
+      }
+
       control_event_hs_descriptor_upload(service_id,
                                          hs_dir->identity_digest,
                                          desc_id_base32, NULL);
-      tor_free(hs_dir_ip);
       /* Remember successful upload to this router for next time. */
       if (!smartlist_contains_digest(successful_uploads,
                                      hs_dir->identity_digest))
@@ -3995,7 +4003,7 @@ remove_invalid_intro_points(rend_service_t *service,
        * accounted for when considiring uploading a descriptor. */
       intro->circuit_established = 0;
 
-      /* Node is gone or we've reached our maximum circuit creationg retry
+      /* Node is gone or we've reached our maximum circuit creation retry
        * count, clean up everything, we'll find a new one. */
       if (node == NULL ||
           intro->circuit_retries >= MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -4235,6 +4243,7 @@ rend_consider_services_intro_points(time_t now)
        * directly ourselves. */
       intro->extend_info = extend_info_from_node(node, 0);
       if (BUG(intro->extend_info == NULL)) {
+        tor_free(intro);
         break;
       }
       intro->intro_key = crypto_pk_new();
@@ -4364,17 +4373,16 @@ rend_consider_descriptor_republication(void)
 void
 rend_service_dump_stats(int severity)
 {
-  int i,j;
   rend_service_t *service;
   rend_intro_point_t *intro;
   const char *safe_name;
   origin_circuit_t *circ;
 
-  for (i=0; i < smartlist_len(rend_service_list); ++i) {
+  for (int i = 0; i < smartlist_len(rend_service_list); ++i) {
     service = smartlist_get(rend_service_list, i);
     tor_log(severity, LD_GENERAL, "Service configured in %s:",
             rend_service_escaped_dir(service));
-    for (j=0; j < smartlist_len(service->intro_nodes); ++j) {
+    for (int j = 0; j < smartlist_len(service->intro_nodes); ++j) {
       intro = smartlist_get(service->intro_nodes, j);
       safe_name = safe_str_client(intro->extend_info->nickname);
 
diff --git a/src/feature/rend/rendservice.h b/src/feature/rend/rendservice.h
index a8eb28bee2..012afc0f9f 100644
--- a/src/feature/rend/rendservice.h
+++ b/src/feature/rend/rendservice.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -139,7 +139,8 @@ STATIC void rend_service_prune_list_impl_(void);
 #endif /* defined(RENDSERVICE_PRIVATE) */
 
 int rend_num_services(void);
-int rend_config_service(const struct config_line_t *line_,
+struct hs_opts_t;
+int rend_config_service(const struct hs_opts_t *hs_opts,
                         const or_options_t *options,
                         hs_service_config_t *config);
 void rend_service_prune_list(void);
diff --git a/src/feature/stats/.may_include b/src/feature/stats/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/stats/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/stats/feature_stats.md b/src/feature/stats/feature_stats.md
new file mode 100644
index 0000000000..d205fe5571
--- /dev/null
+++ b/src/feature/stats/feature_stats.md
@@ -0,0 +1,10 @@
+@dir /feature/stats
+@brief feature/stats: Relay statistics. Also, port prediction.
+
+This module collects anonymized relay statistics in order to publish them in
+relays' routerinfo and extrainfo documents.
+
+Additionally, it contains predict_ports.c, which remembers which ports we've
+visited recently as a client, so we can make sure we have open circuits that
+support them.
+
diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c
index a54b589eb6..f9a2f19d2e 100644
--- a/src/feature/stats/geoip_stats.c
+++ b/src/feature/stats/geoip_stats.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -30,9 +30,9 @@
 #include "core/or/or.h"
 
 #include "ht.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/dnsserv.h"
 #include "core/or/dos.h"
 #include "lib/geoip/geoip.h"
@@ -146,9 +146,9 @@ clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
 }
 
 HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
-             clientmap_entries_eq)
+             clientmap_entries_eq);
 HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
-             clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 #define clientmap_entry_free(ent) \
   FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
@@ -484,9 +484,9 @@ dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
 }
 
 HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
-             dirreq_map_ent_eq)
+             dirreq_map_ent_eq);
 HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
-             dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+             dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** Helper: Put <b>entry</b> into map of directory requests using
  * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
diff --git a/src/feature/stats/geoip_stats.h b/src/feature/stats/geoip_stats.h
index 2fc62b5466..fcfe7a31f0 100644
--- a/src/feature/stats/geoip_stats.h
+++ b/src/feature/stats/geoip_stats.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_GEOIP_STATS_H
 
 #include "core/or/dos.h"
+#include "ext/ht.h"
 
 /** Indicates an action that we might be noting geoip statistics on.
  * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
diff --git a/src/feature/stats/include.am b/src/feature/stats/include.am
new file mode 100644
index 0000000000..8789bc3d96
--- /dev/null
+++ b/src/feature/stats/include.am
@@ -0,0 +1,12 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/stats/geoip_stats.c		\
+	src/feature/stats/rephist.c		\
+	src/feature/stats/predict_ports.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/stats/geoip_stats.h			\
+	src/feature/stats/rephist.h			\
+	src/feature/stats/predict_ports.h
diff --git a/src/feature/stats/predict_ports.c b/src/feature/stats/predict_ports.c
index 3cbba2c831..d728f106a2 100644
--- a/src/feature/stats/predict_ports.c
+++ b/src/feature/stats/predict_ports.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h
index 272344da2f..ed067b6ced 100644
--- a/src/feature/stats/predict_ports.h
+++ b/src/feature/stats/predict_ports.h
@@ -1,11 +1,11 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file predict_portst.h
+ * \file predict_ports.h
  * \brief Header file for predict_ports.c.
  **/
 
@@ -27,4 +27,4 @@ int rep_hist_circbuilding_dormant(time_t now);
 int predicted_ports_prediction_time_remaining(time_t now);
 void predicted_ports_free_all(void);
 
-#endif
+#endif /* !defined(TOR_PREDICT_PORTS_H) */
diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c
index 3f560fbce7..71e2e00086 100644
--- a/src/feature/stats/rephist.c
+++ b/src/feature/stats/rephist.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -1122,7 +1122,7 @@ static bw_array_t *dir_read_array = NULL;
     directory protocol. */
 static bw_array_t *dir_write_array = NULL;
 
-/** Set up [dir-]read_array and [dir-]write_array, freeing them if they
+/** Set up [dir_]read_array and [dir_]write_array, freeing them if they
  * already exist. */
 static void
 bw_arrays_init(void)
@@ -2285,9 +2285,9 @@ bidi_map_ent_hash(const bidi_map_entry_t *entry)
 }
 
 HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
-             bidi_map_ent_eq)
+             bidi_map_ent_eq);
 HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
-             bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+             bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /* DOCDOC bidi_map_free */
 static void
diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h
index 3accc8c610..92c3d2a5a5 100644
--- a/src/feature/stats/rephist.h
+++ b/src/feature/stats/rephist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -103,7 +103,7 @@ typedef struct bw_array_t bw_array_t;
 STATIC uint64_t find_largest_max(bw_array_t *b);
 STATIC void commit_max(bw_array_t *b);
 STATIC void advance_obs(bw_array_t *b);
-#endif
+#endif /* defined(REPHIST_PRIVATE) */
 
 /**
  * Represents the type of a cell for padding accounting
diff --git a/src/include.am b/src/include.am
index d2f83da814..657f6e823a 100644
--- a/src/include.am
+++ b/src/include.am
@@ -1,12 +1,17 @@
 include src/ext/include.am
+
 include src/lib/arch/include.am
+include src/lib/buf/include.am
 include src/lib/err/include.am
 include src/lib/cc/include.am
 include src/lib/ctime/include.am
 include src/lib/compress/include.am
+include src/lib/conf/include.am
+include src/lib/confmgt/include.am
 include src/lib/container/include.am
 include src/lib/crypt_ops/include.am
 include src/lib/defs/include.am
+include src/lib/dispatch/include.am
 include src/lib/encoding/include.am
 include src/lib/evloop/include.am
 include src/lib/fdio/include.am
@@ -14,6 +19,7 @@ include src/lib/fs/include.am
 include src/lib/geoip/include.am
 include src/lib/include.libdonna.am
 include src/lib/intmath/include.am
+include src/lib/llharden/include.am
 include src/lib/lock/include.am
 include src/lib/log/include.am
 include src/lib/math/include.am
@@ -23,8 +29,10 @@ include src/lib/malloc/include.am
 include src/lib/net/include.am
 include src/lib/osinfo/include.am
 include src/lib/process/include.am
+include src/lib/pubsub/include.am
 include src/lib/sandbox/include.am
 include src/lib/string/include.am
+include src/lib/subsys/include.am
 include src/lib/smartlist_core/include.am
 include src/lib/term/include.am
 include src/lib/testsupport/include.am
@@ -32,9 +40,46 @@ include src/lib/thread/include.am
 include src/lib/time/include.am
 include src/lib/tls/include.am
 include src/lib/trace/include.am
+include src/lib/version/include.am
 include src/lib/wallclock/include.am
 include src/trunnel/include.am
 
+noinst_LIBRARIES += src/core/libtor-app.a
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/core/libtor-app-testing.a
+endif
+
+LIBTOR_APP_A_SOURCES =
+#
+# Sources that we only add for the real libtor_a, and not for testing.
+#
+LIBTOR_APP_A_STUB_SOURCES =
+
+include src/core/crypto/include.am
+include src/core/mainloop/include.am
+include src/core/or/include.am
+include src/core/proto/include.am
+
+include src/feature/api/include.am
+include src/feature/client/include.am
+include src/feature/control/include.am
+include src/feature/dirauth/include.am
+include src/feature/dircache/include.am
+include src/feature/dirclient/include.am
+include src/feature/dircommon/include.am
+include src/feature/dirparse/include.am
+include src/feature/hibernate/include.am
+include src/feature/hs_common/include.am
+include src/feature/hs/include.am
+include src/feature/keymgt/include.am
+include src/feature/nodelist/include.am
+include src/feature/relay/include.am
+include src/feature/rend/include.am
+include src/feature/stats/include.am
+
+include src/app/config/include.am
+include src/app/main/include.am
+
 include src/core/include.am
 include src/app/include.am
 
diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h
index fa82241b28..c72ac3eb8e 100644
--- a/src/lib/arch/bytes.h
+++ b/src/lib/arch/bytes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BYTES_H
@@ -16,12 +16,17 @@
 #include <string.h>
 #include "lib/cc/torint.h"
 
-/* The uint8 variants are defined to make the code more uniform. */
+/**
+ * Read an 8-bit from <b>cp</b>.
+ */
 static inline uint8_t
 get_uint8(const void *cp)
 {
   return *(const uint8_t*)(cp);
 }
+/**
+ * Store an 8-bit value from <b>v</b> to <b>cp</b>.
+ */
 static inline void
 set_uint8(void *cp, uint8_t v)
 {
@@ -93,7 +98,7 @@ set_uint64(void *cp, uint64_t v)
   memcpy(cp,&v,8);
 }
 
-#ifdef WORDS_BIGENDIAN
+#if defined(WORDS_BIGENDIAN)
 static inline uint16_t
 tor_htons(uint32_t a)
 {
@@ -129,7 +134,10 @@ tor_ntohll(uint64_t a)
 {
   return a;
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
+/**
+ * Convert a 16-bit value from host order to network order (big-endian).
+ **/
 static inline uint16_t
 tor_htons(uint16_t a)
 {
@@ -139,12 +147,18 @@ tor_htons(uint16_t a)
     ((a & 0xff00) >> 8);
 }
 
+/**
+ * Convert a 16-bit value from network order (big-endian) to host order.
+ **/
 static inline uint16_t
 tor_ntohs(uint16_t a)
 {
   return tor_htons(a);
 }
 
+/**
+ * Convert a 32-bit value from host order to network order (big-endian).
+ **/
 static inline uint32_t
 tor_htonl(uint32_t a)
 {
@@ -156,6 +170,9 @@ tor_htonl(uint32_t a)
     ((a & 0xff000000) >>24);
 }
 
+/**
+ * Convert a 32-bit value from network order (big-endian) to host order.
+ **/
 static inline uint32_t
 tor_ntohl(uint32_t a)
 {
@@ -177,6 +194,6 @@ tor_ntohll(uint64_t a)
 {
   return tor_htonll(a);
 }
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
-#endif
+#endif /* !defined(TOR_BYTES_H) */
diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am
index f92ee9222f..c5926c6330 100644
--- a/src/lib/arch/include.am
+++ b/src/lib/arch/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/arch/bytes.h
diff --git a/src/lib/arch/lib_arch.md b/src/lib/arch/lib_arch.md
new file mode 100644
index 0000000000..9b8bccdf16
--- /dev/null
+++ b/src/lib/arch/lib_arch.md
@@ -0,0 +1,2 @@
+@dir /lib/arch
+@brief lib/arch: Compatibility code for handling different CPU architectures.
diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include
new file mode 100644
index 0000000000..c4be73bce2
--- /dev/null
+++ b/src/lib/buf/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/buf/*.h
+lib/cc/*.h
+lib/ctime/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
+lib/log/*.h
+lib/string/*.h
+lib/time/*.h
diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c
index fe4cf7c385..a5031a47a6 100644
--- a/src/lib/container/buffers.c
+++ b/src/lib/buf/buffers.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -99,6 +99,7 @@
 #define DBG_S(s) (void)0
 #endif
 
+#ifndef COCCI
 #ifdef DISABLE_MEMORY_SENTINELS
 #define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
 #else
@@ -109,6 +110,7 @@
     memset(a,0,SENTINEL_LEN);                                           \
   } while (0)
 #endif /* defined(DISABLE_MEMORY_SENTINELS) */
+#endif /* !defined(COCCI) */
 
 /** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
  * to free up space at the end. */
@@ -158,7 +160,7 @@ chunk_new_with_alloc_size(size_t alloc)
 static inline chunk_t *
 chunk_grow(chunk_t *chunk, size_t sz)
 {
-  off_t offset;
+  ptrdiff_t offset;
   const size_t memlen_orig = chunk->memlen;
   const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
   const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
@@ -283,7 +285,7 @@ buf_t *
 buf_new_with_data(const char *cp, size_t sz)
 {
   /* Validate arguments */
-  if (!cp || sz <= 0 || sz >= INT_MAX) {
+  if (!cp || sz <= 0 || sz > BUF_MAX_LEN) {
     return NULL;
   }
 
@@ -440,7 +442,7 @@ chunk_copy(const chunk_t *in_chunk)
 #endif
   newch->next = NULL;
   if (in_chunk->data) {
-    off_t offset = in_chunk->data - in_chunk->mem;
+    ptrdiff_t offset = in_chunk->data - in_chunk->mem;
     newch->data = newch->mem + offset;
   }
   return newch;
@@ -528,9 +530,9 @@ buf_add(buf_t *buf, const char *string, size_t string_len)
     return (int)buf->datalen;
   check();
 
-  if (BUG(buf->datalen >= INT_MAX))
+  if (BUG(buf->datalen > BUF_MAX_LEN))
     return -1;
-  if (BUG(buf->datalen >= INT_MAX - string_len))
+  if (BUG(buf->datalen > BUF_MAX_LEN - string_len))
     return -1;
 
   while (string_len) {
@@ -549,7 +551,7 @@ buf_add(buf_t *buf, const char *string, size_t string_len)
   }
 
   check();
-  tor_assert(buf->datalen < INT_MAX);
+  tor_assert(buf->datalen <= BUF_MAX_LEN);
   return (int)buf->datalen;
 }
 
@@ -578,6 +580,7 @@ buf_add_vprintf(buf_t *buf, const char *format, va_list args)
   /* XXXX Faster implementations are easy enough, but let's optimize later */
   char *tmp;
   tor_vasprintf(&tmp, format, args);
+  tor_assert(tmp != NULL);
   buf_add(buf, tmp, strlen(tmp));
   tor_free(tmp);
 }
@@ -642,7 +645,7 @@ buf_get_bytes(buf_t *buf, char *string, size_t string_len)
   buf_peek(buf, string, string_len);
   buf_drain(buf, string_len);
   check();
-  tor_assert(buf->datalen < INT_MAX);
+  tor_assert(buf->datalen <= BUF_MAX_LEN);
   return (int)buf->datalen;
 }
 
@@ -657,9 +660,9 @@ buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
   char b[4096];
   size_t cp, len;
 
-  if (BUG(buf_out->datalen >= INT_MAX || *buf_flushlen >= INT_MAX))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN || *buf_flushlen > BUF_MAX_LEN))
     return -1;
-  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN - *buf_flushlen))
     return -1;
 
   len = *buf_flushlen;
@@ -667,7 +670,7 @@ buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
     len = buf_in->datalen;
 
   cp = len; /* Remember the number of bytes we intend to copy. */
-  tor_assert(cp < INT_MAX);
+  tor_assert(cp <= BUF_MAX_LEN);
   while (len) {
     /* This isn't the most efficient implementation one could imagine, since
      * it does two copies instead of 1, but I kinda doubt that this will be
@@ -691,9 +694,9 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in)
     return;
   if (buf_datalen(buf_in) == 0)
     return;
-  if (BUG(buf_out->datalen >= INT_MAX || buf_in->datalen >= INT_MAX))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN || buf_in->datalen > BUF_MAX_LEN))
     return;
-  if (BUG(buf_out->datalen >= INT_MAX - buf_in->datalen))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN - buf_in->datalen))
     return;
 
   if (buf_out->head == NULL) {
@@ -712,7 +715,8 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in)
 /** Internal structure: represents a position in a buffer. */
 typedef struct buf_pos_t {
   const chunk_t *chunk; /**< Which chunk are we pointing to? */
-  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  ptrdiff_t pos;/**< Which character inside the chunk's data are we pointing
+                 * to? */
   size_t chunk_pos; /**< Total length of all previous chunks. */
 } buf_pos_t;
 
@@ -728,15 +732,15 @@ buf_pos_init(const buf_t *buf, buf_pos_t *out)
 /** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
  * position of <b>out</b>, or later.  Return -1 if no instances are found;
  * otherwise returns the absolute position of the character. */
-static off_t
+static ptrdiff_t
 buf_find_pos_of_char(char ch, buf_pos_t *out)
 {
   const chunk_t *chunk;
-  int pos;
+  ptrdiff_t pos;
   tor_assert(out);
   if (out->chunk) {
     if (out->chunk->datalen) {
-      tor_assert(out->pos < (off_t)out->chunk->datalen);
+      tor_assert(out->pos < (ptrdiff_t)out->chunk->datalen);
     } else {
       tor_assert(out->pos == 0);
     }
@@ -746,7 +750,7 @@ buf_find_pos_of_char(char ch, buf_pos_t *out)
     char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
     if (cp) {
       out->chunk = chunk;
-      tor_assert(cp - chunk->data < INT_MAX);
+      tor_assert(cp - chunk->data <= BUF_MAX_LEN);
       out->pos = (int)(cp - chunk->data);
       return out->chunk_pos + out->pos;
     } else {
@@ -762,9 +766,9 @@ buf_find_pos_of_char(char ch, buf_pos_t *out)
 static inline int
 buf_pos_inc(buf_pos_t *pos)
 {
-  tor_assert(pos->pos < INT_MAX - 1);
+  tor_assert(pos->pos < BUF_MAX_LEN);
   ++pos->pos;
-  if (pos->pos == (off_t)pos->chunk->datalen) {
+  if (pos->pos == (ptrdiff_t)pos->chunk->datalen) {
     if (!pos->chunk->next)
       return -1;
     pos->chunk_pos += pos->chunk->datalen;
@@ -809,7 +813,7 @@ buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
   buf_pos_init(buf, &pos);
   while (buf_find_pos_of_char(*s, &pos) >= 0) {
     if (buf_matches_at_pos(&pos, s, n)) {
-      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
+      tor_assert(pos.chunk_pos + pos.pos <= BUF_MAX_LEN);
       return (int)(pos.chunk_pos + pos.pos);
     } else {
       if (buf_pos_inc(&pos)<0)
@@ -838,12 +842,12 @@ buf_peek_startswith(const buf_t *buf, const char *cmd)
 
 /** Return the index within <b>buf</b> at which <b>ch</b> first appears,
  * or -1 if <b>ch</b> does not appear on buf. */
-static off_t
+static ptrdiff_t
 buf_find_offset_of_char(buf_t *buf, char ch)
 {
   chunk_t *chunk;
-  off_t offset = 0;
-  tor_assert(buf->datalen < INT_MAX);
+  ptrdiff_t offset = 0;
+  tor_assert(buf->datalen <= BUF_MAX_LEN);
   for (chunk = buf->head; chunk; chunk = chunk->next) {
     char *cp = memchr(chunk->data, ch, chunk->datalen);
     if (cp)
@@ -865,7 +869,7 @@ int
 buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
 {
   size_t sz;
-  off_t offset;
+  ptrdiff_t offset;
 
   if (!buf->head)
     return 0;
@@ -913,7 +917,7 @@ buf_assert_ok(buf_t *buf)
     for (ch = buf->head; ch; ch = ch->next) {
       total += ch->datalen;
       tor_assert(ch->datalen <= ch->memlen);
-      tor_assert(ch->datalen < INT_MAX);
+      tor_assert(ch->datalen <= BUF_MAX_LEN);
       tor_assert(ch->data >= &ch->mem[0]);
       tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
       if (ch->data == &ch->mem[0]+ch->memlen) {
diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h
index c103b93a82..d8a77feb72 100644
--- a/src/lib/container/buffers.h
+++ b/src/lib/buf/buffers.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,9 @@ void buf_free_(buf_t *buf);
 void buf_clear(buf_t *buf);
 buf_t *buf_copy(const buf_t *buf);
 
+/** Maximum bytes in a buffer, inclusive. */
+#define BUF_MAX_LEN (INT_MAX - 1)
+
 MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
 size_t buf_allocation(const buf_t *buf);
 size_t buf_slack(const buf_t *buf);
diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am
new file mode 100644
index 0000000000..27430d1d38
--- /dev/null
+++ b/src/lib/buf/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-buf.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-buf-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_buf_a_SOURCES =			\
+	src/lib/buf/buffers.c
+
+src_lib_libtor_buf_testing_a_SOURCES = \
+	$(src_lib_libtor_buf_a_SOURCES)
+src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/buf/buffers.h
diff --git a/src/lib/buf/lib_buf.md b/src/lib/buf/lib_buf.md
new file mode 100644
index 0000000000..519ab50a2d
--- /dev/null
+++ b/src/lib/buf/lib_buf.md
@@ -0,0 +1,13 @@
+@dir /lib/buf
+@brief lib/buf: An efficient byte queue.
+
+This module defines the buf_t type, which is used throughout our networking
+code.  The implementation is a singly-linked queue of buffer chunks, similar
+to the BSD kernel's
+["mbuf"](https://www.freebsd.org/cgi/man.cgi?query=mbuf&sektion=9) structure.
+
+The buf_t type is also reasonable for use in constructing long strings.
+
+See \refdir{lib/net} for networking code that uses buf_t, and
+\refdir{lib/tls} for cryptographic code that uses buf_t.
+
diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include
index 2b06e8519c..fa1478ce46 100644
--- a/src/lib/cc/.may_include
+++ b/src/lib/cc/.may_include
@@ -1 +1,2 @@
 orconfig.h
+lib/cc/*.h
\ No newline at end of file
diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h
index fbe6a38f1f..96aa912652 100644
--- a/src/lib/cc/compat_compiler.h
+++ b/src/lib/cc/compat_compiler.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,11 +25,11 @@
 #endif /* defined(__has_feature) */
 
 #ifndef NULL_REP_IS_ZERO_BYTES
-#error "It seems your platform does not represent NULL as zero. We can't cope."
+#error "Your platform does not represent NULL as zero. We can't cope."
 #endif
 
 #ifndef DOUBLE_0_REP_IS_ZERO_BYTES
-#error "It seems your platform does not represent 0.0 as zeros. We can't cope."
+#error "Your platform does not represent 0.0 as zeros. We can't cope."
 #endif
 
 #if 'a'!=97 || 'z'!=122 || 'A'!=65 || ' '!=32
@@ -65,8 +65,6 @@
 
 /* Temporarily enable and disable warnings. */
 #ifdef __GNUC__
-#  define PRAGMA_STRINGIFY_(s) #s
-#  define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b)
 /* Support for macro-generated pragmas (c99) */
 #  define PRAGMA_(x) _Pragma (#x)
 #  ifdef __clang__
@@ -78,17 +76,17 @@
 /* we have push/pop support */
 #    define DISABLE_GCC_WARNING(warningopt) \
           PRAGMA_DIAGNOSTIC_(push) \
-          PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+          PRAGMA_DIAGNOSTIC_(ignored warningopt)
 #    define ENABLE_GCC_WARNING(warningopt) \
           PRAGMA_DIAGNOSTIC_(pop)
 #else /* !(defined(__clang__) || GCC_VERSION >= 406) */
 /* older version of gcc: no push/pop support. */
 #    define DISABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+         PRAGMA_DIAGNOSTIC_(ignored warningopt)
 #    define ENABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+         PRAGMA_DIAGNOSTIC_(warning warningopt)
 #endif /* defined(__clang__) || GCC_VERSION >= 406 */
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 /* not gcc at all */
 # define DISABLE_GCC_WARNING(warning)
 # define ENABLE_GCC_WARNING(warning)
@@ -200,8 +198,8 @@
 /** Macro: yield a pointer to the field at position <b>off</b> within the
  * structure <b>st</b>.  Example:
  * <pre>
- *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = offsetof(struct a, bar);
+ *   struct a_t { int foo; int bar; } x;
+ *   ptrdiff_t bar_offset = offsetof(struct a_t, bar);
  *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
  *   *bar_p = 3;
  * </pre>
@@ -211,10 +209,10 @@
 /** Macro: yield a pointer to an enclosing structure given a pointer to
  * a substructure at offset <b>off</b>. Example:
  * <pre>
- *   struct base { ... };
- *   struct subtype { int x; struct base b; } x;
- *   struct base *bp = &x.base;
- *   struct *sp = SUBTYPE_P(bp, struct subtype, b);
+ *   struct base_t { ... };
+ *   struct subtype_t { int x; struct base_t b; } x;
+ *   struct base_t *bp = &x.base;
+ *   struct *sp = SUBTYPE_P(bp, struct subtype_t, b);
  * </pre>
  */
 #define SUBTYPE_P(p, subtype, basemember) \
@@ -223,4 +221,29 @@
 /** Macro: Yields the number of elements in array x. */
 #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
 
-#endif /* !defined(TOR_COMPAT_H) */
+/**
+ * "Eat" a semicolon that somebody puts at the end of a top-level macro.
+ *
+ * Frequently, we want to declare a macro that people will use at file scope,
+ * and we want to allow people to put a semicolon after the macro.
+ *
+ * This declaration of a struct can be repeated any number of times, and takes
+ * a trailing semicolon afterwards.
+ **/
+#define EAT_SEMICOLON                                   \
+  struct dummy_semicolon_eater__
+
+/**
+ * Tell our static analysis tool to believe that (clang's scan-build or
+ * coverity scan) that an expression might be true.  We use this to suppress
+ * dead-code warnings.
+ **/
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* By calling getenv, we force the analyzer not to conclude that 'expr' is
+ * false. */
+#define POSSIBLE(expr) ((expr) || getenv("STATIC_ANALYZER_DEADCODE_DUMMY_"))
+#else
+#define POSSIBLE(expr) (expr)
+#endif /* defined(__COVERITY__) || defined(__clang_analyzer__) */
+
+#endif /* !defined(TOR_COMPAT_COMPILER_H) */
diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h
new file mode 100644
index 0000000000..d9d3aa40b0
--- /dev/null
+++ b/src/lib/cc/ctassert.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ctassert.h
+ *
+ * \brief Compile-time assertions: CTASSERT(expression).
+ */
+
+#ifndef TOR_CTASSERT_H
+#define TOR_CTASSERT_H
+
+#include "lib/cc/compat_compiler.h"
+
+/**
+ * CTASSERT(expression)
+ *
+ *       Trigger a compiler error if expression is false.
+ */
+#if __STDC_VERSION__ >= 201112L
+
+/* If C11 is available, just use _Static_assert.  */
+#define CTASSERT(x) _Static_assert((x), #x)
+
+#else /* !(__STDC_VERSION__ >= 201112L) */
+
+/*
+ * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__
+ * and __LINE__, or just __LINE__, with an intermediate preprocessor
+ * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the
+ * expansions together into a unique name.
+ *
+ * We use this name as a typedef of an array type with a positive
+ * length if the assertion is true, and a negative length of the
+ * assertion is false, which is invalid and hence triggers a compiler
+ * error.
+ */
+#if defined(__COUNTER__)
+#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__)
+#elif defined(__INCLUDE_LEVEL__)
+#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__)
+#else
+/* hope it's unique enough */
+#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__)
+#endif /* defined(__COUNTER__) || ... */
+
+#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b)
+#define CTASSERT_DECL(x, a, b) \
+  typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED; EAT_SEMICOLON
+
+#endif /* __STDC_VERSION__ >= 201112L */
+
+#endif /* !defined(TOR_CTASSERT_H) */
diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am
index 2ae90f97dd..d2a415e956 100644
--- a/src/lib/cc/include.am
+++ b/src/lib/cc/include.am
@@ -1,4 +1,7 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/cc/compat_compiler.h \
+	src/lib/cc/ctassert.h \
+	src/lib/cc/tokpaste.h \
 	src/lib/cc/torint.h
diff --git a/src/lib/cc/lib_cc.md b/src/lib/cc/lib_cc.md
new file mode 100644
index 0000000000..bd49005ba2
--- /dev/null
+++ b/src/lib/cc/lib_cc.md
@@ -0,0 +1,2 @@
+@dir /lib/cc
+@brief lib/cc: Macros for managing the C compiler and language.
diff --git a/src/lib/cc/tokpaste.h b/src/lib/cc/tokpaste.h
new file mode 100644
index 0000000000..068621b5bd
--- /dev/null
+++ b/src/lib/cc/tokpaste.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file tokpaste.h
+ * @brief Token-pasting macros.
+ **/
+
+#ifndef TOR_LIB_CC_TOKPASTE_H
+#define TOR_LIB_CC_TOKPASTE_H
+
+/**
+ * Concatenate `a` and `b` in a way that allows their result itself to be
+ * expanded by the preprocessor.
+ *
+ * Ordinarily you could just say `a ## b` in a macro definition.  But doing so
+ * results in a symbol which the preprocessor will not then expand.  If you
+ * wanted to use `a ## b` to create the name of a macro and have the
+ * preprocessor expand _that_ macro, you need to have another level of
+ * indirection, as this macro provides.
+ **/
+#define PASTE(a,b) PASTE__(a,b)
+
+/** Helper for PASTE(). */
+#define PASTE__(a,b) a ## b
+
+#endif /* !defined(TOR_LIB_CC_TOKPASTE_H) */
diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h
index c9b2d329f2..af7a90431c 100644
--- a/src/lib/cc/torint.h
+++ b/src/lib/cc/torint.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,7 +49,7 @@ typedef int32_t ssize_t;
  * aren't 2's complement, and you don't define LONG_MAX, then you're so
  * bizarre that I want nothing to do with you. */
 #ifndef USING_TWOS_COMPLEMENT
-#error "Seems that your platform doesn't use 2's complement arithmetic. Argh."
+#error "Your platform doesn't use 2's complement arithmetic."
 #endif
 
 #ifndef TIME_MAX
@@ -96,9 +96,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIuSZ PRIu32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIuSZ "zu"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef _WIN32
 #  ifdef _WIN64
@@ -106,9 +106,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIdSZ PRId32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIdSZ "zd"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef SSIZE_MAX
 #if (SIZEOF_SIZE_T == 4)
@@ -125,4 +125,12 @@ typedef int32_t ssize_t;
 /** Any size_t larger than this amount is likely to be an underflow. */
 #define SIZE_T_CEILING  ((size_t)(SSIZE_MAX-16))
 
+#if SIZEOF_INT > SIZEOF_VOID_P
+#error "sizeof(int) > sizeof(void *) - Can't build Tor here."
+#endif
+
+#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P
+#error "sizeof(unsigned int) > sizeof(void *) - Can't build Tor here."
+#endif
+
 #endif /* !defined(TOR_TORINT_H) */
diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include
index 68fe9f1c54..6cd80086e6 100644
--- a/src/lib/compress/.may_include
+++ b/src/lib/compress/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/compress/*.h
 lib/container/*.h
@@ -8,5 +9,6 @@ lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c
index 95fd73bb32..7ce3910d84 100644
--- a/src/lib/compress/compress.c
+++ b/src/lib/compress/compress.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,10 +29,12 @@
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_none.h"
+#include "lib/compress/compress_sys.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
 #include "lib/intmath/cmp.h"
 #include "lib/malloc/malloc.h"
+#include "lib/subsys/subsys.h"
 #include "lib/thread/threads.h"
 
 /** Total number of bytes allocated for compression state overhead. */
@@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state)
 }
 
 /** Initialize all compression modules. */
-void
+int
 tor_compress_init(void)
 {
   atomic_counter_init(&total_compress_allocation);
@@ -668,6 +670,8 @@ tor_compress_init(void)
   tor_zlib_init();
   tor_lzma_init();
   tor_zstd_init();
+
+  return 0;
 }
 
 /** Warn if we had any problems while setting up our compression libraries.
@@ -677,5 +681,21 @@ tor_compress_init(void)
 void
 tor_compress_log_init_warnings(void)
 {
+  // XXXX can we move this into tor_compress_init() after all?  log.c queues
+  // XXXX log messages at startup.
   tor_zstd_warn_if_version_mismatched();
 }
+
+static int
+subsys_compress_initialize(void)
+{
+  return tor_compress_init();
+}
+
+const subsys_fns_t sys_compress = {
+  .name = "compress",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -55,
+  .initialize = subsys_compress_initialize,
+};
diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h
index 5f16a2ab27..f36cdb82aa 100644
--- a/src/lib/compress/compress.h
+++ b/src/lib/compress/compress.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state);
 
 size_t tor_compress_state_size(const tor_compress_state_t *state);
 
-void tor_compress_init(void);
+int tor_compress_init(void);
 void tor_compress_log_init_warnings(void);
 
 struct buf_t;
diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c
index 198128b261..d1941c9da6 100644
--- a/src/lib/compress/compress_buf.c
+++ b/src/lib/compress/compress_buf.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/cc/compat_compiler.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/compress/compress.h"
 #include "lib/log/util_bug.h"
 
diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c
index 2dab37e433..8884b020e8 100644
--- a/src/lib/compress/compress_lzma.c
+++ b/src/lib/compress/compress_lzma.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress,
   tor_free(result);
   return NULL;
  /* LCOV_EXCL_STOP */
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)compress;
   (void)method;
   (void)level;
@@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state,
                lzma_error_str(retval));
       return TOR_COMPRESS_ERROR;
   }
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)state;
   (void)out;
   (void)out_len;
diff --git a/src/lib/compress/compress_lzma.h b/src/lib/compress/compress_lzma.h
index 556ab437dc..de03cda91c 100644
--- a/src/lib/compress/compress_lzma.h
+++ b/src/lib/compress/compress_lzma.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_none.c b/src/lib/compress/compress_none.c
index 0b5760773a..43c021c788 100644
--- a/src/lib/compress/compress_none.c
+++ b/src/lib/compress/compress_none.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_none.h b/src/lib/compress/compress_none.h
index 2bb9c3d66c..df696a11aa 100644
--- a/src/lib/compress/compress_none.h
+++ b/src/lib/compress/compress_none.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h
new file mode 100644
index 0000000000..dce0549924
--- /dev/null
+++ b/src/lib/compress/compress_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_sys.h
+ * \brief Declare subsystem object for the compress module
+ **/
+
+#ifndef TOR_COMPRESS_SYS_H
+#define TOR_COMPRESS_SYS_H
+
+extern const struct subsys_fns_t sys_compress;
+
+#endif /* !defined(TOR_COMPRESS_SYS_H) */
diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c
index df0d1bff5f..3711e46222 100644
--- a/src/lib/compress/compress_zlib.c
+++ b/src/lib/compress/compress_zlib.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_zlib.h b/src/lib/compress/compress_zlib.h
index e4f248cd9b..7328ce899b 100644
--- a/src/lib/compress/compress_zlib.h
+++ b/src/lib/compress/compress_zlib.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c
index 45d0d4d602..5913d823e1 100644
--- a/src/lib/compress/compress_zstd.c
+++ b/src/lib/compress/compress_zstd.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,17 +25,17 @@
  * all invocations of zstd's static-only functions in a check to make sure
  * that the compile-time version matches the run-time version. */
 #define ZSTD_STATIC_LINKING_ONLY
-#endif
+#endif /* defined(ENABLE_ZSTD_ADVANCED_APIS) */
 
 #ifdef HAVE_ZSTD
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
-DISABLE_GCC_WARNING(unused-const-variable)
+DISABLE_GCC_WARNING("-Wunused-const-variable")
 #endif
 #include <zstd.h>
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
-ENABLE_GCC_WARNING(unused-const-variable)
-#endif
+ENABLE_GCC_WARNING("-Wunused-const-variable")
 #endif
+#endif /* defined(HAVE_ZSTD) */
 
 /** Total number of bytes allocated for Zstandard state. */
 static atomic_counter_t total_zstd_allocation;
@@ -77,7 +77,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
                version_number / 100 % 100,
                version_number % 100);
 }
-#endif
+#endif /* defined(HAVE_ZSTD) */
 
 #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
 
@@ -93,7 +93,7 @@ tor_zstd_get_version_str(void)
                           ZSTD_versionNumber());
 
   return version_str;
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   return NULL;
 #endif /* defined(HAVE_ZSTD) */
 }
@@ -125,9 +125,9 @@ tor_zstd_can_use_static_apis(void)
   }
 #endif
   return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
-#else
+#else /* !(defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)) */
   return 0;
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) */
 }
 
 /** Internal Zstandard state for incremental compression/decompression.
@@ -237,7 +237,7 @@ tor_zstd_state_size_precalc(int compress, int preset)
 #endif
     }
   }
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) */
   return tor_zstd_state_size_precalc_fake(compress, preset);
 }
 #endif /* defined(HAVE_ZSTD) */
@@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress,
   tor_free(result);
   return NULL;
   // LCOV_EXCL_STOP
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)compress;
   (void)method;
   (void)level;
@@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state,
       return TOR_COMPRESS_OK;
   }
 
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)state;
   (void)out;
   (void)out_len;
@@ -527,7 +527,7 @@ tor_zstd_warn_if_version_mismatched(void)
              "For safety, we'll avoid using advanced zstd functionality.",
              header_version, runtime_version);
   }
-#endif
+#endif /* defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) */
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -538,4 +538,4 @@ tor_zstd_set_static_apis_disabled_for_testing(int disabled)
 {
   static_apis_disable_for_testing = disabled;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/compress/compress_zstd.h b/src/lib/compress/compress_zstd.h
index 47f950b9e0..0fc71db749 100644
--- a/src/lib/compress/compress_zstd.h
+++ b/src/lib/compress/compress_zstd.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am
index 75c9032bd2..60dd447d4e 100644
--- a/src/lib/compress/include.am
+++ b/src/lib/compress/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-compress-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_compress_a_SOURCES =		\
 	src/lib/compress/compress.c		\
 	src/lib/compress/compress_buf.c		\
@@ -18,9 +19,11 @@ src_lib_libtor_compress_testing_a_SOURCES = \
 src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS	+=				\
 	src/lib/compress/compress.h		\
 	src/lib/compress/compress_lzma.h	\
 	src/lib/compress/compress_none.h	\
+	src/lib/compress/compress_sys.h		\
 	src/lib/compress/compress_zlib.h	\
 	src/lib/compress/compress_zstd.h
diff --git a/src/lib/compress/lib_compress.md b/src/lib/compress/lib_compress.md
new file mode 100644
index 0000000000..c43f223fe7
--- /dev/null
+++ b/src/lib/compress/lib_compress.md
@@ -0,0 +1,6 @@
+@dir /lib/compress
+@brief lib/compress: Wraps several compression libraries
+
+Currently supported are zlib (mandatory), zstd (optional), and lzma
+(optional).
+
diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include
new file mode 100644
index 0000000000..629e2f897d
--- /dev/null
+++ b/src/lib/conf/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
diff --git a/src/lib/conf/confdecl.h b/src/lib/conf/confdecl.h
new file mode 100644
index 0000000000..c2d3fb335d
--- /dev/null
+++ b/src/lib/conf/confdecl.h
@@ -0,0 +1,221 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file confdecl.h
+ * @brief Macros for generating a configuration struct from a list
+ *   of its individual fields.
+ *
+ * This header defines three important macros: BEGIN_CONF_STRUCT(),
+ * END_CONF_STRUCT(), and CONF_VAR().  They're meant to be used together to
+ * define a configuration structure and the means for encoding and decoding
+ * it.
+ *
+ * To use them, make a new header with a name like `MOD_options.inc`.  Start
+ * it with a BEGIN_CONF_STRUCT(), then define your variables with CONF_VAR(),
+ * then end the header with END_CONF_STRUCT(), as in:
+ *
+ *     BEGIN_CONF_STRUCT(module_options_t)
+ *     CONF_VAR(ModuleIsActive, BOOLEAN, 0, "1")
+ *     END_CONF_STRUCT(module_options_t)
+ *
+ * Once you've done that, you can use that header to define a configuration
+ * structure by saying:
+ *
+ *     typedef struct module_options_t module_options_t;
+ *     #define CONF_CONTEXT STRUCT
+ *     #include "MOD_options.inc"
+ *     #undef CONF_CONTEXT
+ *
+ * And you can define your field definition table by saying:
+ *
+ *     #define CONF_CONTEXT TABLE
+ *     #include "MOD_options.inc"
+ *     #undef CONF_CONTEXT
+ *
+ * The two above snippets will define a structure called `module_options_t`
+ * with appropriate members, and a table of config_var_t objects called
+ * `module_options_t_vars[]`.
+ *
+ * For lower-level modules, you can say <tt>\#define CONF_TABLE LL_TABLE</tt>,
+ * and get a table definition suitable for use in modules that are at a lower
+ * level than lib/confmgt.  Note that the types for these tables cannot
+ * include any extended types.
+ **/
+
+#ifndef TOR_LIB_CONF_CONFDECL_H
+#define TOR_LIB_CONF_CONFDECL_H
+
+#undef CONF_CONTEXT
+#include "lib/cc/tokpaste.h"
+#include "lib/cc/torint.h"
+
+/**
+ * Begin the definition of a configuration object called `name`.
+ **/
+#define BEGIN_CONF_STRUCT(name) \
+  PASTE(BEGIN_CONF_STRUCT__, CONF_CONTEXT)(name)
+/**
+ * End the definition of a configuration object called `name`.
+ **/
+#define END_CONF_STRUCT(name) \
+  PASTE(END_CONF_STRUCT__, CONF_CONTEXT)(name)
+/**
+ * Declare a single configuration field with name `varname`, type `vartype`,
+ * flags `varflags`, and initial value `initval`.
+ **/
+#define CONF_VAR(varname, vartype, varflags, initval)   \
+  PASTE(CONF_VAR__, CONF_CONTEXT)(varname, vartype, varflags, initval)
+
+#ifndef COCCI
+/**
+ * @defgroup STRUCT_MACROS Internal macros: struct definitions.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to STRUCT.  Don't use them directly.
+ * @{*/
+#define BEGIN_CONF_STRUCT__STRUCT(name)         \
+  struct name {                                 \
+  uint32_t magic;
+#define END_CONF_STRUCT__STRUCT(name)           \
+  };
+#define CONF_VAR__STRUCT(varname, vartype, varflags, initval)   \
+  config_decl_ ## vartype varname;
+/** @} */
+
+/**
+ * @defgroup TABLE_MACROS Internal macros: table definitions.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to TABLE.  Don't use them directly.
+ * @{*/
+#define BEGIN_CONF_STRUCT__TABLE(structname)                            \
+  /* We use this typedef so we can refer to the config type */          \
+  /* without having its name as a macro argument to CONF_VAR. */        \
+  typedef struct structname config_var_reference__obj;  \
+  static const config_var_t structname##_vars[] = {
+#define END_CONF_STRUCT__TABLE(structname)      \
+  { .member = { .name = NULL } }                \
+    };
+#define CONF_VAR__TABLE(varname, vartype, varflags, initval)    \
+  {                                                             \
+   .member =                                                    \
+   { .name = #varname,                                          \
+     .type = CONFIG_TYPE_EXTENDED,                              \
+     .type_def = &vartype ## _type_defn,                        \
+     .offset=offsetof(config_var_reference__obj, varname),      \
+   },                                                           \
+   .flags = varflags,                                           \
+   .initvalue = initval                                         \
+  },
+/**@}*/
+
+/**
+ * @defgroup LL_TABLE_MACROS Internal macros: low-level table definitions.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to LL_TABLE.  Don't use them directly.
+ * @{*/
+#define BEGIN_CONF_STRUCT__LL_TABLE(structname)                         \
+  /* We use this typedef so we can refer to the config type */          \
+  /* without having its name as a macro argument to CONF_VAR. */        \
+  typedef struct structname config_var_reference__obj;  \
+  static const config_var_t structname##_vars[] = {
+#define END_CONF_STRUCT__LL_TABLE(structname)   \
+  { .member = { .name = NULL } }                \
+    };
+#define CONF_VAR__LL_TABLE(varname, vartype, varflags, initval) \
+  {                                                             \
+   .member =                                                    \
+   { .name = #varname,                                          \
+     .type = CONFIG_TYPE_ ## vartype,                           \
+     .offset=offsetof(config_var_reference__obj, varname),      \
+   },                                                           \
+   .flags = varflags,                                           \
+   .initvalue = initval                                         \
+  },
+/**@}*/
+
+/** @defgroup STUB_TABLE_MACROS Internal macros: stub table declarations,
+ * for use when a module is disabled.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to LL_TABLE.  Don't use them directly.
+ * @{ */
+#define BEGIN_CONF_STRUCT__STUB_TABLE(structname)                       \
+  static const config_var_t structname##_vars[] = {
+#define END_CONF_STRUCT__STUB_TABLE(structname)   \
+  { .member = { .name = NULL } }                \
+    };
+#define CONF_VAR__STUB_TABLE(varname, vartype, varflags, initval)       \
+  {                                                             \
+   .member =                                                    \
+   { .name = #varname,                                          \
+     .type = CONFIG_TYPE_IGNORE,                                \
+     .offset = -1,                                              \
+   },                                                           \
+   .flags = CFLG_GROUP_DISABLED,                                \
+  },
+/**@}*/
+
+#endif /* !defined(COCCI) */
+
+/** Type aliases for the "commonly used" configuration types.
+ *
+ * Defining them in this way allows our CONF_VAR__STRUCT() macro to declare
+ * structure members corresponding to the configuration types.  For example,
+ * when the macro sees us declare a configuration option "foo" of type STRING,
+ * it can emit `config_decl_STRING foo;`, which is an alias for `char *foo`.
+ */
+/**@{*/
+typedef char *config_decl_STRING;
+typedef char *config_decl_FILENAME;
+/* Yes, "POSINT" is really an int, and not an unsigned int.  For
+ * historical reasons, many configuration values are restricted
+ * to the range [0,INT_MAX], and stored in signed ints.
+ */
+typedef int config_decl_POSINT;
+typedef uint64_t config_decl_UINT64;
+typedef int config_decl_INT;
+typedef int config_decl_INTERVAL;
+typedef int config_decl_MSEC_INTERVAL;
+typedef uint64_t config_decl_MEMUNIT;
+typedef double config_decl_DOUBLE;
+typedef int config_decl_BOOL;
+typedef int config_decl_AUTOBOOL;
+typedef time_t config_decl_ISOTIME;
+typedef struct smartlist_t config_decl_CSV;
+typedef int config_decl_CSV_INTERVAL;
+typedef struct config_line_t *config_decl_LINELIST;
+typedef struct config_line_t *config_decl_LINELIST_V;
+typedef struct nonexistent_struct *config_decl_LINELIST_S;
+/**@}*/
+
+struct var_type_def_t;
+
+/* Forward declarations for configuration type definitions. These are used by
+ * the CONF_VAR__TABLE macro to set the definition of each variable type
+ * correctly.
+ */
+/**@{*/
+extern const struct var_type_def_t STRING_type_defn;
+extern const struct var_type_def_t FILENAME_type_defn;
+extern const struct var_type_def_t POSINT_type_defn;
+extern const struct var_type_def_t UINT64_type_defn;
+extern const struct var_type_def_t INT_type_defn;
+extern const struct var_type_def_t INTERVAL_type_defn;
+extern const struct var_type_def_t MSEC_INTERVAL_type_defn;
+extern const struct var_type_def_t MEMUNIT_type_defn;
+extern const struct var_type_def_t DOUBLE_type_defn;
+extern const struct var_type_def_t BOOL_type_defn;
+extern const struct var_type_def_t AUTOBOOL_type_defn;
+extern const struct var_type_def_t ISOTIME_type_defn;
+extern const struct var_type_def_t CSV_type_defn;
+extern const struct var_type_def_t CSV_INTERVAL_type_defn;
+extern const struct var_type_def_t LINELIST_type_defn;
+extern const struct var_type_def_t LINELIST_V_type_defn;
+extern const struct var_type_def_t LINELIST_S_type_defn;
+extern const struct var_type_def_t IGNORE_type_defn;
+extern const struct var_type_def_t OBSOLETE_type_defn;
+/**@}*/
+
+#endif /* !defined(TOR_LIB_CONF_CONFDECL_H) */
diff --git a/src/lib/conf/config.md b/src/lib/conf/config.md
new file mode 100644
index 0000000000..7741e21f42
--- /dev/null
+++ b/src/lib/conf/config.md
@@ -0,0 +1,126 @@
+
+@page configuration Configuration options and persistent state
+
+@tableofcontents
+
+## Introduction
+
+Tor uses a shared, table-driven mechanism to handle its
+configuration (torrc) files and its state files.  Each module can
+declare a set of named fields for these files, and get notified
+whenever the configuration changes, or when the state is about to be
+flushed to disk.
+
+## Declaring options
+
+Most modules will only need to use the macros in confdecl.h to
+declare a configuration or state structure.
+
+You'll write something like this:
+
+    // my_module_config.inc
+    BEGIN_CONF_STRUCT(module_options_t)
+    CONF_VAR(FieldOne, INT, 0, "7")
+    CONF_VAR(FieldTwo, STRING, 0, NULL)
+    END_CONF_STRUCT(module_options_t)
+
+The above example will result in a structure called module_config_t
+with two fields: one an integer called FieldOne and one a string
+called FieldTwo.  The integer gets a default value of 7; the
+string's default value is NULL.
+
+After making a definition file like that, you include it twice: once
+in a header, after saying \#define CONF_CONTEXT STRUCT, and once in
+a C file, after saying \#define CONF_CONTEXT TABLE.  The first time
+defines a module_options_t structure, and the second time defines a
+table that tells the configuration manager how to use it.
+
+Using the table, you declare a `const` config_format_t, which
+associates the fields with a set of functions for validating and
+normalizing them, a list of abbreviations and deprecations, and
+other features.
+
+See confdecl.h and conftypes.h for more information. For example
+usage, see crypto_options.inc or mainloop_state.inc.
+
+## Getting notifications
+
+After using those macros, you must tell the subsystem management
+code about your module's configuration/state.
+
+If you're writing configuration code, you'll need a function that receives
+the configuration object, and acts upon it.  This function needs to be safe
+to call multiple times, since Tor will reconfigure its subsystems whenever it
+re-reads the torrc, gets a configuration change from a controller, or
+restarts in process.  This function goes in your subsystem's
+subsys_fns_t.set_options field.
+
+If you're writing state code, you'll need a function that receives
+state (subsys_fns_t.set_state), and a function that flushes the
+application state into a state object (subsys_fns_t.flush_state).
+The `set_state` function will be called once (@ref config_once_per
+"1") when Tor is starting, whereas the `flush_state` function will
+be called whenever Tor is about to save the state to disk.
+
+See subsys_fns_t for more information here, and \ref initialization
+for more information about initialization and subsystems in general.
+
+> @anchor config_once_per 1. Technically, state is set once _per startup_.
+> Remember that Tor can be stopped and started multiple times in
+> the same process.  If this happens, then your set_state() function
+> is called once every time Tor starts.
+
+## How it works
+
+The common logic used to handle configuration and state files lives
+in @refdir{lib/confmgt}.  At the highest level, a configuration
+manager object (config_mgr_t) maintains a list of each module's
+configuration objects, and a list of all their fields.  When the
+user specifies a configuration value, the manager finds out how to
+parse it, where to store it, and which configuration object is
+affected.
+
+The top-level configuration module (config.c) and state module
+(statefile.c) use config_mgr_t to create, initialize, set, compare,
+and free a "top level configuration object".  This object contains a
+list of sub-objects: one for each module that participates in the
+configuration/state system.  This top-level code then invokes the
+subsystem manager code (subsysmgr.c) to pass the corresponding
+configuration or state objects to each module that has one.
+
+Note that the top level code does not have easy access to the
+configuration objects used by the sub-modules.  This is by design.  A
+module _may_ expose some or all of its configuration or state object via
+accessor functions, if it likes, but if it does not, that object should
+be considered module-local.
+
+## Adding new types
+
+Configuration and state fields each have a "type".  These types
+specify how the fields' values are represented in C; how they are
+stored in files; and how they are encoded back and forth.
+
+There is a set of built-in types listed in conftypes.h, but
+higher-level code can define its own types.  To do so, you make an
+instance of var_type_fns_t that describes how to manage your type,
+and an instance of var_type_def_t that wraps your var_type_fns_t
+with a name and optional parameters and flags.
+
+For an example of how a higher-level type is defined, see
+ROUTERSET_type_defn in routerset.c.  Also see the typedef
+`config_decl_ROUTERSET`.  Together, these let the routerset type be
+used with the macros in confdecl.h.
+
+## Legacy configuration and state
+
+As of this writing (November 2019), most of the configuration and state is
+still handled directly in config.c and statefile.c, and stored in the
+monolithic structures or_options_t and or_state_t respectively.
+
+These top-level structures are accessed with get_options() and
+get_state(), and used throughout much of the code, at the level of
+@refdir{core} and higher.
+
+With time we hope to refactor this configuration into more
+reasonable pieces, so that they are no longer (effectively) global
+variables used throughout the code.
diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h
new file mode 100644
index 0000000000..9f85d21740
--- /dev/null
+++ b/src/lib/conf/confmacros.h
@@ -0,0 +1,72 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file confmacros.h
+ * @brief Macro definitions for declaring configuration variables
+ **/
+
+#ifndef TOR_LIB_CONF_CONFMACROS_H
+#define TOR_LIB_CONF_CONFMACROS_H
+
+#include "orconfig.h"
+#include "lib/conf/conftesting.h"
+
+#ifndef COCCI
+/**
+ * Used to indicate the end of an array of configuration variables.
+ **/
+#define END_OF_CONFIG_VARS                                      \
+  { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS }
+#endif /* !defined(COCCI) */
+
+/**
+ * Declare a config_var_t as a member named <b>membername</b> of the structure
+ * <b>structtype</b>, whose user-visible name is <b>varname</b>, whose
+ * type corresponds to the config_type_t member CONFIG_TYPE_<b>vartype</b>,
+ * and whose initial value is <b>intval</b>.
+ *
+ * Most modules that use this macro should wrap it in a local macro that
+ * sets structtype to the local configuration type.
+ **/
+#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername,      \
+                         varflags, initval)                             \
+  { .member =                                                           \
+    { .name = varname,                                                  \
+      .type = CONFIG_TYPE_ ## vartype,                                  \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+/**
+ * As CONFIG_VAR_ETYPE, but declares a value using an extension type whose
+ * type definition is <b>vartype</b>_type_defn.
+ **/
+#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername,       \
+                        varflags, initval)                              \
+  { .member =                                                           \
+    { .name = varname,                                                \
+      .type = CONFIG_TYPE_EXTENDED,                                     \
+      .type_def = &vartype ## _type_defn,                               \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+/**
+ * Declare an obsolete configuration variable with a given name.
+ **/
+#define CONFIG_VAR_OBSOLETE(varname)            \
+  { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE },        \
+    .flags = CFLG_GROUP_OBSOLETE                                        \
+  }
+
+#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */
diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h
new file mode 100644
index 0000000000..4707c919d3
--- /dev/null
+++ b/src/lib/conf/conftesting.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftesting.h
+ * @brief Macro and type declarations for testing
+ **/
+
+#ifndef TOR_LIB_CONF_CONFTESTING_H
+#define TOR_LIB_CONF_CONFTESTING_H
+
+#include "lib/cc/torint.h"
+
+#ifndef COCCI
+#ifdef TOR_UNIT_TESTS
+#define USE_CONF_TESTING
+/**
+ * Union used when building in test mode typechecking the members of a type
+ * used with confmgt.c.  See CONF_CHECK_VAR_TYPE for a description of how
+ * it is used. */
+typedef union {
+  char **STRING;
+  char **FILENAME;
+  int *POSINT; /* yes, this is really an int, and not an unsigned int.  For
+                * historical reasons, many configuration values are restricted
+                * to the range [0,INT_MAX], and stored in signed ints.
+                */
+  uint64_t *UINT64;
+  int *INT;
+  int *INTERVAL;
+  int *MSEC_INTERVAL;
+  uint64_t *MEMUNIT;
+  double *DOUBLE;
+  int *BOOL;
+  int *AUTOBOOL;
+  time_t *ISOTIME;
+  struct smartlist_t **CSV;
+  int *CSV_INTERVAL;
+  struct config_line_t **LINELIST;
+  struct config_line_t **LINELIST_S;
+  struct config_line_t **LINELIST_V;
+  // XXXX this doesn't belong at this level of abstraction.
+  struct routerset_t **ROUTERSET;
+} confparse_dummy_values_t;
+
+/* Macros to define extra members inside config_var_t fields, and at the
+ * end of a list of them.
+ */
+/* This is a somewhat magic type-checking macro for users of confmgt.c.
+ * It initializes a union member "confparse_dummy_values_t.conftype" with
+ * the address of a static member "tp_dummy.member".   This
+ * will give a compiler warning unless the member field is of the correct
+ * type.
+ *
+ * (This warning is mandatory, because a type mismatch here violates the type
+ * compatibility constraint for simple assignment, and requires a diagnostic,
+ * according to the C spec.)
+ *
+ * For example, suppose you say:
+ *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
+ * Then this macro will evaluate to:
+ *     { .STRING = &or_options_t_dummy.Address }
+ * And since confparse_dummy_values_t.STRING has type "char **", that
+ * expression will create a warning unless or_options_t.Address also
+ * has type "char *".
+ */
+#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
+  { . conftype = &tp ## _dummy . member }
+#define CONF_TEST_MEMBERS(tp, conftype, member) \
+  , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member)
+#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL }
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  static tp tp ## _dummy
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* !defined(COCCI) */
+
+#ifndef USE_CONF_TESTING
+#define CONF_TEST_MEMBERS(tp, conftype, member)
+/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  struct tor_semicolon_eater
+#define DUMMY_CONF_TEST_MEMBERS
+
+#endif /* !defined(USE_CONF_TESTING) */
+
+#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */
diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h
new file mode 100644
index 0000000000..081ebf397f
--- /dev/null
+++ b/src/lib/conf/conftypes.h
@@ -0,0 +1,383 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftypes.h
+ * @brief Types used to specify configurable options.
+ *
+ * This header defines the types that different modules will use in order to
+ * declare their configuration and state variables, and tell the configuration
+ * management code about those variables.  From the individual module's point
+ * of view, its configuration and state are simply data structures.
+ *
+ * For defining new variable types, see var_type_def_st.h.
+ *
+ * For the code that manipulates variables defined via this module, see
+ * lib/confmgt/, especially typedvar.h and (later) structvar.h.  The
+ * configuration manager is responsible for encoding, decoding, and
+ * maintaining the configuration structures used by the various modules.
+ *
+ * STATUS NOTE: This is a work in process refactoring.  It is not yet possible
+ *   for modules to define their own variables, and much of the configuration
+ *   management code is still in src/app/config/.
+ **/
+
+#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H
+#define TOR_SRC_LIB_CONF_CONFTYPES_H
+
+#include "lib/cc/torint.h"
+#ifdef TOR_UNIT_TESTS
+#include "lib/conf/conftesting.h"
+#endif
+
+#include <stddef.h>
+
+/** Enumeration of types which option values can take */
+typedef enum config_type_t {
+  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
+  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
+  CONFIG_TYPE_POSINT,       /**< A non-negative integer less than MAX_INT */
+  CONFIG_TYPE_INT,          /**< Any integer. */
+  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
+  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
+  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
+                              * units */
+  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
+  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
+  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
+  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
+                             * 1 for true, and -1 for auto  */
+  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
+  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
+                              * optional whitespace. */
+  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
+                              * optional whitespace, representing intervals in
+                              * seconds, with optional units.  We allow
+                              * multiple values here for legacy reasons, but
+                              * ignore every value after the first. */
+  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
+  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
+                             * mixed with other keywords. */
+  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
+                             * context-sensitive config lines when fetching.
+                             */
+  /** Ignored (obsolete) option. Uses no storage.
+   *
+   * Reported as "obsolete" when its type is queried.
+   */
+  CONFIG_TYPE_OBSOLETE,
+  /** Ignored option. Uses no storage.
+   *
+   * Reported as "ignored" when its type is queried. For use with options used
+   * by disabled modules.
+   **/
+  CONFIG_TYPE_IGNORE,
+
+  /**
+   * Extended type: definition appears in the <b>type_def</b> pointer
+   * of the corresponding struct_member_t.
+   *
+   * For some types, we cannot define them as particular values of this
+   * enumeration, since those types are abstractions defined at a higher level
+   * than this module.  (For example, parsing a routerset_t is higher-level
+   * than this module.)  To handle this, we use CONFIG_TYPE_EXTENDED for those
+   * types, and give a definition for them in the struct_member_t.type_def.
+   **/
+  CONFIG_TYPE_EXTENDED,
+} config_type_t;
+
+/* Forward delcaration for var_type_def_t, for extended types. */
+struct var_type_def_t;
+
+/** Structure to specify a named, typed member within a structure. */
+typedef struct struct_member_t {
+  /** Name of the field. */
+  const char *name;
+  /**
+   * Type of the field, according to the config_type_t enumeration.
+   *
+   * For any type not otherwise listed in config_type_t, this field's value
+   * should be CONFIG_TYPE_EXTENDED.  When it is, the <b>type_def</b> pointer
+   * must be set.
+   **/
+  /*
+   * NOTE: In future refactoring, we might remove this field entirely, along
+   * with its corresponding enumeration.  In that case, we will require that
+   * type_def be set in all cases. If we do, we will also need a new mechanism
+   * to enforce consistency between configuration variable types and their
+   * corresponding structures, since our current design in
+   * lib/conf/conftesting.h won't work any more.
+   */
+  config_type_t type;
+  /**
+   * Pointer to a type definition for the type of this field. Overrides
+   * <b>type</b> if it is not NULL.  Must be set when <b>type</b> is
+   * CONFIG_TYPE_EXTENDED.
+   **/
+  const struct var_type_def_t *type_def;
+  /**
+   * Offset of this field within the structure.  Compute this with
+   * offsetof(structure, fieldname).
+   **/
+  ptrdiff_t offset;
+} struct_member_t;
+
+/**
+ * Structure to describe the location and preferred value of a "magic number"
+ * field within a structure.
+ *
+ * These 'magic numbers' are 32-bit values used to tag objects to make sure
+ * that they have the correct type.
+ *
+ * If all fields in this structure are zero or 0, the magic-number check is
+ * not performed.
+ */
+typedef struct struct_magic_decl_t {
+  /** The name of the structure */
+  const char *typename;
+  /** A value used to recognize instances of this structure. */
+  uint32_t magic_val;
+  /** The location within the structure at which we expect to find
+   * <b>magic_val</b>. */
+  ptrdiff_t magic_offset;
+} struct_magic_decl_t;
+
+/**
+ * Flag to indicate that an option or type is "undumpable". An
+ * undumpable option is never saved to disk.
+ *
+ * For historical reasons its name is usually is prefixed with __.
+ **/
+#define CFLG_NODUMP    (1u<<0)
+/**
+ * Flag to indicate that an option or type is "unlisted".
+ *
+ * We don't tell the controller about unlisted options when it asks for a
+ * list of them.
+ **/
+#define CFLG_NOLIST (1u<<1)
+/**
+ * Flag to indicate that an option or type is "unsettable".
+ *
+ * An unsettable option can never be set directly by name.
+ **/
+#define CFLG_NOSET (1u<<2)
+/**
+ * Flag to indicate that an option or type does not need to be copied when
+ * copying the structure that contains it.
+ *
+ * (Usually, if an option does not need to be copied, then either it contains
+ * no data, or the data that it does contain is completely contained within
+ * another option.)
+ **/
+#define CFLG_NOCOPY (1u<<3)
+/**
+ * Flag to indicate that an option or type does not need to be compared
+ * when telling the controller about the differences between two
+ * configurations.
+ *
+ * (Usually, if an option does not need to be compared, then either it
+ * contains no data, or the data that it does contain is completely contained
+ * within another option.)
+ **/
+#define CFLG_NOCMP (1u<<4)
+/**
+ * Flag to indicate that an option or type should not be replaced when setting
+ * it.
+ *
+ * For most options, setting them replaces their old value.  For some options,
+ * however, setting them appends to their old value.
+ */
+#define CFLG_NOREPLACE    (1u<<5)
+/**
+ * Flag to indicate that an option or type cannot be changed while Tor is
+ * running.
+ **/
+#define CFLG_IMMUTABLE (1u<<6)
+/**
+ * Flag to indicate that we should warn that an option or type is obsolete
+ * whenever the user tries to use it.
+ **/
+#define CFLG_WARN_OBSOLETE (1u<<7)
+/**
+ * Flag to indicate that we should warn that an option applies only to
+ * a disabled module, whenever the user tries to use it.
+ **/
+#define CFLG_WARN_DISABLED (1u<<8)
+
+/**
+ * A group of flags that should be set on all obsolete options and types.
+ **/
+#define CFLG_GROUP_OBSOLETE \
+  (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST|\
+   CFLG_WARN_OBSOLETE)
+
+/**
+ * A group of fflags that should be set on all disabled options.
+ **/
+#define CFLG_GROUP_DISABLED \
+  (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST|\
+   CFLG_WARN_DISABLED)
+
+/** A variable allowed in the configuration file or on the command line. */
+typedef struct config_var_t {
+  struct_member_t member; /** A struct member corresponding to this
+                           * variable. */
+  const char *initvalue; /**< String (or null) describing initial value. */
+  uint32_t flags; /**< One or more flags describing special handling for this
+                   * variable */
+#ifdef TOR_UNIT_TESTS
+  /** Used for compiler-magic to typecheck the corresponding field in the
+   * corresponding struct. Only used in unit test mode, at compile-time. */
+  confparse_dummy_values_t var_ptr_dummy;
+#endif
+} config_var_t;
+
+/**
+ * An abbreviation or alias for a configuration option.
+ **/
+typedef struct config_abbrev_t {
+  /** The option name as abbreviated.  Not case-sensitive. */
+  const char *abbreviated;
+  /** The full name of the option. Not case-sensitive. */
+  const char *full;
+  /** True if this abbreviation should only be allowed on the command line. */
+  int commandline_only;
+  /** True if we should warn whenever this abbreviation is used. */
+  int warn;
+} config_abbrev_t;
+
+/**
+ * A note that a configuration option is deprecated, with an explanation why.
+ */
+typedef struct config_deprecation_t {
+  /** The option that is deprecated. */
+  const char *name;
+  /** A user-facing string explaining why the option is deprecated. */
+  const char *why_deprecated;
+} config_deprecation_t;
+
+#ifndef COCCI
+/**
+ * Handy macro for declaring "In the config file or on the command line, you
+ * can abbreviate <b>tok</b>s as <b>tok</b>". Used inside an array of
+ * config_abbrev_t.
+ *
+ * For example, to declare "NumCpu" as an abbreviation for "NumCPUs",
+ * you can say PLURAL(NumCpu).
+ **/
+#define PLURAL(tok) { (#tok), (#tok "s"), 0, 0 }
+#endif /* !defined(COCCI) */
+
+/**
+ * Validation function: verify whether a configuation object is well-formed
+ * and consistent.
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*validate_fn_t)(const void *value, char **msg_out);
+/**
+ * Validation function: verify whether a configuration object (`value`) is an
+ * allowable value given the previous configuration value (`old_value`).
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*check_transition_fn_t)(const void *old_value, const void *value,
+                                     char **msg_out);
+/**
+ * Validation function: normalize members of `value`, and compute derived
+ * members.
+ *
+ * This function is called before any other validation of `value`, and must
+ * not assume that validate_fn or check_transition_fn has passed.
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*pre_normalize_fn_t)(void *value, char **msg_out);
+/**
+ * Validation function: normalize members of `value`, and compute derived
+ * members.
+ *
+ * This function is called after validation of `value`, and may
+ * assume that validate_fn or check_transition_fn has passed.
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*post_normalize_fn_t)(void *value, char **msg_out);
+
+/**
+ * Legacy function to validate whether a given configuration is
+ * well-formed and consistent.
+ *
+ * The configuration to validate is passed as <b>newval</b>. The previous
+ * configuration, if any, is provided in <b>oldval</b>.
+ *
+ * This API is deprecated, since it mixes the responsibilities of
+ * pre_normalize_fn_t, post_normalize_fn_t, validate_fn_t, and
+ * check_transition_fn_t.  No new instances of this function type should
+ * be written.
+ *
+ * On success, return 0.  On failure, set *<b>msg_out</b> to a newly allocated
+ * error message, and return -1.
+ */
+typedef int (*legacy_validate_fn_t)(const void *oldval,
+                                    void *newval,
+                                    char **msg_out);
+
+struct config_mgr_t;
+
+/**
+ * Callback to clear all non-managed fields of a configuration object.
+ *
+ * <b>obj</b> is the configuration object whose non-managed fields should be
+ * cleared.
+ *
+ * (Regular fields get cleared by config_reset(), but you might have fields
+ * in the object that do not correspond to configuration variables.  If those
+ * fields need to be cleared or freed, this is where to do it.)
+ */
+typedef void (*clear_cfg_fn_t)(const struct config_mgr_t *mgr, void *obj);
+
+/** Information on the keys, value types, key-to-struct-member mappings,
+ * variable descriptions, validation functions, and abbreviations for a
+ * configuration or storage format. */
+typedef struct config_format_t {
+  size_t size; /**< Size of the struct that everything gets parsed into. */
+  struct_magic_decl_t magic; /**< Magic number info for this struct. */
+  const config_abbrev_t *abbrevs; /**< List of abbreviations that we expand
+                             * when parsing this format. */
+  const config_deprecation_t *deprecations; /** List of deprecated options */
+  const config_var_t *vars; /**< List of variables we recognize, their default
+                             * values, and where we stick them in the
+                             * structure. */
+
+  /** Early-stage normalization callback. Invoked by config_validate(). */
+  pre_normalize_fn_t pre_normalize_fn;
+  /** Configuration validation function. Invoked by config_validate(). */
+  validate_fn_t validate_fn;
+    /** Legacy validation function. Invoked by config_validate(). */
+  legacy_validate_fn_t legacy_validate_fn;
+  /** Transition checking function. Invoked by config_validate(). */
+  check_transition_fn_t check_transition_fn;
+  /** Late-stage normalization callback. Invoked by config_validate(). */
+  post_normalize_fn_t post_normalize_fn;
+
+  clear_cfg_fn_t clear_fn; /**< Function to clear the configuration. */
+  /** If present, extra denotes a LINELIST variable for unrecognized
+   * lines.  Otherwise, unrecognized lines are an error. */
+  const struct_member_t *extra;
+  /**
+   * If true, this format describes a top-level configuration, with
+   * a suite containing multiple sub-configuration objects.
+   */
+  bool has_config_suite;
+  /** The position of a config_suite_t pointer within the toplevel object.
+   * Ignored unless have_config_suite is true.
+   */
+  ptrdiff_t config_suite_offset;
+} config_format_t;
+
+#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */
diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am
new file mode 100644
index 0000000000..cb0b83fa64
--- /dev/null
+++ b/src/lib/conf/include.am
@@ -0,0 +1,7 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/conf/confdecl.h              \
+	src/lib/conf/conftesting.h              \
+	src/lib/conf/conftypes.h                \
+	src/lib/conf/confmacros.h
diff --git a/src/lib/conf/lib_conf.md b/src/lib/conf/lib_conf.md
new file mode 100644
index 0000000000..60dd04e99e
--- /dev/null
+++ b/src/lib/conf/lib_conf.md
@@ -0,0 +1,3 @@
+@dir /lib/conf
+@brief lib/conf: Types and macros for declaring configuration options.
+
diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include
new file mode 100644
index 0000000000..5ff949f103
--- /dev/null
+++ b/src/lib/confmgt/.may_include
@@ -0,0 +1,12 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
+lib/confmgt/*.h
+lib/container/*.h
+lib/encoding/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+ext/*.h
diff --git a/src/lib/confmgt/confmgt.c b/src/lib/confmgt/confmgt.c
new file mode 100644
index 0000000000..bf2764160e
--- /dev/null
+++ b/src/lib/confmgt/confmgt.c
@@ -0,0 +1,1406 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confmgt.c
+ *
+ * \brief Back-end for parsing and generating key-value files, used to
+ *   implement the torrc file format and the state file.
+ *
+ * This module is used by config.c to parse and encode torrc
+ * configuration files, and by statefile.c to parse and encode the
+ * $DATADIR/state file.
+ *
+ * To use this module, its callers provide an instance of
+ * config_format_t to describe the mappings from a set of configuration
+ * options to a number of fields in a C structure.  With this mapping,
+ * the functions here can convert back and forth between the C structure
+ * specified, and a linked list of key-value pairs.
+ */
+
+#define CONFMGT_PRIVATE
+#include "orconfig.h"
+#include "lib/confmgt/confmgt.h"
+
+#include "lib/confmgt/structvar.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include "ext/siphash.h"
+
+/**
+ * A managed_var_t is an internal wrapper around a config_var_t in
+ * a config_format_t structure.  It is used by config_mgr_t to
+ * keep track of which option goes with which structure. */
+typedef struct managed_var_t {
+  /**
+   * A pointer to the config_var_t for this option.
+   */
+  const config_var_t *cvar;
+  /**
+   * The index of the object in which this option is stored.  It is
+   * IDX_TOPLEVEL to indicate that the object is the top-level object.
+   **/
+  int object_idx;
+} managed_var_t;
+
+static void config_reset(const config_mgr_t *fmt, void *options,
+                         const managed_var_t *var, int use_defaults);
+static void config_mgr_register_fmt(config_mgr_t *mgr,
+                                    const config_format_t *fmt,
+                                    int object_idx);
+
+/** Release all storage held in a managed_var_t. */
+static void
+managed_var_free_(managed_var_t *mv)
+{
+  if (!mv)
+    return;
+  tor_free(mv);
+}
+#define managed_var_free(mv) \
+  FREE_AND_NULL(managed_var_t, managed_var_free_, (mv))
+
+struct config_suite_t {
+  /** A list of configuration objects managed by a given configuration
+   * manager. They are stored in the same order as the config_format_t
+   * objects in the manager's list of subformats. */
+  smartlist_t *configs;
+};
+
+/**
+ * Allocate a new empty config_suite_t.
+ **/
+static config_suite_t *
+config_suite_new(void)
+{
+  config_suite_t *suite = tor_malloc_zero(sizeof(config_suite_t));
+  suite->configs = smartlist_new();
+  return suite;
+}
+
+/** Release all storage held by a config_suite_t.  (Does not free
+ * any configuration objects it holds; the caller must do that first.) */
+static void
+config_suite_free_(config_suite_t *suite)
+{
+  if (!suite)
+    return;
+  smartlist_free(suite->configs);
+  tor_free(suite);
+}
+
+#define config_suite_free(suite) \
+  FREE_AND_NULL(config_suite_t, config_suite_free_, (suite))
+
+struct config_mgr_t {
+  /** The 'top-level' configuration format.  This one is used for legacy
+   * options that have not yet been assigned to different sub-modules.
+   *
+   * (NOTE: for now, this is the only config_format_t that a config_mgr_t
+   * contains.  A subsequent commit will add more. XXXX)
+   */
+  const config_format_t *toplevel;
+  /**
+   * List of second-level configuration format objects that this manager
+   * also knows about.
+   */
+  smartlist_t *subconfigs;
+  /** A smartlist of managed_var_t objects for all configuration formats. */
+  smartlist_t *all_vars;
+  /** A smartlist of config_abbrev_t objects for all configuration
+   * formats. These objects are used to track synonyms and abbreviations for
+   * different configuration options. */
+  smartlist_t *all_abbrevs;
+  /** A smartlist of config_deprecation_t for all configuration formats. */
+  smartlist_t *all_deprecations;
+  /** True if this manager has been frozen and cannot have any more formats
+   * added to it. A manager must be frozen before it can be used to construct
+   * or manipulate objects. */
+  bool frozen;
+  /** A replacement for the magic number of the toplevel object. We override
+   * that number to make it unique for this particular config_mgr_t, so that
+   * an object constructed with one mgr can't be used with another, even if
+   * those managers' contents are equal.
+   */
+  struct_magic_decl_t toplevel_magic;
+};
+
+#define IDX_TOPLEVEL (-1)
+
+/** Create a new config_mgr_t to manage a set of configuration objects to be
+ * wrapped under <b>toplevel_fmt</b>. */
+config_mgr_t *
+config_mgr_new(const config_format_t *toplevel_fmt)
+{
+  config_mgr_t *mgr = tor_malloc_zero(sizeof(config_mgr_t));
+  mgr->subconfigs = smartlist_new();
+  mgr->all_vars = smartlist_new();
+  mgr->all_abbrevs = smartlist_new();
+  mgr->all_deprecations = smartlist_new();
+
+  config_mgr_register_fmt(mgr, toplevel_fmt, IDX_TOPLEVEL);
+  mgr->toplevel = toplevel_fmt;
+
+  return mgr;
+}
+
+/** Add a config_format_t to a manager, with a specified (unique) index. */
+static void
+config_mgr_register_fmt(config_mgr_t *mgr,
+                        const config_format_t *fmt,
+                        int object_idx)
+{
+  int i;
+
+  tor_assertf(!mgr->frozen,
+              "Tried to add a format to a configuration manager after "
+              "it had been frozen.");
+
+  if (object_idx != IDX_TOPLEVEL) {
+    tor_assertf(! fmt->has_config_suite,
+          "Tried to register a toplevel format in a non-toplevel position");
+  }
+  if (fmt->config_suite_offset) {
+    tor_assertf(fmt->has_config_suite,
+                "config_suite_offset was set, but has_config_suite was not.");
+  }
+
+  tor_assertf(fmt != mgr->toplevel &&
+              ! smartlist_contains(mgr->subconfigs, fmt),
+              "Tried to register an already-registered format.");
+
+  /* register variables */
+  for (i = 0; fmt->vars[i].member.name; ++i) {
+    managed_var_t *mv = tor_malloc_zero(sizeof(managed_var_t));
+    mv->cvar = &fmt->vars[i];
+    mv->object_idx = object_idx;
+    smartlist_add(mgr->all_vars, mv);
+  }
+
+  /* register abbrevs */
+  if (fmt->abbrevs) {
+    for (i = 0; fmt->abbrevs[i].abbreviated; ++i) {
+      smartlist_add(mgr->all_abbrevs, (void*)&fmt->abbrevs[i]);
+    }
+  }
+
+  /* register deprecations. */
+  if (fmt->deprecations) {
+    const config_deprecation_t *d;
+    for (d = fmt->deprecations; d->name; ++d) {
+      smartlist_add(mgr->all_deprecations, (void*)d);
+    }
+  }
+}
+
+/**
+ * Add a new format to this configuration object.  Asserts on failure.
+ *
+ * Returns an internal "index" value used to identify this format within
+ * all of those formats contained in <b>mgr</b>.  This index value
+ * should not generally be used outside of this module.
+ **/
+int
+config_mgr_add_format(config_mgr_t *mgr,
+                      const config_format_t *fmt)
+{
+  tor_assert(mgr);
+  int idx = smartlist_len(mgr->subconfigs);
+  config_mgr_register_fmt(mgr, fmt, idx);
+  smartlist_add(mgr->subconfigs, (void *)fmt);
+  return idx;
+}
+
+/** Return a pointer to the config_suite_t * pointer inside a
+ * configuration object; returns NULL if there is no such member. */
+static inline config_suite_t **
+config_mgr_get_suite_ptr(const config_mgr_t *mgr, void *toplevel)
+{
+  if (! mgr->toplevel->has_config_suite)
+    return NULL;
+  return STRUCT_VAR_P(toplevel, mgr->toplevel->config_suite_offset);
+}
+
+/**
+ * Return a pointer to the configuration object within <b>toplevel</b> whose
+ * index is <b>idx</b>.
+ *
+ * NOTE: XXXX Eventually, there will be multiple objects supported within the
+ * toplevel object.  For example, the or_options_t will contain pointers
+ * to configuration objects for other modules.  This function gets
+ * the sub-object for a particular module.
+ */
+void *
+config_mgr_get_obj_mutable(const config_mgr_t *mgr, void *toplevel, int idx)
+{
+  tor_assert(mgr);
+  tor_assert(toplevel);
+  if (idx == IDX_TOPLEVEL)
+    return toplevel;
+
+  tor_assertf(idx >= 0 && idx < smartlist_len(mgr->subconfigs),
+              "Index %d is out of range.", idx);
+  config_suite_t **suite = config_mgr_get_suite_ptr(mgr, toplevel);
+  tor_assert(suite);
+  tor_assert(smartlist_len(mgr->subconfigs) ==
+             smartlist_len((*suite)->configs));
+
+  return smartlist_get((*suite)->configs, idx);
+}
+
+/** As config_mgr_get_obj_mutable(), but return a const pointer. */
+const void *
+config_mgr_get_obj(const config_mgr_t *mgr, const void *toplevel, int idx)
+{
+  return config_mgr_get_obj_mutable(mgr, (void*)toplevel, idx);
+}
+
+/** Sorting helper for smartlist of managed_var_t */
+static int
+managed_var_cmp(const void **a, const void **b)
+{
+  const managed_var_t *mv1 = *(const managed_var_t**)a;
+  const managed_var_t *mv2 = *(const managed_var_t**)b;
+
+  return strcasecmp(mv1->cvar->member.name, mv2->cvar->member.name);
+}
+
+/**
+ * Mark a configuration manager as "frozen", so that no more formats can be
+ * added, and so that it can be used for manipulating configuration objects.
+ **/
+void
+config_mgr_freeze(config_mgr_t *mgr)
+{
+  static uint64_t mgr_count = 0;
+
+  smartlist_sort(mgr->all_vars, managed_var_cmp);
+  memcpy(&mgr->toplevel_magic, &mgr->toplevel->magic,
+         sizeof(struct_magic_decl_t));
+  uint64_t magic_input[3] = { mgr->toplevel_magic.magic_val,
+                              (uint64_t) (uintptr_t) mgr,
+                              ++mgr_count };
+  mgr->toplevel_magic.magic_val =
+    (uint32_t)siphash24g(magic_input, sizeof(magic_input));
+  mgr->frozen = true;
+}
+
+/** Release all storage held in <b>mgr</b> */
+void
+config_mgr_free_(config_mgr_t *mgr)
+{
+  if (!mgr)
+    return;
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, managed_var_free(mv));
+  smartlist_free(mgr->all_vars);
+  smartlist_free(mgr->all_abbrevs);
+  smartlist_free(mgr->all_deprecations);
+  smartlist_free(mgr->subconfigs);
+  memset(mgr, 0, sizeof(*mgr));
+  tor_free(mgr);
+}
+
+/** Return a new smartlist_t containing a config_var_t for every variable that
+ * <b>mgr</b> knows about.  The elements of this smartlist do not need
+ * to be freed; they have the same lifespan as <b>mgr</b>. */
+smartlist_t *
+config_mgr_list_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv,
+                    smartlist_add(result, (void*) mv->cvar));
+  return result;
+}
+
+/** Return a new smartlist_t containing the names of all deprecated variables.
+ * The elements of this smartlist do not need to be freed; they have the same
+ * lifespan as <b>mgr</b>.
+ */
+smartlist_t *
+config_mgr_list_deprecated_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_deprecations, config_deprecation_t *, d,
+                    smartlist_add(result, (char*)d->name));
+  return result;
+}
+
+/**
+ * Check the magic number on <b>object</b> to make sure it's a valid toplevel
+ * object, created with <b>mgr</b>.  Exit with an assertion if it isn't.
+ **/
+void
+config_check_toplevel_magic(const config_mgr_t *mgr,
+                            const void *object)
+{
+  struct_check_magic(object, &mgr->toplevel_magic);
+}
+
+/** Assert that the magic fields in <b>options</b> and its subsidiary
+ * objects are all okay. */
+static void
+config_mgr_assert_magic_ok(const config_mgr_t *mgr,
+                           const void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+  tor_assert(mgr->frozen);
+  struct_check_magic(options, &mgr->toplevel_magic);
+
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, (void*)options);
+  if (suitep == NULL) {
+    tor_assert(smartlist_len(mgr->subconfigs) == 0);
+    return;
+  }
+
+  tor_assert(smartlist_len((*suitep)->configs) ==
+             smartlist_len(mgr->subconfigs));
+  SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+    void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+    tor_assert(obj);
+    struct_check_magic(obj, &fmt->magic);
+  } SMARTLIST_FOREACH_END(fmt);
+}
+
+/** Macro: assert that <b>cfg</b> has the right magic field for
+ * <b>mgr</b>. */
+#define CONFIG_CHECK(mgr, cfg) STMT_BEGIN                               \
+    config_mgr_assert_magic_ok((mgr), (cfg));                           \
+  STMT_END
+
+/** Allocate an empty configuration object of a given format type. */
+void *
+config_new(const config_mgr_t *mgr)
+{
+  tor_assert(mgr->frozen);
+  void *opts = tor_malloc_zero(mgr->toplevel->size);
+  struct_set_magic(opts, &mgr->toplevel_magic);
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, opts);
+  if (suitep) {
+    *suitep = config_suite_new();
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = tor_malloc_zero(fmt->size);
+      struct_set_magic(obj, &fmt->magic);
+      smartlist_add((*suitep)->configs, obj);
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+  CONFIG_CHECK(mgr, opts);
+  return opts;
+}
+
+/*
+ * Functions to parse config options
+ */
+
+/** If <b>option</b> is an official abbreviation for a longer option,
+ * return the longer option.  Otherwise return <b>option</b>.
+ * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
+ * apply abbreviations that work for the config file and the command line.
+ * If <b>warn_obsolete</b> is set, warn about deprecated names. */
+const char *
+config_expand_abbrev(const config_mgr_t *mgr, const char *option,
+                     int command_line, int warn_obsolete)
+{
+  SMARTLIST_FOREACH_BEGIN(mgr->all_abbrevs, const config_abbrev_t *, abbrev) {
+    /* Abbreviations are case insensitive. */
+    if (!strcasecmp(option, abbrev->abbreviated) &&
+        (command_line || !abbrev->commandline_only)) {
+      if (warn_obsolete && abbrev->warn) {
+        log_warn(LD_CONFIG,
+                 "The configuration option '%s' is deprecated; "
+                 "use '%s' instead.",
+                 abbrev->abbreviated,
+                 abbrev->full);
+      }
+      /* Keep going through the list in case we want to rewrite it more.
+       * (We could imagine recursing here, but I don't want to get the
+       * user into an infinite loop if we craft our list wrong.) */
+      option = abbrev->full;
+    }
+  } SMARTLIST_FOREACH_END(abbrev);
+  return option;
+}
+
+/** If <b>key</b> is a deprecated configuration option, return the message
+ * explaining why it is deprecated (which may be an empty string). Return NULL
+ * if it is not deprecated. The <b>key</b> field must be fully expanded. */
+const char *
+config_find_deprecation(const config_mgr_t *mgr, const char *key)
+{
+  if (BUG(mgr == NULL) || BUG(key == NULL))
+    return NULL; // LCOV_EXCL_LINE
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_deprecations, const config_deprecation_t *,
+                          d) {
+    if (!strcasecmp(d->name, key)) {
+      return d->why_deprecated ? d->why_deprecated : "";
+    }
+  } SMARTLIST_FOREACH_END(d);
+  return NULL;
+}
+
+/**
+ * Find the managed_var_t object for a variable whose name is <b>name</b>
+ * according to <b>mgr</b>.   Return that object, or NULL if none exists.
+ *
+ * If <b>allow_truncated</b> is true, then accept any variable whose
+ * name begins with <b>name</b>.
+ *
+ * If <b>idx_out</b> is not NULL, set *<b>idx_out</b> to the position of
+ * that variable within mgr-&gt;all_vars, or to -1 if the variable is
+ * not found.
+ */
+static const managed_var_t *
+config_mgr_find_var(const config_mgr_t *mgr,
+                    const char *key,
+                    bool allow_truncated, int *idx_out)
+{
+  const size_t keylen = strlen(key);
+  if (idx_out)
+    *idx_out = -1;
+
+  if (!keylen)
+    return NULL; /* if they say "--" on the command line, it's not an option */
+
+  /* First, check for an exact (case-insensitive) match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strcasecmp(mv->cvar->member.name, key)) {
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (!allow_truncated)
+    return NULL;
+
+  /* If none, check for an abbreviated match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strncasecmp(key, mv->cvar->member.name, keylen)) {
+      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
+               "Please use '%s' instead",
+               key, mv->cvar->member.name);
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  /* Okay, unrecognized option */
+  return NULL;
+}
+
+/**
+ * If <b>key</b> is a name or an abbreviation configuration option, return
+ * the corresponding canonical name for it.  Warn if the abbreviation is
+ * non-standard.  Return NULL if the option does not exist.
+ */
+const char *
+config_find_option_name(const config_mgr_t *mgr, const char *key)
+{
+  key = config_expand_abbrev(mgr, key, 0, 0);
+  const managed_var_t *mv = config_mgr_find_var(mgr, key, true, NULL);
+  if (mv)
+    return mv->cvar->member.name;
+  else
+    return NULL;
+}
+
+/** Return the number of option entries in <b>fmt</b>. */
+static int
+config_count_options(const config_mgr_t *mgr)
+{
+  return smartlist_len(mgr->all_vars);
+}
+
+/**
+ * Return true iff at least one bit from <b>flag</b> is set on <b>var</b>,
+ * either in <b>var</b>'s flags, or on the flags of its type.
+ **/
+static bool
+config_var_has_flag(const config_var_t *var, uint32_t flag)
+{
+  uint32_t have_flags = var->flags | struct_var_get_flags(&var->member);
+
+  return (have_flags & flag) != 0;
+}
+
+/**
+ * Return true if assigning a value to <b>var</b> replaces the previous
+ * value.  Return false if assigning a value to <b>var</b> appends
+ * to the previous value.
+ **/
+static bool
+config_var_is_replaced_on_set(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOREPLACE);
+}
+
+/**
+ * Return true iff <b>var</b> may be assigned by name (e.g., via the
+ * CLI, the configuration files, or the controller API).
+ **/
+bool
+config_var_is_settable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOSET);
+}
+
+/**
+ * Return true iff the controller is allowed to fetch the value of
+ * <b>var</b>.
+ **/
+static bool
+config_var_is_gettable(const config_var_t *var)
+{
+  /* Arguably, invisible or obsolete options should not be gettable.  However,
+   * they have been gettable for a long time, and making them ungettable could
+   * have compatibility effects.  For now, let's leave them alone.
+   */
+
+  // return ! config_var_has_flag(var, CVFLAG_OBSOLETE|CFGLAGS_INVISIBLE);
+  (void)var;
+  return true;
+}
+
+/**
+ * Return true iff we need to check <b>var</b> for changes when we are
+ * comparing config options for changes.
+ *
+ * A false result might mean that the variable is a derived variable, and that
+ * comparing the variable it derives from compares this one too-- or it might
+ * mean that there is no data to compare.
+ **/
+static bool
+config_var_should_list_changes(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCMP);
+}
+
+/**
+ * Return true iff we need to copy the data for <b>var</b> when we are
+ * copying a config option.
+ *
+ * A false option might mean that the variable is a derived variable, and that
+ * copying the variable it derives from copies it-- or it might mean that
+ * there is no data to copy.
+ **/
+static bool
+config_var_needs_copy(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCOPY);
+}
+
+/**
+ * Return true iff variable <b>var</b> should appear on list of variable
+ * names given to the controller or the CLI.
+ *
+ * (Note that this option is imperfectly obeyed. The
+ * --list-torrc-options command looks at the "settable" flag, whereas
+ * "GETINFO config/defaults" and "list_deprecated_*()" do not filter
+ * their results. It would be good for consistency to try to converge
+ * these behaviors in the future.)
+ **/
+bool
+config_var_is_listable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOLIST);
+}
+
+/**
+ * Return true iff variable <b>var</b> should be written out when we
+ * are writing our configuration to disk, to a controller, or via the
+ * --dump-config command.
+ *
+ * This option may be set because a variable is hidden, or because it is
+ * derived from another variable which will already be written out.
+ **/
+static bool
+config_var_is_dumpable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NODUMP);
+}
+
+/*
+ * Functions to assign config options.
+ */
+
+/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
+ * with <b>c</b>-\>value and return 0, or return -1 if bad value.
+ *
+ * Called from config_assign_line() and option_reset().
+ */
+static int
+config_assign_value(const config_mgr_t *mgr, void *options,
+                    config_line_t *c, char **msg)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, c->key, true, NULL);
+  tor_assert(var);
+  tor_assert(!strcmp(c->key, var->cvar->member.name));
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+
+  if (config_var_has_flag(var->cvar, CFLG_WARN_OBSOLETE)) {
+    log_warn(LD_GENERAL, "Skipping obsolete configuration option \"%s\".",
+             var->cvar->member.name);
+  } else if (config_var_has_flag(var->cvar, CFLG_WARN_DISABLED)) {
+    log_warn(LD_GENERAL, "This copy of Tor was built without support for "
+             "the option \"%s\". Skipping.", var->cvar->member.name);
+  }
+
+  return struct_var_kvassign(object, c, msg, &var->cvar->member);
+}
+
+/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
+ * to it will replace old ones. */
+static void
+config_mark_lists_fragile(const config_mgr_t *mgr, void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    void *object = config_mgr_get_obj_mutable(mgr, options, mv->object_idx);
+    struct_var_mark_fragile(object, &mv->cvar->member);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/**
+ * Log a warning that declaring that the option called <b>what</b>
+ * is deprecated because of the reason in <b>why</b>.
+ *
+ * (Both arguments must be non-NULL.)
+ **/
+void
+warn_deprecated_option(const char *what, const char *why)
+{
+  const char *space = (why && strlen(why)) ? " " : "";
+  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
+           "be removed in a future version of Tor.%s%s (If you think this is "
+           "a mistake, please let us know!)",
+           what, space, why);
+}
+
+/** If <b>c</b> is a syntactically valid configuration line, update
+ * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
+ * key, -2 for bad value.
+ *
+ * If <b>clear_first</b> is set, clear the value first. Then if
+ * <b>use_defaults</b> is set, set the value to the default.
+ *
+ * Called from config_assign().
+ */
+static int
+config_assign_line(const config_mgr_t *mgr, void *options,
+                   config_line_t *c, unsigned flags,
+                   bitarray_t *options_seen, char **msg)
+{
+  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
+  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
+  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
+  const managed_var_t *mvar;
+
+  CONFIG_CHECK(mgr, options);
+
+  int var_index = -1;
+  mvar = config_mgr_find_var(mgr, c->key, true, &var_index);
+  if (!mvar) {
+    const config_format_t *fmt = mgr->toplevel;
+    if (fmt->extra) {
+      void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset);
+      log_info(LD_CONFIG,
+               "Found unrecognized option '%s'; saving it.", c->key);
+      config_line_append((config_line_t**)lvalue, c->key, c->value);
+      return 0;
+    } else {
+      tor_asprintf(msg,
+                "Unknown option '%s'.  Failing.", c->key);
+      return -1;
+    }
+  }
+
+  const config_var_t *cvar = mvar->cvar;
+  tor_assert(cvar);
+
+  /* Put keyword into canonical case. */
+  if (strcmp(cvar->member.name, c->key)) {
+    tor_free(c->key);
+    c->key = tor_strdup(cvar->member.name);
+  }
+
+  const char *deprecation_msg;
+  if (warn_deprecations &&
+      (deprecation_msg = config_find_deprecation(mgr, cvar->member.name))) {
+    warn_deprecated_option(cvar->member.name, deprecation_msg);
+  }
+
+  if (!strlen(c->value)) {
+    /* reset or clear it, then return */
+    if (!clear_first) {
+      if (! config_var_is_replaced_on_set(cvar) &&
+          c->command != CONFIG_LINE_CLEAR) {
+        /* We got an empty linelist from the torrc or command line.
+           As a special case, call this an error. Warn and ignore. */
+        log_warn(LD_CONFIG,
+                 "Linelist option '%s' has no value. Skipping.", c->key);
+      } else { /* not already cleared */
+        config_reset(mgr, options, mvar, use_defaults);
+      }
+    }
+    return 0;
+  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
+    // This block is unreachable, since a CLEAR line always has an
+    // empty value, and so will trigger be handled by the previous
+    // "if (!strlen(c->value))" block.
+
+    // LCOV_EXCL_START
+    tor_assert_nonfatal_unreached();
+    config_reset(mgr, options, mvar, use_defaults);
+    // LCOV_EXCL_STOP
+  }
+
+  if (options_seen && config_var_is_replaced_on_set(cvar)) {
+    /* We're tracking which options we've seen, and this option is not
+     * supposed to occur more than once. */
+    tor_assert(var_index >= 0);
+    if (bitarray_is_set(options_seen, var_index)) {
+      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
+               "value will be ignored.", cvar->member.name);
+    }
+    bitarray_set(options_seen, var_index);
+  }
+
+  if (config_assign_value(mgr, options, c, msg) < 0)
+    return -2;
+  return 0;
+}
+
+/** Restore the option named <b>key</b> in options to its default value.
+ * Called from config_assign(). */
+STATIC void
+config_reset_line(const config_mgr_t *mgr, void *options,
+                  const char *key, int use_defaults)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var)
+    return; /* give error on next pass. */
+
+  config_reset(mgr, options, var, use_defaults);
+}
+
+/** Return true iff value needs to be quoted and escaped to be used in
+ * a configuration file. */
+static int
+config_value_needs_escape(const char *value)
+{
+  if (*value == '\"')
+    return 1;
+  while (*value) {
+    switch (*value)
+    {
+    case '\r':
+    case '\n':
+    case '#':
+      /* Note: quotes and backspaces need special handling when we are using
+       * quotes, not otherwise, so they don't trigger escaping on their
+       * own. */
+      return 1;
+    default:
+      if (!TOR_ISPRINT(*value))
+        return 1;
+    }
+    ++value;
+  }
+  return 0;
+}
+
+/** Return newly allocated line or lines corresponding to <b>key</b> in the
+ * configuration <b>options</b>.  If <b>escape_val</b> is true and a
+ * value needs to be quoted before it's put in a config file, quote and
+ * escape that value. Return NULL if no such key exists. */
+config_line_t *
+config_get_assigned_option(const config_mgr_t *mgr, const void *options,
+                           const char *key, int escape_val)
+{
+  const managed_var_t *var;
+  config_line_t *result;
+
+  tor_assert(options && key);
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var) {
+    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
+    return NULL;
+  }
+  if (! config_var_is_gettable(var->cvar)) {
+    log_warn(LD_CONFIG, "Option '%s' is obsolete or unfetchable. Failing.",
+             key);
+    return NULL;
+  }
+  const void *object = config_mgr_get_obj(mgr, options, var->object_idx);
+
+  result = struct_var_kvencode(object, &var->cvar->member);
+
+  if (escape_val) {
+    config_line_t *line;
+    for (line = result; line; line = line->next) {
+      if (line->value && config_value_needs_escape(line->value)) {
+        char *newval = esc_for_log(line->value);
+        tor_free(line->value);
+        line->value = newval;
+      }
+    }
+  }
+
+  return result;
+}
+/** Iterate through the linked list of requested options <b>list</b>.
+ * For each item, convert as appropriate and assign to <b>options</b>.
+ * If an item is unrecognized, set *msg and return -1 immediately,
+ * else return 0 for success.
+ *
+ * If <b>clear_first</b>, interpret config options as replacing (not
+ * extending) their previous values. If <b>clear_first</b> is set,
+ * then <b>use_defaults</b> to decide if you set to defaults after
+ * clearing, or make the value 0 or NULL.
+ *
+ * Here are the use cases:
+ * 1. A non-empty AllowInvalid line in your torrc. Appends to current
+ *    if linelist, replaces current if csv.
+ * 2. An empty AllowInvalid line in your torrc. Should clear it.
+ * 3. "RESETCONF AllowInvalid" sets it to default.
+ * 4. "SETCONF AllowInvalid" makes it NULL.
+ * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
+ *
+ * Use_defaults   Clear_first
+ *    0                0       "append"
+ *    1                0       undefined, don't use
+ *    0                1       "set to null first"
+ *    1                1       "set to defaults first"
+ * Return 0 on success, -1 on bad key, -2 on bad value.
+ *
+ * As an additional special case, if a LINELIST config option has
+ * no value and clear_first is 0, then warn and ignore it.
+ */
+
+/*
+There are three call cases for config_assign() currently.
+
+Case one: Torrc entry
+options_init_from_torrc() calls config_assign(0, 0)
+  calls config_assign_line(0, 0).
+    if value is empty, calls config_reset(0) and returns.
+    calls config_assign_value(), appends.
+
+Case two: setconf
+options_trial_assign() calls config_assign(0, 1)
+  calls config_reset_line(0)
+    calls config_reset(0)
+      calls option_clear().
+  calls config_assign_line(0, 1).
+    if value is empty, returns.
+    calls config_assign_value(), appends.
+
+Case three: resetconf
+options_trial_assign() calls config_assign(1, 1)
+  calls config_reset_line(1)
+    calls config_reset(1)
+      calls option_clear().
+      calls config_assign_value(default)
+  calls config_assign_line(1, 1).
+    returns.
+*/
+int
+config_assign(const config_mgr_t *mgr, void *options, config_line_t *list,
+              unsigned config_assign_flags, char **msg)
+{
+  config_line_t *p;
+  bitarray_t *options_seen;
+  const int n_options = config_count_options(mgr);
+  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
+  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
+
+  CONFIG_CHECK(mgr, options);
+
+  /* pass 1: normalize keys */
+  for (p = list; p; p = p->next) {
+    const char *full = config_expand_abbrev(mgr, p->key, 0, 1);
+    if (strcmp(full,p->key)) {
+      tor_free(p->key);
+      p->key = tor_strdup(full);
+    }
+  }
+
+  /* pass 2: if we're reading from a resetting source, clear all
+   * mentioned config options, and maybe set to their defaults. */
+  if (clear_first) {
+    for (p = list; p; p = p->next)
+      config_reset_line(mgr, options, p->key, use_defaults);
+  }
+
+  options_seen = bitarray_init_zero(n_options);
+  /* pass 3: assign. */
+  while (list) {
+    int r;
+    if ((r=config_assign_line(mgr, options, list, config_assign_flags,
+                              options_seen, msg))) {
+      bitarray_free(options_seen);
+      return r;
+    }
+    list = list->next;
+  }
+  bitarray_free(options_seen);
+
+  /** Now we're done assigning a group of options to the configuration.
+   * Subsequent group assignments should _replace_ linelists, not extend
+   * them. */
+  config_mark_lists_fragile(mgr, options);
+
+  return 0;
+}
+
+/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
+ * Called from config_reset() and config_free(). */
+static void
+config_clear(const config_mgr_t *mgr, void *options, const managed_var_t *var)
+{
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+  struct_var_free(object, &var->cvar->member);
+}
+
+/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
+ * <b>use_defaults</b>, set it to its default value.
+ * Called by config_init() and option_reset_line() and option_assign_line(). */
+static void
+config_reset(const config_mgr_t *mgr, void *options,
+             const managed_var_t *var, int use_defaults)
+{
+  config_line_t *c;
+  char *msg = NULL;
+  CONFIG_CHECK(mgr, options);
+  config_clear(mgr, options, var); /* clear it first */
+
+  if (!use_defaults)
+    return; /* all done */
+
+  if (var->cvar->initvalue) {
+    c = tor_malloc_zero(sizeof(config_line_t));
+    c->key = tor_strdup(var->cvar->member.name);
+    c->value = tor_strdup(var->cvar->initvalue);
+    if (config_assign_value(mgr, options, c, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Failed to assign default: %s", msg);
+      tor_free(msg); /* if this happens it's a bug */
+      // LCOV_EXCL_STOP
+    }
+    config_free_lines(c);
+  }
+}
+
+/** Release storage held by <b>options</b>. */
+void
+config_free_(const config_mgr_t *mgr, void *options)
+{
+  if (!options)
+    return;
+
+  tor_assert(mgr);
+
+  if (mgr->toplevel->clear_fn) {
+    mgr->toplevel->clear_fn(mgr, options);
+  }
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, options);
+  if (suitep) {
+    tor_assert(smartlist_len((*suitep)->configs) ==
+               smartlist_len(mgr->subconfigs));
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+      if (fmt->clear_fn) {
+        fmt->clear_fn(mgr, obj);
+      }
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    config_clear(mgr, options, mv);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (mgr->toplevel->extra) {
+    config_line_t **linep = STRUCT_VAR_P(options,
+                                         mgr->toplevel->extra->offset);
+    config_free_lines(*linep);
+    *linep = NULL;
+  }
+
+  if (suitep) {
+    SMARTLIST_FOREACH((*suitep)->configs, void *, obj, tor_free(obj));
+    config_suite_free(*suitep);
+  }
+
+  tor_free(options);
+}
+
+/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
+ * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
+ */
+int
+config_is_same(const config_mgr_t *mgr,
+               const void *o1, const void *o2,
+               const char *name)
+{
+  CONFIG_CHECK(mgr, o1);
+  CONFIG_CHECK(mgr, o2);
+
+  const managed_var_t *var = config_mgr_find_var(mgr, name, true, NULL);
+  if (!var) {
+    return true;
+  }
+  const void *obj1 = config_mgr_get_obj(mgr, o1, var->object_idx);
+  const void *obj2 = config_mgr_get_obj(mgr, o2, var->object_idx);
+
+  return struct_var_eq(obj1, obj2, &var->cvar->member);
+}
+
+/**
+ * Return a list of the options which have changed between <b>options1</b> and
+ * <b>options2</b>. If an option has reverted to its default value, it has a
+ * value entry of NULL.
+ *
+ * <b>options1</b> and <b>options2</b> must be top-level configuration objects
+ * of the type managed by <b>mgr</b>.
+ **/
+config_line_t *
+config_get_changes(const config_mgr_t *mgr,
+                   const void *options1, const void *options2)
+{
+  config_line_t *result = NULL;
+  config_line_t **next = &result;
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_should_list_changes(mv->cvar)) {
+      /* something else will check this var, or it doesn't need checking */
+      continue;
+    }
+    const void *obj1 = config_mgr_get_obj(mgr, options1, mv->object_idx);
+    const void *obj2 = config_mgr_get_obj(mgr, options2, mv->object_idx);
+
+    if (struct_var_eq(obj1, obj2, &mv->cvar->member)) {
+      continue;
+    }
+
+    const char *varname = mv->cvar->member.name;
+    config_line_t *line =
+      config_get_assigned_option(mgr, options2, varname, 1);
+
+    if (line) {
+      *next = line;
+    } else {
+      *next = tor_malloc_zero(sizeof(config_line_t));
+      (*next)->key = tor_strdup(varname);
+    }
+    while (*next)
+      next = &(*next)->next;
+  } SMARTLIST_FOREACH_END(mv);
+
+  return result;
+}
+
+/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
+void *
+config_dup(const config_mgr_t *mgr, const void *old)
+{
+  void *newopts;
+
+  newopts = config_new(mgr);
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_needs_copy(mv->cvar)) {
+      // Something else will copy this option, or it doesn't need copying.
+      continue;
+    }
+    const void *oldobj = config_mgr_get_obj(mgr, old, mv->object_idx);
+    void *newobj = config_mgr_get_obj_mutable(mgr, newopts, mv->object_idx);
+    if (struct_var_copy(newobj, oldobj, &mv->cvar->member) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Unable to copy value for %s.",
+              mv->cvar->member.name);
+      tor_assert_unreached();
+      // LCOV_EXCL_STOP
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return newopts;
+}
+/** Set all vars in the configuration object <b>options</b> to their default
+ * values. */
+void
+config_init(const config_mgr_t *mgr, void *options)
+{
+  CONFIG_CHECK(mgr, options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!mv->cvar->initvalue)
+      continue; /* defaults to NULL or 0 */
+    config_reset(mgr, options, mv, 1);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/**
+ * Helper for config_validate_single: see whether any immutable option
+ * has changed between old_options and new_options.
+ *
+ * On success return 0; on failure set *msg_out to a newly allocated
+ * string explaining what is wrong, and return -1.
+ */
+static int
+config_check_immutable_flags(const config_format_t *fmt,
+                             const void *old_options,
+                             const void *new_options,
+                             char **msg_out)
+{
+  tor_assert(fmt);
+  tor_assert(new_options);
+  if (BUG(! old_options))
+    return 0;
+
+  unsigned i;
+  for (i = 0; fmt->vars[i].member.name; ++i) {
+    const config_var_t *v = &fmt->vars[i];
+    if (! config_var_has_flag(v, CFLG_IMMUTABLE))
+      continue;
+
+    if (! struct_var_eq(old_options, new_options, &v->member)) {
+      tor_asprintf(msg_out,
+                   "While Tor is running, changing %s is not allowed",
+                   v->member.name);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Normalize and validate a single object `options` within a configuration
+ * suite, according to its format.  `options` may be modified as appropriate
+ * in order to set ancillary data.  If `old_options` is provided, make sure
+ * that the transition from `old_options` to `options` is permitted.
+ *
+ * On success return VSTAT_OK; on failure set *msg_out to a newly allocated
+ * string explaining what is wrong, and return a different validation_status_t
+ * to describe which step failed.
+ **/
+static validation_status_t
+config_validate_single(const config_format_t *fmt,
+                       const void *old_options, void *options,
+                       char **msg_out)
+{
+  tor_assert(fmt);
+  tor_assert(options);
+
+  if (fmt->pre_normalize_fn) {
+    if (fmt->pre_normalize_fn(options, msg_out) < 0) {
+      return VSTAT_PRE_NORMALIZE_ERR;
+    }
+  }
+
+  if (fmt->legacy_validate_fn) {
+    if (fmt->legacy_validate_fn(old_options, options, msg_out) < 0) {
+      return VSTAT_LEGACY_ERR;
+    }
+  }
+
+  if (fmt->validate_fn) {
+    if (fmt->validate_fn(options, msg_out) < 0) {
+      return VSTAT_VALIDATE_ERR;
+    }
+  }
+
+  if (old_options) {
+    if (config_check_immutable_flags(fmt, old_options, options, msg_out) < 0) {
+      return VSTAT_TRANSITION_ERR;
+    }
+
+    if (fmt->check_transition_fn) {
+      if (fmt->check_transition_fn(old_options, options, msg_out) < 0) {
+        return VSTAT_TRANSITION_ERR;
+      }
+    }
+  }
+
+  if (fmt->post_normalize_fn) {
+    if (fmt->post_normalize_fn(options, msg_out) < 0) {
+      return VSTAT_POST_NORMALIZE_ERR;
+    }
+  }
+
+  return VSTAT_OK;
+}
+
+/**
+ * Normalize and validate all the options in configuration object `options`
+ * and its sub-objects. `options` may be modified as appropriate in order to
+ * set ancillary data.  If `old_options` is provided, make sure that the
+ * transition from `old_options` to `options` is permitted.
+ *
+ * On success return VSTAT_OK; on failure set *msg_out to a newly allocated
+ * string explaining what is wrong, and return a different validation_status_t
+ * to describe which step failed.
+ **/
+validation_status_t
+config_validate(const config_mgr_t *mgr,
+                const void *old_options, void *options,
+                char **msg_out)
+{
+  validation_status_t rv;
+  CONFIG_CHECK(mgr, options);
+  if (old_options) {
+    CONFIG_CHECK(mgr, old_options);
+  }
+
+  config_suite_t **suitep_new = config_mgr_get_suite_ptr(mgr, options);
+  config_suite_t **suitep_old = NULL;
+  if (old_options)
+    suitep_old = config_mgr_get_suite_ptr(mgr, (void*) old_options);
+
+  /* Validate the sub-objects */
+  if (suitep_new) {
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = smartlist_get((*suitep_new)->configs, fmt_sl_idx);
+      const void *obj_old=NULL;
+      if (suitep_old)
+        obj_old = smartlist_get((*suitep_old)->configs, fmt_sl_idx);
+
+      rv = config_validate_single(fmt, obj_old, obj, msg_out);
+      if (rv < 0)
+        return rv;
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+
+  /* Validate the top-level object. */
+  rv = config_validate_single(mgr->toplevel, old_options, options, msg_out);
+  if (rv < 0)
+    return rv;
+
+  return VSTAT_OK;
+}
+
+/** Allocate and return a new string holding the written-out values of the vars
+ * in 'options'.  If 'minimal', do not write out any default-valued vars.
+ * Else, if comment_defaults, write default values as comments.
+ */
+char *
+config_dump(const config_mgr_t *mgr, const void *default_options,
+            const void *options, int minimal,
+            int comment_defaults)
+{
+  const config_format_t *fmt = mgr->toplevel;
+  smartlist_t *elements;
+  const void *defaults = default_options;
+  void *defaults_tmp = NULL;
+  config_line_t *line, *assigned;
+  char *result;
+  char *msg = NULL;
+
+  if (defaults == NULL) {
+    defaults = defaults_tmp = config_new(mgr);
+    config_init(mgr, defaults_tmp);
+  }
+
+  /* XXX use a 1 here so we don't add a new log line while dumping */
+  if (default_options == NULL) {
+    if (config_validate(mgr, NULL, defaults_tmp, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Failed to validate default config: %s", msg);
+      tor_free(msg);
+      tor_assert(0);
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  elements = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    int comment_option = 0;
+    /* Don't save 'hidden' control variables. */
+    if (! config_var_is_dumpable(mv->cvar))
+      continue;
+    const char *name = mv->cvar->member.name;
+    if (minimal && config_is_same(mgr, options, defaults, name))
+      continue;
+    else if (comment_defaults &&
+             config_is_same(mgr, options, defaults, name))
+      comment_option = 1;
+
+    line = assigned =
+      config_get_assigned_option(mgr, options, name, 1);
+
+    for (; line; line = line->next) {
+      if (!strcmpstart(line->key, "__")) {
+        /* This check detects "hidden" variables inside LINELIST_V structures.
+         */
+        continue;
+      }
+      int value_exists = line->value && *(line->value);
+      smartlist_add_asprintf(elements, "%s%s%s%s\n",
+                   comment_option ? "# " : "",
+                   line->key, value_exists ? " " : "", line->value);
+    }
+    config_free_lines(assigned);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (fmt->extra) {
+    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset);
+    for (; line; line = line->next) {
+      int value_exists = line->value && *(line->value);
+      smartlist_add_asprintf(elements, "%s%s%s\n",
+                             line->key, value_exists ? " " : "", line->value);
+    }
+  }
+
+  result = smartlist_join_strings(elements, "", 0, NULL);
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+  config_free(mgr, defaults_tmp);
+  return result;
+}
+
+/**
+ * Return true if every member of <b>options</b> is in-range and well-formed.
+ * Return false otherwise.  Log errors at level <b>severity</b>.
+ */
+bool
+config_check_ok(const config_mgr_t *mgr, const void *options, int severity)
+{
+  bool all_ok = true;
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!struct_var_ok(options, &mv->cvar->member)) {
+      log_fn(severity, LD_BUG, "Invalid value for %s",
+             mv->cvar->member.name);
+      all_ok = false;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return all_ok;
+}
diff --git a/src/lib/confmgt/confmgt.h b/src/lib/confmgt/confmgt.h
new file mode 100644
index 0000000000..5065c13b60
--- /dev/null
+++ b/src/lib/confmgt/confmgt.h
@@ -0,0 +1,136 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confmgt.h
+ *
+ * \brief Header for confmgt.c.
+ */
+
+#ifndef TOR_CONFMGT_H
+#define TOR_CONFMGT_H
+
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confmacros.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * A collection of config_format_t objects to describe several objects
+ * that are all configured with the same configuration file.
+ *
+ * (NOTE: for now, this only handles a single config_format_t.)
+ **/
+typedef struct config_mgr_t config_mgr_t;
+
+config_mgr_t *config_mgr_new(const config_format_t *toplevel_fmt);
+void config_mgr_free_(config_mgr_t *mgr);
+int config_mgr_add_format(config_mgr_t *mgr,
+                          const config_format_t *fmt);
+void config_mgr_freeze(config_mgr_t *mgr);
+#define config_mgr_free(mgr) \
+  FREE_AND_NULL(config_mgr_t, config_mgr_free_, (mgr))
+struct smartlist_t *config_mgr_list_vars(const config_mgr_t *mgr);
+struct smartlist_t *config_mgr_list_deprecated_vars(const config_mgr_t *mgr);
+
+/** A collection of managed configuration objects. */
+typedef struct config_suite_t config_suite_t;
+
+/**
+ * Flag for config_assign: if set, then "resetting" an option changes it to
+ * its default value, as specified in the config_var_t.  Otherwise,
+ * "resetting" an option changes it to a type-dependent null value --
+ * typically 0 or NULL.
+ *
+ * (An option is "reset" when it is set to an empty value, or as described in
+ * CAL_CLEAR_FIRST).
+ **/
+#define CAL_USE_DEFAULTS      (1u<<0)
+/**
+ * Flag for config_assign: if set, then we reset every provided config
+ * option before we set it.
+ *
+ * For example, if this flag is not set, then passing a multi-line option to
+ * config_assign will cause any previous value to be extended. But if this
+ * flag is set, then a multi-line option will replace any previous value.
+ **/
+#define CAL_CLEAR_FIRST       (1u<<1)
+/**
+ * Flag for config_assign: if set, we warn about deprecated options.
+ **/
+#define CAL_WARN_DEPRECATIONS (1u<<2)
+
+void *config_new(const config_mgr_t *fmt);
+void config_free_(const config_mgr_t *fmt, void *options);
+#define config_free(mgr, options) do {                \
+    config_free_((mgr), (options));                   \
+    (options) = NULL;                                 \
+  } while (0)
+
+struct config_line_t *config_get_assigned_option(const config_mgr_t *mgr,
+                                          const void *options, const char *key,
+                                          int escape_val);
+int config_is_same(const config_mgr_t *fmt,
+                   const void *o1, const void *o2,
+                   const char *name);
+struct config_line_t *config_get_changes(const config_mgr_t *mgr,
+                                  const void *options1, const void *options2);
+void config_init(const config_mgr_t *mgr, void *options);
+
+/** An enumeration to report which validation step failed. */
+typedef enum {
+  VSTAT_PRE_NORMALIZE_ERR = -5,
+  VSTAT_VALIDATE_ERR = -4,
+  VSTAT_LEGACY_ERR = -3,
+  VSTAT_TRANSITION_ERR = -2,
+  VSTAT_POST_NORMALIZE_ERR = -1,
+  VSTAT_OK = 0,
+} validation_status_t;
+
+validation_status_t config_validate(const config_mgr_t *mgr,
+                                    const void *old_options, void *options,
+                                    char **msg_out);
+void *config_dup(const config_mgr_t *mgr, const void *old);
+char *config_dump(const config_mgr_t *mgr, const void *default_options,
+                  const void *options, int minimal,
+                  int comment_defaults);
+void config_check_toplevel_magic(const config_mgr_t *mgr,
+                                 const void *object);
+bool config_check_ok(const config_mgr_t *mgr, const void *options,
+                     int severity);
+int config_assign(const config_mgr_t *mgr, void *options,
+                  struct config_line_t *list,
+                  unsigned flags, char **msg);
+const char *config_find_deprecation(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_find_option_name(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_expand_abbrev(const config_mgr_t *mgr,
+                                 const char *option,
+                                 int command_line, int warn_obsolete);
+void warn_deprecated_option(const char *what, const char *why);
+
+bool config_var_is_settable(const config_var_t *var);
+bool config_var_is_listable(const config_var_t *var);
+
+/* Helper macros to compare an option across two configuration objects */
+#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
+#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
+#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
+#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
+
+void *config_mgr_get_obj_mutable(const config_mgr_t *mgr,
+                                 void *toplevel, int idx);
+const void *config_mgr_get_obj(const config_mgr_t *mgr,
+                               const void *toplevel, int idx);
+
+#ifdef CONFMGT_PRIVATE
+STATIC void config_reset_line(const config_mgr_t *mgr, void *options,
+                              const char *key, int use_defaults);
+#endif /* defined(CONFMGT_PRIVATE) */
+
+#endif /* !defined(TOR_CONFMGT_H) */
diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am
new file mode 100644
index 0000000000..d3a7a7cd69
--- /dev/null
+++ b/src/lib/confmgt/include.am
@@ -0,0 +1,27 @@
+noinst_LIBRARIES += src/lib/libtor-confmgt.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_confmgt_a_SOURCES =			\
+	src/lib/confmgt/confmgt.c      			\
+	src/lib/confmgt/structvar.c                      \
+	src/lib/confmgt/type_defs.c                      \
+	src/lib/confmgt/typedvar.c                      \
+	src/lib/confmgt/unitparse.c
+
+src_lib_libtor_confmgt_testing_a_SOURCES = \
+	$(src_lib_libtor_confmgt_a_SOURCES)
+src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/confmgt/confmgt.h			\
+	src/lib/confmgt/structvar.h			\
+	src/lib/confmgt/type_defs.h			\
+	src/lib/confmgt/typedvar.h			\
+	src/lib/confmgt/unitparse.h			\
+	src/lib/confmgt/var_type_def_st.h
diff --git a/src/lib/confmgt/lib_confmgt.md b/src/lib/confmgt/lib_confmgt.md
new file mode 100644
index 0000000000..861e720f64
--- /dev/null
+++ b/src/lib/confmgt/lib_confmgt.md
@@ -0,0 +1,7 @@
+@dir /lib/confmgt
+@brief lib/confmgt: Parse, encode, manipulate configuration files.
+
+This logic is used in common by our state files (statefile.c) and
+configuration files (config.c) to manage a set of named, typed fields,
+reading and writing them to disk and to the controller.
+
diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c
new file mode 100644
index 0000000000..55deb4759c
--- /dev/null
+++ b/src/lib/confmgt/structvar.c
@@ -0,0 +1,239 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.c
+ * @brief Functions to manipulate named and typed elements of
+ *    a structure.
+ *
+ * These functions represent a low-level API for accessing a member of a
+ * structure.  They use typedvar.c to work, and they are used in turn by the
+ * configuration system to examine and set fields in configuration objects
+ * used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/structvar.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/log/util_bug.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+
+/**
+ * Return true iff all fields on <b>decl</b> are NULL or 0, indicating that
+ * there is no object or no magic number to check.
+ **/
+static inline bool
+magic_is_null(const struct_magic_decl_t *decl)
+{
+  return decl->typename == NULL &&
+    decl->magic_offset == 0 &&
+    decl->magic_val == 0;
+}
+
+/**
+ * Set the 'magic number' on <b>object</b> to correspond to decl.
+ **/
+void
+struct_set_magic(void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(decl);
+  if (magic_is_null(decl))
+    return;
+
+  tor_assert(object);
+  uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  *ptr = decl->magic_val;
+}
+
+/**
+ * Assert that the 'magic number' on <b>object</b> to corresponds to decl.
+ **/
+void
+struct_check_magic(const void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(decl);
+  if (magic_is_null(decl))
+    return;
+
+  tor_assert(object);
+
+  const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  tor_assertf(*ptr == decl->magic_val,
+              "Bad magic number on purported %s object. "
+              "Expected %"PRIu32"x but got %"PRIu32"x.",
+              decl->typename, decl->magic_val, *ptr);
+}
+
+/**
+ * Return a mutable pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+void *
+struct_get_mptr(void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Return a const pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+const void *
+struct_get_ptr(const void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Helper: given a struct_member_t, look up the type definition for its
+ * variable.
+ */
+static const var_type_def_t *
+get_type_def(const struct_member_t *member)
+{
+  if (member->type_def)
+    return member->type_def;
+
+  return lookup_type_def(member->type);
+}
+
+/**
+ * (As typed_var_free, but free and clear the member of <b>object</b> defined
+ * by <b>member</b>.)
+ **/
+void
+struct_var_free(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  typed_var_free(p, def);
+}
+
+/**
+ * (As typed_var_copy, but copy from <b>src</b> to <b>dest</b> the member
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_copy(void *dest, const void *src, const struct_member_t *member)
+{
+  void *p_dest = struct_get_mptr(dest, member);
+  const void *p_src = struct_get_ptr(src, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_copy(p_dest, p_src, def);
+}
+
+/**
+ * (As typed_var_eq, but compare the members of <b>a</b> and <b>b</b>
+ * defined by <b>member</b>.)
+ **/
+bool
+struct_var_eq(const void *a, const void *b, const struct_member_t *member)
+{
+  const void *p_a = struct_get_ptr(a, member);
+  const void *p_b = struct_get_ptr(b, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_eq(p_a, p_b, def);
+}
+
+/**
+ * (As typed_var_ok, but validate the member of <b>object</b> defined by
+ * <b>member</b>.)
+ **/
+bool
+struct_var_ok(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_ok(p, def);
+}
+
+/**
+ * (As typed_var_kvassign, but assign a value to the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_kvassign(void *object, const struct config_line_t *line,
+                    char **errmsg,
+                    const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvassign(p, line, errmsg, def);
+}
+
+/**
+ * (As typed_var_kvencode, but encode the value of the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+struct config_line_t *
+struct_var_kvencode(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvencode(member->name, p, def);
+}
+
+/**
+ * Mark the field in <b>object</b> determined by <b>member</b> -- a variable
+ * that ordinarily would be extended by assignment -- as "fragile", so that it
+ * will get replaced by the next assignment instead.
+ */
+void
+struct_var_mark_fragile(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+  return typed_var_mark_fragile(p, def);
+}
+
+/**
+ * Return the official name of this struct member.
+ **/
+const char *
+struct_var_get_name(const struct_member_t *member)
+{
+  return member->name;
+}
+
+/**
+ * Return the type name for this struct member.
+ *
+ * Do not use the output of this function to inspect a type within Tor.  It is
+ * suitable for debugging, informing the controller or user of a variable's
+ * type, etc.
+ **/
+const char *
+struct_var_get_typename(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->name : NULL;
+}
+
+/** Return all of the flags set for this struct member. */
+uint32_t
+struct_var_get_flags(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->flags : 0;
+}
diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h
new file mode 100644
index 0000000000..91334fa8c5
--- /dev/null
+++ b/src/lib/confmgt/structvar.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.h
+ * @brief Header for lib/confmgt/structvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H
+#define TOR_LIB_CONFMGT_STRUCTVAR_H
+
+struct struct_magic_decl_t;
+struct struct_member_t;
+struct config_line_t;
+
+#include <stdbool.h>
+#include "lib/cc/torint.h"
+
+void struct_set_magic(void *object,
+                      const struct struct_magic_decl_t *decl);
+void struct_check_magic(const void *object,
+                        const struct struct_magic_decl_t *decl);
+
+void *struct_get_mptr(void *object,
+                      const struct struct_member_t *member);
+const void *struct_get_ptr(const void *object,
+                           const struct struct_member_t *member);
+
+void struct_var_free(void *object,
+                     const struct struct_member_t *member);
+int struct_var_copy(void *dest, const void *src,
+                    const struct struct_member_t *member);
+bool struct_var_eq(const void *a, const void *b,
+                   const struct struct_member_t *member);
+bool struct_var_ok(const void *object,
+                   const struct struct_member_t *member);
+void struct_var_mark_fragile(void *object,
+                             const struct struct_member_t *member);
+
+const char *struct_var_get_name(const struct struct_member_t *member);
+const char *struct_var_get_typename(const struct struct_member_t *member);
+uint32_t struct_var_get_flags(const struct struct_member_t *member);
+
+int struct_var_kvassign(void *object, const struct config_line_t *line,
+                        char **errmsg,
+                        const struct struct_member_t *member);
+struct config_line_t *struct_var_kvencode(
+                        const void *object,
+                        const struct struct_member_t *member);
+
+#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */
diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c
new file mode 100644
index 0000000000..d9e5e1e4c2
--- /dev/null
+++ b/src/lib/confmgt/type_defs.c
@@ -0,0 +1,849 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.c
+ * @brief Definitions for various low-level configuration types.
+ *
+ * This module creates a number of var_type_def_t objects, to be used by
+ * typedvar.c in manipulating variables.
+ *
+ * The types here are common types that can be implemented with Tor's
+ * low-level functionality.  To define new types, see var_type_def_st.h.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confdecl.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/unitparse.h"
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+#include <errno.h>
+
+//////
+// CONFIG_TYPE_STRING
+// CONFIG_TYPE_FILENAME
+//
+// These two types are the same for now, but they have different names.
+//
+// Warning: For this type, the default value (NULL) and "" are considered
+// different values.  That is generally risky, and best avoided for other
+// types in the future.
+//////
+
+static int
+string_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  char **p = (char**)target;
+  *p = tor_strdup(value);
+  return 0;
+}
+
+static char *
+string_encode(const void *value, const void *params)
+{
+  (void)params;
+  const char **p = (const char**)value;
+  return *p ? tor_strdup(*p) : NULL;
+}
+
+static void
+string_clear(void *value, const void *params)
+{
+  (void)params;
+  char **p = (char**)value;
+  tor_free(*p); // sets *p to NULL.
+}
+
+static const var_type_fns_t string_fns = {
+  .parse = string_parse,
+  .encode = string_encode,
+  .clear = string_clear,
+};
+
+/////
+// CONFIG_TYPE_INT
+// CONFIG_TYPE_POSINT
+//
+// These types are implemented as int, possibly with a restricted range.
+/////
+
+/**
+ * Parameters for parsing an integer type.
+ **/
+typedef struct int_type_params_t {
+  int minval; /**< Lowest allowed value */
+  int maxval; /**< Highest allowed value */
+} int_parse_params_t;
+
+static const int_parse_params_t INT_PARSE_UNRESTRICTED = {
+  .minval = INT_MIN,
+  .maxval = INT_MAX,
+};
+
+static const int_parse_params_t INT_PARSE_POSINT = {
+  .minval = 0,
+  .maxval = INT_MAX,
+};
+
+static int
+int_parse(void *target, const char *value, char **errmsg, const void *params)
+{
+  const int_parse_params_t *pp;
+  if (params) {
+    pp = params;
+  } else {
+    pp = &INT_PARSE_UNRESTRICTED;
+  }
+  int *p = target;
+  int ok=0;
+  *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds. "
+                 "Allowed values are between %d and %d.",
+                 value, pp->minval, pp->maxval);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+int_encode(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  char *result;
+  tor_asprintf(&result, "%d", v);
+  return result;
+}
+
+static void
+int_clear(void *value, const void *params)
+{
+  (void)params;
+  *(int*)value = 0;
+}
+
+static bool
+int_ok(const void *value, const void *params)
+{
+  const int_parse_params_t *pp = params;
+  if (pp) {
+    int v = *(int*)value;
+    return pp->minval <= v && v <= pp->maxval;
+  } else {
+    return true;
+  }
+}
+
+static const var_type_fns_t int_fns = {
+  .parse = int_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+  .ok = int_ok,
+};
+
+/////
+// CONFIG_TYPE_UINT64
+//
+// This type is an unrestricted u64.
+/////
+
+static int
+uint64_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  uint64_t *p = target;
+  int ok=0;
+  *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.",
+                 value);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+uint64_encode(const void *value, const void *params)
+{
+  (void)params;
+  uint64_t v = *(uint64_t*)value;
+  char *result;
+  tor_asprintf(&result, "%"PRIu64, v);
+  return result;
+}
+
+static void
+uint64_clear(void *value, const void *params)
+{
+  (void)params;
+  *(uint64_t*)value = 0;
+}
+
+static const var_type_fns_t uint64_fns = {
+  .parse = uint64_parse,
+  .encode = uint64_encode,
+  .clear = uint64_clear,
+};
+
+/////
+// CONFIG_TYPE_INTERVAL
+// CONFIG_TYPE_MSEC_INTERVAL
+// CONFIG_TYPE_MEMUNIT
+//
+// These types are implemented using the config_parse_units() function.
+// The intervals are stored as ints, whereas memory units are stored as
+// uint64_ts.
+/////
+
+static int
+units_parse_u64(void *target, const char *value, char **errmsg,
+                const void *params)
+{
+  const unit_table_t *table = params;
+  tor_assert(table);
+  uint64_t *v = (uint64_t*)target;
+  int ok=1;
+  char *msg = NULL;
+  *v = config_parse_units(value, table, &ok, &msg);
+  if (!ok) {
+    tor_asprintf(errmsg, "Provided value is malformed or out of bounds: %s",
+                 msg);
+    tor_free(msg);
+    return -1;
+  }
+  if (BUG(msg)) {
+    tor_free(msg);
+  }
+  return 0;
+}
+
+static int
+units_parse_int(void *target, const char *value, char **errmsg,
+               const void *params)
+{
+  const unit_table_t *table = params;
+  tor_assert(table);
+  int *v = (int*)target;
+  int ok=1;
+  char *msg = NULL;
+  uint64_t u64 = config_parse_units(value, table, &ok, &msg);
+  if (!ok) {
+    tor_asprintf(errmsg, "Provided value is malformed or out of bounds: %s",
+                 msg);
+    tor_free(msg);
+    return -1;
+  }
+  if (BUG(msg)) {
+    tor_free(msg);
+  }
+  if (u64 > INT_MAX) {
+    tor_asprintf(errmsg, "Provided value %s is too large", value);
+    return -1;
+  }
+  *v = (int) u64;
+  return 0;
+}
+
+static bool
+units_ok_int(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  return v >= 0;
+}
+
+static const var_type_fns_t memunit_fns = {
+  .parse = units_parse_u64,
+  .encode = uint64_encode, // doesn't use params
+  .clear = uint64_clear, // doesn't use params
+};
+
+static const var_type_fns_t interval_fns = {
+  .parse = units_parse_int,
+  .encode = int_encode, // doesn't use params
+  .clear = int_clear, // doesn't use params,
+  .ok = units_ok_int // can't use int_ok, since that expects int params.
+};
+
+/////
+// CONFIG_TYPE_DOUBLE
+//
+// This is a nice simple double.
+/////
+
+static int
+double_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  double *v = (double*)target;
+  char *endptr=NULL;
+  errno = 0;
+  *v = strtod(value, &endptr);
+  if (endptr == value || *endptr != '\0') {
+    // Either there are no converted characters, or there were some characters
+    // that didn't get converted.
+    tor_asprintf(errmsg, "Could not convert %s to a number.", escaped(value));
+    return -1;
+  }
+  if (errno == ERANGE) {
+    // strtod will set errno to ERANGE on underflow or overflow.
+    bool underflow = -.00001 < *v && *v < .00001;
+    tor_asprintf(errmsg,
+                 "%s is too %s to express as a floating-point number.",
+                 escaped(value), underflow ? "small" : "large");
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+double_encode(const void *value, const void *params)
+{
+  (void)params;
+  double v = *(double*)value;
+  char *result;
+  tor_asprintf(&result, "%f", v);
+  return result;
+}
+
+static void
+double_clear(void *value, const void *params)
+{
+  (void)params;
+  double *v = (double *)value;
+  *v = 0.0;
+}
+
+static const var_type_fns_t double_fns = {
+  .parse = double_parse,
+  .encode = double_encode,
+  .clear = double_clear,
+};
+
+/////
+// CONFIG_TYPE_BOOL
+// CONFIG_TYPE_AUTOBOOL
+//
+// These types are implemented as a case-insensitive string-to-integer
+// mapping.
+/////
+
+typedef struct enumeration_table_t {
+  const char *name;
+  int value;
+} enumeration_table_t;
+
+typedef struct enumeration_params_t {
+  const char *allowed_val_string;
+  const enumeration_table_t *table;
+} enumeration_params_t;
+
+static int
+enum_parse(void *target, const char *value, char **errmsg,
+           const void *params_)
+{
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  int *p = (int *)target;
+  for (; table->name; ++table) {
+    if (!strcasecmp(value, table->name)) {
+      *p = table->value;
+      return 0;
+    }
+  }
+  tor_asprintf(errmsg, "Unrecognized value %s. %s",
+               value, params->allowed_val_string);
+  return -1;
+}
+
+static char *
+enum_encode(const void *value, const void *params_)
+{
+  int v = *(const int*)value;
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return tor_strdup(table->name);
+  }
+  return NULL; // error.
+}
+
+static void
+enum_clear(void *value, const void *params_)
+{
+  int *p = (int*)value;
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  tor_assert(table->name);
+  *p = table->value;
+}
+
+static bool
+enum_ok(const void *value, const void *params_)
+{
+  int v = *(const int*)value;
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return true;
+  }
+  return false;
+}
+
+static const enumeration_table_t enum_table_bool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { NULL, 0 },
+};
+
+static const enumeration_params_t enum_params_bool = {
+  "Allowed values are 0 and 1.",
+  enum_table_bool
+};
+
+static const enumeration_table_t enum_table_autobool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { "auto", -1 },
+  { NULL, 0 },
+};
+
+static const enumeration_params_t enum_params_autobool = {
+  "Allowed values are 0, 1, and auto.",
+  enum_table_autobool
+};
+
+static const var_type_fns_t enum_fns = {
+  .parse = enum_parse,
+  .encode = enum_encode,
+  .clear = enum_clear,
+  .ok = enum_ok,
+};
+
+/////
+// CONFIG_TYPE_ISOTIME
+//
+// This is a time_t, encoded in ISO8601 format.
+/////
+
+static int
+time_parse(void *target, const char *value, char **errmsg,
+           const void *params)
+{
+  (void) params;
+  time_t *p = target;
+  if (parse_iso_time(value, p) < 0) {
+    tor_asprintf(errmsg, "Invalid time %s", escaped(value));
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+time_encode(const void *value, const void *params)
+{
+  (void)params;
+  time_t v = *(const time_t *)value;
+  char *result = tor_malloc(ISO_TIME_LEN+1);
+  format_iso_time(result, v);
+  return result;
+}
+
+static void
+time_clear(void *value, const void *params)
+{
+  (void)params;
+  time_t *t = value;
+  *t = 0;
+}
+
+static const var_type_fns_t time_fns = {
+  .parse = time_parse,
+  .encode = time_encode,
+  .clear = time_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV
+//
+// This type is a comma-separated list of strings, stored in a smartlist_t.
+// An empty list may be encoded either as an empty smartlist, or as NULL.
+/////
+
+static int
+csv_parse(void *target, const char *value, char **errmsg,
+          const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  smartlist_t **sl = (smartlist_t**)target;
+  *sl = smartlist_new();
+  smartlist_split_string(*sl, value, ",",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  return 0;
+}
+
+static char *
+csv_encode(const void *value, const void *params)
+{
+  (void)params;
+  const smartlist_t *sl = *(const smartlist_t **)value;
+  if (! sl)
+    return tor_strdup("");
+
+  return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
+}
+
+static void
+csv_clear(void *value, const void *params)
+{
+  (void)params;
+  smartlist_t **sl = (smartlist_t**)value;
+  if (!*sl)
+    return;
+  SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp));
+  smartlist_free(*sl); // clears pointer.
+}
+
+static const var_type_fns_t csv_fns = {
+  .parse = csv_parse,
+  .encode = csv_encode,
+  .clear = csv_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV_INTERVAL
+//
+// This type used to be a list of time intervals, used to determine a download
+// schedule.  Now, only the first interval counts: everything after the first
+// comma is discarded.
+/////
+
+static int
+legacy_csv_interval_parse(void *target, const char *value, char **errmsg,
+                          const void *params)
+{
+  (void)params;
+  /* We used to have entire smartlists here.  But now that all of our
+   * download schedules use exponential backoff, only the first part
+   * matters. */
+  const char *comma = strchr(value, ',');
+  const char *val = value;
+  char *tmp = NULL;
+  if (comma) {
+    tmp = tor_strndup(val, comma - val);
+    val = tmp;
+  }
+
+  int rv = units_parse_int(target, val, errmsg, &time_units);
+  tor_free(tmp);
+  return rv;
+}
+
+static const var_type_fns_t legacy_csv_interval_fns = {
+  .parse = legacy_csv_interval_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+};
+
+/////
+// CONFIG_TYPE_LINELIST
+// CONFIG_TYPE_LINELIST_S
+// CONFIG_TYPE_LINELIST_V
+//
+// A linelist is a raw config_line_t list.  Order is preserved.
+//
+// The LINELIST type is used for homogeneous lists, where all the lines
+// have the same key.
+//
+// The LINELIST_S and LINELIST_V types are used for the case where multiple
+// lines of different keys are kept in a single list, to preserve their
+// relative order.  The unified list is stored as a "virtual" variable whose
+// type is LINELIST_V; the individual sublists are treated as variables of
+// type LINELIST_S.
+//
+// A linelist may be fragile or non-fragile. Assigning a line to a fragile
+// linelist replaces the list with the line.  If the line has the "APPEND"
+// command set on it, or if the list is non-fragile, the line is appended.
+// Either way, the new list is non-fragile.
+/////
+
+static int
+linelist_kv_parse(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  config_line_t **lines = target;
+
+  if (*lines && (*lines)->fragile) {
+    if (line->command == CONFIG_LINE_APPEND) {
+      (*lines)->fragile = 0;
+    } else {
+      config_free_lines(*lines); // sets it to NULL
+    }
+  }
+
+  config_line_append(lines, line->key, line->value);
+  return 0;
+}
+
+static int
+linelist_kv_virt_noparse(void *target, const struct config_line_t *line,
+                         char **errmsg, const void *params)
+{
+  (void)target;
+  (void)line;
+  (void)params;
+  *errmsg = tor_strdup("Cannot assign directly to virtual option.");
+  return -1;
+}
+
+static struct config_line_t *
+linelist_kv_encode(const char *key, const void *value,
+                   const void *params)
+{
+  (void)key;
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup(lines);
+}
+
+static struct config_line_t *
+linelist_s_kv_encode(const char *key, const void *value,
+                     const void *params)
+{
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup_and_filter(lines, key);
+}
+
+static void
+linelist_clear(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **lines = target;
+  config_free_lines(*lines); // sets it to NULL
+}
+
+static bool
+linelist_eq(const void *a, const void *b, const void *params)
+{
+  (void)params;
+  const config_line_t *lines_a = *(const config_line_t **)a;
+  const config_line_t *lines_b = *(const config_line_t **)b;
+  return config_lines_eq(lines_a, lines_b);
+}
+
+static int
+linelist_copy(void *target, const void *value, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  const config_line_t *val = *(const config_line_t **)value;
+  config_free_lines(*ptr);
+  *ptr = config_lines_dup(val);
+  return 0;
+}
+
+static void
+linelist_mark_fragile(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  if (*ptr)
+    (*ptr)->fragile = 1;
+}
+
+static const var_type_fns_t linelist_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_v_fns = {
+  .kv_parse = linelist_kv_virt_noparse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_s_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_s_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+};
+
+/////
+// CONFIG_TYPE_ROUTERSET
+//
+// XXXX to this module.
+/////
+
+/////
+// CONFIG_TYPE_IGNORE
+//
+// Used to indicate an option that cannot be stored or encoded.
+/////
+
+static int
+ignore_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)target;
+  (void)value;
+  (void)errmsg;
+  (void)params;
+  return 0;
+}
+
+static char *
+ignore_encode(const void *value, const void *params)
+{
+  (void)value;
+  (void)params;
+  return NULL;
+}
+
+static const var_type_fns_t ignore_fns = {
+  .parse = ignore_parse,
+  .encode = ignore_encode,
+};
+
+const var_type_def_t STRING_type_defn = {
+  .name="String", .fns=&string_fns };
+const var_type_def_t FILENAME_type_defn = {
+  .name="Filename", .fns=&string_fns };
+const var_type_def_t INT_type_defn = {
+  .name="SignedInteger", .fns=&int_fns,
+  .params=&INT_PARSE_UNRESTRICTED };
+const var_type_def_t POSINT_type_defn = {
+  .name="Integer", .fns=&int_fns,
+  .params=&INT_PARSE_POSINT };
+const var_type_def_t UINT64_type_defn = {
+  .name="Integer", .fns=&uint64_fns, };
+const var_type_def_t MEMUNIT_type_defn = {
+  .name="DataSize", .fns=&memunit_fns,
+  .params=&memory_units };
+const var_type_def_t INTERVAL_type_defn = {
+  .name="TimeInterval", .fns=&interval_fns,
+  .params=&time_units };
+const var_type_def_t MSEC_INTERVAL_type_defn = {
+  .name="TimeMsecInterval",
+  .fns=&interval_fns,
+  .params=&time_msec_units };
+const var_type_def_t DOUBLE_type_defn = {
+  .name="Float", .fns=&double_fns, };
+const var_type_def_t BOOL_type_defn = {
+  .name="Boolean", .fns=&enum_fns,
+  .params=&enum_params_bool };
+const var_type_def_t AUTOBOOL_type_defn = {
+  .name="Boolean+Auto", .fns=&enum_fns,
+  .params=&enum_params_autobool };
+const var_type_def_t ISOTIME_type_defn = {
+  .name="Time", .fns=&time_fns, };
+const var_type_def_t CSV_type_defn = {
+  .name="CommaList", .fns=&csv_fns, };
+const var_type_def_t CSV_INTERVAL_type_defn = {
+  .name="TimeInterval",
+  .fns=&legacy_csv_interval_fns, };
+const var_type_def_t LINELIST_type_defn = {
+  .name="LineList", .fns=&linelist_fns,
+  .flags=CFLG_NOREPLACE };
+/*
+ * A "linelist_s" is a derived view of a linelist_v: inspecting
+ * it gets part of a linelist_v, and setting it adds to the linelist_v.
+ */
+const var_type_def_t LINELIST_S_type_defn = {
+  .name="Dependent", .fns=&linelist_s_fns,
+  .flags=CFLG_NOREPLACE|
+  /* The operations we disable here are
+   * handled by the linelist_v. */
+  CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP };
+const var_type_def_t LINELIST_V_type_defn = {
+  .name="Virtual", .fns=&linelist_v_fns,
+  .flags=CFLG_NOREPLACE|CFLG_NOSET };
+const var_type_def_t IGNORE_type_defn = {
+  .name="Ignored", .fns=&ignore_fns,
+  .flags=CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET,
+};
+const var_type_def_t OBSOLETE_type_defn = {
+  .name="Obsolete", .fns=&ignore_fns,
+  .flags=CFLG_GROUP_OBSOLETE,
+};
+
+/**
+ * Table mapping conf_type_t values to var_type_def_t objects.
+ **/
+static const var_type_def_t *type_definitions_table[] = {
+  [CONFIG_TYPE_STRING] = &STRING_type_defn,
+  [CONFIG_TYPE_FILENAME] = &FILENAME_type_defn,
+  [CONFIG_TYPE_INT] = &INT_type_defn,
+  [CONFIG_TYPE_POSINT] = &POSINT_type_defn,
+  [CONFIG_TYPE_UINT64] = &UINT64_type_defn,
+  [CONFIG_TYPE_MEMUNIT] = &MEMUNIT_type_defn,
+  [CONFIG_TYPE_INTERVAL] = &INTERVAL_type_defn,
+  [CONFIG_TYPE_MSEC_INTERVAL] = &MSEC_INTERVAL_type_defn,
+  [CONFIG_TYPE_DOUBLE] = &DOUBLE_type_defn,
+  [CONFIG_TYPE_BOOL] = &BOOL_type_defn,
+  [CONFIG_TYPE_AUTOBOOL] = &AUTOBOOL_type_defn,
+  [CONFIG_TYPE_ISOTIME] = &ISOTIME_type_defn,
+  [CONFIG_TYPE_CSV] = &CSV_type_defn,
+  [CONFIG_TYPE_CSV_INTERVAL] = &CSV_INTERVAL_type_defn,
+  [CONFIG_TYPE_LINELIST] = &LINELIST_type_defn,
+  [CONFIG_TYPE_LINELIST_S] = &LINELIST_S_type_defn,
+  [CONFIG_TYPE_LINELIST_V] = &LINELIST_V_type_defn,
+  [CONFIG_TYPE_IGNORE] = &IGNORE_type_defn,
+  [CONFIG_TYPE_OBSOLETE] = &OBSOLETE_type_defn,
+};
+
+/**
+ * Return a pointer to the var_type_def_t object for the given
+ * config_type_t value, or NULL if no such type definition exists.
+ **/
+const var_type_def_t *
+lookup_type_def(config_type_t type)
+{
+  int t = type;
+  tor_assert(t >= 0);
+  if (t >= (int)ARRAY_LENGTH(type_definitions_table))
+    return NULL;
+  return type_definitions_table[t];
+}
diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h
new file mode 100644
index 0000000000..fec002b1d3
--- /dev/null
+++ b/src/lib/confmgt/type_defs.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.h
+ * @brief Header for lib/confmgt/type_defs.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H
+#define TOR_LIB_CONFMGT_TYPE_DEFS_H
+
+const struct var_type_def_t *lookup_type_def(config_type_t type);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */
diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c
new file mode 100644
index 0000000000..1955302cdc
--- /dev/null
+++ b/src/lib/confmgt/typedvar.c
@@ -0,0 +1,236 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.c
+ * @brief Functions for accessing a pointer as an object of a given type.
+ *
+ * These functions represent a low-level API for accessing a typed variable.
+ * They are used in the configuration system to examine and set fields in
+ * configuration objects used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+
+/**
+ * Try to parse a string in <b>value</b> that encodes an object of the type
+ * defined by <b>def</b>.
+ *
+ * On success, adjust the lvalue pointed to by <b>target</b> to hold that
+ * value, and return 0.  On failure, set *<b>errmsg</b> to a newly allocated
+ * string holding an error message, and return -1.
+ **/
+int
+typed_var_assign(void *target, const char *value, char **errmsg,
+                    const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  // clear old value if needed.
+  typed_var_free(target, def);
+
+  tor_assert(def->fns->parse);
+  return def->fns->parse(target, value, errmsg, def->params);
+}
+
+/**
+ * Try to parse a single line from the head of<b>line</b> that encodes an
+ * object of the type defined in <b>def</b>. On success and failure, behave as
+ * typed_var_assign().
+ *
+ * All types for which keys are significant should use this function.
+ *
+ * Note that although multiple lines may be provided in <b>line</b>,
+ * only the first one is handled by this function.
+ **/
+int
+typed_var_kvassign(void *target, const config_line_t *line,
+                      char **errmsg, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+
+  if (def->fns->kv_parse) {
+    // We do _not_ free the old value here, since linelist options
+    // sometimes have append semantics.
+    return def->fns->kv_parse(target, line, errmsg, def->params);
+  }
+
+  int rv = typed_var_assign(target, line->value, errmsg, def);
+  if (rv < 0 && *errmsg != NULL) {
+    /* typed_var_assign() didn't know the line's keyword, but we do.
+     * Let's add it to the error message. */
+    char *oldmsg = *errmsg;
+    tor_asprintf(errmsg, "Could not parse %s: %s", line->key, oldmsg);
+    tor_free(oldmsg);
+  }
+  return rv;
+}
+
+/**
+ * Release storage held by a variable in <b>target</b> of type defined by
+ * <b>def</b>, and set <b>target</b> to a reasonable default.
+ **/
+void
+typed_var_free(void *target, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return; // LCOV_EXCL_LINE
+  if (def->fns->clear) {
+    def->fns->clear(target, def->params);
+  }
+}
+
+/**
+ * Encode a value of type <b>def</b> pointed to by <b>value</b>, and return
+ * its result in a newly allocated string.  The string may need to be escaped.
+ *
+ * Returns NULL if this option has a NULL value, or on internal error.
+ **/
+char *
+typed_var_encode(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  tor_assert(def->fns->encode);
+  return def->fns->encode(value, def->params);
+}
+
+/**
+ * As typed_var_encode(), but returns a newly allocated config_line_t
+ * object.  The provided <b>key</b> is used as the key of the lines, unless
+ * the type is one (line a linelist) that encodes its own keys.
+ *
+ * This function may return a list of multiple lines.
+ *
+ * Returns NULL if there are no lines to encode, or on internal error.
+ */
+config_line_t *
+typed_var_kvencode(const char *key, const void *value,
+                      const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  if (def->fns->kv_encode) {
+    return def->fns->kv_encode(key, value, def->params);
+  }
+  char *encoded_value = typed_var_encode(value, def);
+  if (!encoded_value)
+    return NULL;
+
+  config_line_t *result = tor_malloc_zero(sizeof(config_line_t));
+  result->key = tor_strdup(key);
+  result->value = encoded_value;
+  return result;
+}
+
+/**
+ * Set <b>dest</b> to contain the same value as <b>src</b>. Both types
+ * must be as defined by <b>def</b>.
+ *
+ * Return 0 on success, and -1 on failure.
+ **/
+int
+typed_var_copy(void *dest, const void *src, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  if (def->fns->copy) {
+    // If we have been provided a copy fuction, use it.
+    return def->fns->copy(dest, src, def);
+  }
+
+  // Otherwise, encode 'src' and parse the result into 'def'.
+  char *enc = typed_var_encode(src, def);
+  if (!enc) {
+    typed_var_free(dest, def);
+    return 0;
+  }
+  char *err = NULL;
+  int rv = typed_var_assign(dest, enc, &err, def);
+  if (BUG(rv < 0)) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s",
+             escaped(enc), def->name, err?err:"");
+    // LCOV_EXCL_STOP
+  }
+  tor_free(err);
+  tor_free(enc);
+  return rv;
+}
+
+/**
+ * Return true if <b>a</b> and <b>b</b> are semantically equivalent.
+ * Both types must be as defined by <b>def</b>.
+ **/
+bool
+typed_var_eq(const void *a, const void *b, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->eq) {
+    // Use a provided eq function if we got one.
+    return def->fns->eq(a, b, def->params);
+  }
+
+  // Otherwise, encode the values and compare them.
+  char *enc_a = typed_var_encode(a, def);
+  char *enc_b = typed_var_encode(b, def);
+  bool eq = !strcmp_opt(enc_a,enc_b);
+  tor_free(enc_a);
+  tor_free(enc_b);
+  return eq;
+}
+
+/**
+ * Check whether <b>value</b> encodes a valid value according to the
+ * type definition in <b>def</b>.
+ */
+bool
+typed_var_ok(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->ok)
+    return def->fns->ok(value, def->params);
+
+  return true;
+}
+
+/**
+ * Mark <b>value</b> -- a variable that ordinarily would be extended by
+ * assignment -- as "fragile", so that it will get replaced by the next
+ * assignment instead.
+ **/
+void
+typed_var_mark_fragile(void *value, const var_type_def_t *def)
+{
+  if (BUG(!def)) {
+    return; // LCOV_EXCL_LINE
+  }
+
+  if (def->fns->mark_fragile)
+    def->fns->mark_fragile(value, def->params);
+}
diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h
new file mode 100644
index 0000000000..cc90ed10a3
--- /dev/null
+++ b/src/lib/confmgt/typedvar.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.h
+ * @brief Header for lib/confmgt/typedvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H
+#define TOR_LIB_CONFMGT_TYPEDVAR_H
+
+#include <stdbool.h>
+
+enum config_type_t;
+struct config_line_t;
+
+typedef struct var_type_fns_t var_type_fns_t;
+typedef struct var_type_def_t var_type_def_t;
+
+int typed_var_assign(void *target, const char *value, char **errmsg,
+                        const var_type_def_t *def);
+void typed_var_free(void *target, const var_type_def_t *def);
+char *typed_var_encode(const void *value, const var_type_def_t *def);
+int typed_var_copy(void *dest, const void *src, const var_type_def_t *def);
+bool typed_var_eq(const void *a, const void *b, const var_type_def_t *def);
+bool typed_var_ok(const void *value, const var_type_def_t *def);
+
+int typed_var_kvassign(void *target, const struct config_line_t *line,
+                           char **errmsg, const var_type_def_t *def);
+struct config_line_t *typed_var_kvencode(const char *key, const void *value,
+                                            const var_type_def_t *def);
+
+void typed_var_mark_fragile(void *value, const var_type_def_t *def);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */
diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c
new file mode 100644
index 0000000000..99716e8d9d
--- /dev/null
+++ b/src/lib/confmgt/unitparse.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.c
+ * @brief Functions for parsing values with units from a configuration file.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/intmath/muldiv.h"
+
+#include <string.h>
+
+/** Table to map the names of memory units to the number of bytes they
+ * contain. */
+// clang-format off
+const struct unit_table_t memory_units[] = {
+  { "",          1 },
+  { "b",         1<< 0 },
+  { "byte",      1<< 0 },
+  { "bytes",     1<< 0 },
+  { "kb",        1<<10 },
+  { "kbyte",     1<<10 },
+  { "kbytes",    1<<10 },
+  { "kilobyte",  1<<10 },
+  { "kilobytes", 1<<10 },
+  { "kilobits",  1<<7  },
+  { "kilobit",   1<<7  },
+  { "kbits",     1<<7  },
+  { "kbit",      1<<7  },
+  { "m",         1<<20 },
+  { "mb",        1<<20 },
+  { "mbyte",     1<<20 },
+  { "mbytes",    1<<20 },
+  { "megabyte",  1<<20 },
+  { "megabytes", 1<<20 },
+  { "megabits",  1<<17 },
+  { "megabit",   1<<17 },
+  { "mbits",     1<<17 },
+  { "mbit",      1<<17 },
+  { "gb",        1<<30 },
+  { "gbyte",     1<<30 },
+  { "gbytes",    1<<30 },
+  { "gigabyte",  1<<30 },
+  { "gigabytes", 1<<30 },
+  { "gigabits",  1<<27 },
+  { "gigabit",   1<<27 },
+  { "gbits",     1<<27 },
+  { "gbit",      1<<27 },
+  { "tb",        UINT64_C(1)<<40 },
+  { "tbyte",     UINT64_C(1)<<40 },
+  { "tbytes",    UINT64_C(1)<<40 },
+  { "terabyte",  UINT64_C(1)<<40 },
+  { "terabytes", UINT64_C(1)<<40 },
+  { "terabits",  UINT64_C(1)<<37 },
+  { "terabit",   UINT64_C(1)<<37 },
+  { "tbits",     UINT64_C(1)<<37 },
+  { "tbit",      UINT64_C(1)<<37 },
+  { NULL, 0 },
+};
+// clang-format on
+
+/** Table to map the names of time units to the number of seconds they
+ * contain. */
+// clang-format off
+const struct unit_table_t time_units[] = {
+  { "",         1 },
+  { "second",   1 },
+  { "seconds",  1 },
+  { "minute",   60 },
+  { "minutes",  60 },
+  { "hour",     60*60 },
+  { "hours",    60*60 },
+  { "day",      24*60*60 },
+  { "days",     24*60*60 },
+  { "week",     7*24*60*60 },
+  { "weeks",    7*24*60*60 },
+  { "month",    2629728, }, /* about 30.437 days */
+  { "months",   2629728, },
+  { NULL, 0 },
+};
+// clang-format on
+
+/** Table to map the names of time units to the number of milliseconds
+ * they contain. */
+// clang-format off
+const struct unit_table_t time_msec_units[] = {
+  { "",         1 },
+  { "msec",     1 },
+  { "millisecond", 1 },
+  { "milliseconds", 1 },
+  { "second",   1000 },
+  { "seconds",  1000 },
+  { "minute",   60*1000 },
+  { "minutes",  60*1000 },
+  { "hour",     60*60*1000 },
+  { "hours",    60*60*1000 },
+  { "day",      24*60*60*1000 },
+  { "days",     24*60*60*1000 },
+  { "week",     7*24*60*60*1000 },
+  { "weeks",    7*24*60*60*1000 },
+  { NULL, 0 },
+};
+// clang-format on
+
+/** Parse a string <b>val</b> containing a number, zero or more
+ * spaces, and an optional unit string.  If the unit appears in the
+ * table <b>u</b>, then multiply the number by the unit multiplier.
+ * On success, set *<b>ok</b> to 1 and return this product.
+ * Otherwise, set *<b>ok</b> to 0.
+ *
+ * If an error (like overflow or a negative value is detected), put an error
+ * message in *<b>errmsg_out</b> if that pointer is non-NULL, and otherwise
+ * log a warning.
+ */
+uint64_t
+config_parse_units(const char *val, const unit_table_t *u, int *ok,
+                   char **errmsg_out)
+{
+  uint64_t v = 0;
+  double d = 0;
+  int use_float = 0;
+  char *cp;
+  char *errmsg = NULL;
+
+  tor_assert(ok);
+
+  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
+  if (!*ok || (cp && *cp == '.')) {
+    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
+    if (!*ok) {
+      tor_asprintf(&errmsg, "Unable to parse %s as a number", val);
+      goto done;
+    }
+    use_float = 1;
+  }
+
+  if (BUG(!cp)) {
+    // cp should always be non-NULL if the parse operation succeeds.
+
+    // LCOV_EXCL_START
+    *ok = 1;
+    v = use_float ? ((uint64_t)d) :  v;
+    goto done;
+    // LCOV_EXCL_STOP
+  }
+
+  cp = (char*) eat_whitespace(cp);
+
+  for ( ;u->unit;++u) {
+    if (!strcasecmp(u->unit, cp)) {
+      if (use_float) {
+        d = u->multiplier * d;
+
+        if (d < 0) {
+          tor_asprintf(&errmsg, "Got a negative value while parsing %s %s",
+                       val, u->unit);
+          *ok = 0;
+          goto done;
+        }
+
+        // Some compilers may warn about casting a double to an unsigned type
+        // because they don't know if d is >= 0
+        if (d >= 0 && (d > (double)INT64_MAX || (uint64_t)d > INT64_MAX)) {
+          tor_asprintf(&errmsg, "Overflow while parsing %s %s",
+                       val, u->unit);
+          *ok = 0;
+          goto done;
+        }
+
+        v = (uint64_t) d;
+      } else {
+        v = tor_mul_u64_nowrap(v, u->multiplier);
+
+        if (v > INT64_MAX) {
+          tor_asprintf(&errmsg, "Overflow while parsing %s %s",
+                       val, u->unit);
+          *ok = 0;
+          goto done;
+        }
+      }
+
+      *ok = 1;
+      goto done;
+    }
+  }
+  tor_asprintf(&errmsg, "Unknown unit in %s", val);
+  *ok = 0;
+ done:
+
+  if (errmsg) {
+    tor_assert_nonfatal(!*ok);
+    if (errmsg_out) {
+      *errmsg_out = errmsg;
+    } else {
+      log_warn(LD_CONFIG, "%s", errmsg);
+      tor_free(errmsg);
+    }
+  }
+
+  if (*ok)
+    return v;
+  else
+    return 0;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
+ * and return the number of bytes specified.  Otherwise, set
+ * *<b>ok</b> to false and return 0. */
+uint64_t
+config_parse_memunit(const char *s, int *ok)
+{
+  uint64_t u = config_parse_units(s, memory_units, ok, NULL);
+  return u;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * time in milliseconds.  On success, set *<b>ok</b> to true and return
+ * the number of milliseconds in the provided interval.  Otherwise, set
+ * *<b>ok</b> to 0 and return -1. */
+int
+config_parse_msec_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_msec_units, ok, NULL);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of time.
+ * On success, set *<b>ok</b> to true and return the number of seconds in
+ * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
+ */
+int
+config_parse_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_units, ok, NULL);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h
new file mode 100644
index 0000000000..047e11b424
--- /dev/null
+++ b/src/lib/confmgt/unitparse.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.h
+ * @brief Header for lib/confmgt/unitparse.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_UNITPARSE_H
+#define TOR_LIB_CONFMGT_UNITPARSE_H
+
+#include <lib/cc/torint.h>
+
+/** Mapping from a unit name to a multiplier for converting that unit into a
+ * base unit.  Used by config_parse_unit. */
+typedef struct unit_table_t {
+  const char *unit; /**< The name of the unit */
+  uint64_t multiplier; /**< How many of the base unit appear in this unit */
+} unit_table_t;
+
+extern const unit_table_t memory_units[];
+extern const unit_table_t time_units[];
+extern const struct unit_table_t time_msec_units[];
+
+uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok,
+                            char **errmsg_out);
+
+uint64_t config_parse_memunit(const char *s, int *ok);
+int config_parse_msec_interval(const char *s, int *ok);
+int config_parse_interval(const char *s, int *ok);
+
+#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */
diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h
new file mode 100644
index 0000000000..2519b86aa0
--- /dev/null
+++ b/src/lib/confmgt/var_type_def_st.h
@@ -0,0 +1,167 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file var_type_def_st.h
+ * @brief Structure declarations for typedvar type definitions.
+ *
+ * This structure is used for defining new variable types.  If you are not
+ * defining a new variable type for use by the configuration management
+ * system, you don't need this structure.
+ *
+ * For defining new variables, see the types in conftypes.h.
+ *
+ * For data-driven access to configuration variables, see the other members of
+ * lib/confmgt/.
+ *
+ * STATUS NOTE: It is not yet possible to actually define new variables
+ *   outside of config.c, and many of the types that will eventually be used
+ *   to do so are not yet moved. This will change as more of #29211 is
+ *   completed.
+ **/
+
+#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+
+#include <stdbool.h>
+
+struct config_line_t;
+
+/**
+ * A structure full of functions pointers to implement a variable type.
+ *
+ * Every type MUST implement parse or kv_parse and encode or kv_encode;
+ * the other functions pointers MAY be NULL.
+ *
+ * All functions here take a <b>params</b> argument, whose value
+ * is determined by the type definition.  Two types may have the
+ * same functions, but differ only in parameters.
+ *
+ * Implementation considerations: If "" encodes a valid value for a type, try
+ * to make sure that it encodes the same thing as the default value for the
+ * type (that is, the value that is set by config_clear() or memset(0)). If
+ * this is not the case, you need to make extra certain that your parse/encode
+ * implementations preserve the NULL/"" distinction.
+ **/
+struct var_type_fns_t {
+  /**
+   * Try to parse a string in <b>value</b> that encodes an object of this
+   * type.  On success, adjust the lvalue pointed to by <b>target</b> to hold
+   * that value, and return 0.  On failure, set *<b>errmsg</b> to a newly
+   * allocated string holding an error message, and return -1.
+   **/
+  int (*parse)(void *target, const char *value, char **errmsg,
+               const void *params);
+  /**
+   * Try to parse a single line from the head of<b>line</b> that encodes
+   * an object of this type.  On success and failure, behave as in the parse()
+   * function.
+   *
+   * If this function is absent, it is implemented in terms of parse().
+   *
+   * All types for which keys are significant should use this method. For
+   * example, a "linelist" type records the actual keys that are given
+   * for each line, and so should use this method.
+   *
+   * Note that although multiple lines may be provided in <b>line</b>,
+   * only the first one should be handled by this function.
+   **/
+  int (*kv_parse)(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params);
+  /**
+   * Encode a value pointed to by <b>value</b> and return its result
+   * in a newly allocated string.  The string may need to be escaped.
+   *
+   * If this function is absent, it is implemented in terms of kv_encode().
+   *
+   * Returns NULL if this option has a NULL value, or on internal error.
+   *
+   * Requirement: all strings generated by encode() should produce a
+   * semantically equivalent value when given to parse().
+   **/
+  char *(*encode)(const void *value, const void *params);
+  /**
+   * As encode(), but returns a newly allocated config_line_t object.  The
+   * provided <b>key</b> is used as the key of the lines, unless the type is
+   * one that encodes its own keys.
+   *
+   * Unlike kv_parse(), this function will return a list of multiple lines,
+   * if <b>value</b> is such that it must be encoded by multiple lines.
+   *
+   * Returns NULL if there are no lines to encode, or on internal error.
+   *
+   * If this function is absent, it is implemented in terms of encode().
+   **/
+  struct config_line_t *(*kv_encode)(const char *key, const void *value,
+                                     const void *params);
+  /**
+   * Free all storage held in <b>arg</b>, and set <b>arg</b> to a default
+   * value -- usually zero or NULL.
+   *
+   * If this function is absent, the default implementation does nothing.
+   **/
+  void (*clear)(void *arg, const void *params);
+  /**
+   * Return true if <b>a</b> and <b>b</b> hold the same value, and false
+   * otherwise.
+   *
+   * If this function is absent, it is implemented by encoding both a and
+   * b and comparing their encoded strings for equality.
+   **/
+  bool (*eq)(const void *a, const void *b, const void *params);
+  /**
+   * Try to copy the value from <b>value</b> into <b>target</b>.
+   * On success return 0; on failure return -1.
+   *
+   * If this function is absent, it is implemented by encoding the value
+   * into a string, and then parsing it into the target.
+   **/
+  int (*copy)(void *target, const void *value, const void *params);
+  /**
+   * Check whether <b>value</b> holds a valid value according to the
+   * rules of this type; return true if it does and false if it doesn't.
+   *
+   * The default implementation for this function assumes that all
+   * values are valid.
+   **/
+  bool (*ok)(const void *value, const void *params);
+  /**
+   * Mark a value of this variable as "fragile", so that future attempts to
+   * assign to this variable will replace rather than extending it.
+   *
+   * The default implementation for this function does nothing.
+   *
+   * Only meaningful for types with is_cumulative set.
+   **/
+  void (*mark_fragile)(void *value, const void *params);
+};
+
+/**
+ * A structure describing a type that can be manipulated with the typedvar_*
+ * functions.
+ **/
+struct var_type_def_t {
+  /**
+   * The name of this type. Should not include spaces. Used for
+   * debugging, log messages, and the controller API. */
+  const char *name;
+  /**
+   * A function table for this type.
+   */
+  const struct var_type_fns_t *fns;
+  /**
+   * A pointer to a value that should be passed as the 'params' argument when
+   * calling the functions in this type's function table.
+   */
+  const void *params;
+  /**
+   * A bitwise OR of one or more VTFLAG_* values, describing properties
+   * for all values of this type.
+   **/
+  uint32_t flags;
+};
+
+#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */
diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include
index 90de5eda40..81507527d3 100644
--- a/src/lib/container/.may_include
+++ b/src/lib/container/.may_include
@@ -7,12 +7,9 @@ lib/malloc/*.h
 lib/err/*.h
 lib/smartlist_core/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
 lib/intmath/*.h
 lib/log/*.h
 
-# XXXX I am unsure about this one. It's only here for buffers.c
-lib/time/*.h
-
-ht.h
-siphash.h
+ext/ht.h
+ext/siphash.h
diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h
index 910d5fea65..41409e350a 100644
--- a/src/lib/container/bitarray.h
+++ b/src/lib/container/bitarray.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BITARRAY_H
@@ -83,4 +83,4 @@ bitarray_is_set(bitarray_t *b, int bit)
   return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_BITARRAY_H) */
diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c
index 9aa9b1ee56..34b1265d81 100644
--- a/src/lib/container/bloomfilt.c
+++ b/src/lib/container/bloomfilt.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,7 +14,7 @@
 #include "lib/container/bloomfilt.h"
 #include "lib/intmath/bits.h"
 #include "lib/log/util_bug.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** How many bloom-filter bits we set per address. This is twice the
  * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit
diff --git a/src/lib/container/bloomfilt.h b/src/lib/container/bloomfilt.h
index 0ce18bd3ec..6d36056b5a 100644
--- a/src/lib/container/bloomfilt.h
+++ b/src/lib/container/bloomfilt.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BLOOMFILT_H
diff --git a/src/lib/container/handles.h b/src/lib/container/handles.h
index ca7c94559e..6b1bbd5167 100644
--- a/src/lib/container/handles.h
+++ b/src/lib/container/handles.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,33 +16,33 @@
  * To enable a type to have handles, add a HANDLE_ENTRY() field in its
  * definition, as in:
  *
- *     struct walrus {
- *         HANDLE_ENTRY(wlr, walrus);
+ *     struct walrus_t {
+ *         HANDLE_ENTRY(wlr, walrus_t);
  *         // ...
  *     };
  *
- * And invoke HANDLE_DECL(wlr, walrus, [static]) to declare the handle
+ * And invoke HANDLE_DECL(wlr, walrus_t, [static]) to declare the handle
  * manipulation functions (typically in a header):
  *
  *     // opaque handle to walrus.
  *     typedef struct wlr_handle_t wlr_handle_t;
  *
  *     // make a new handle
- *     struct wlr_handle_t *wlr_handle_new(struct walrus *);
+ *     struct wlr_handle_t *wlr_handle_new(struct walrus_t *);
  *
  *     // release a handle
  *     void wlr_handle_free(wlr_handle_t *);
  *
  *     // return the pointed-to walrus, or NULL.
- *     struct walrus *wlr_handle_get(wlr_handle_t *).
+ *     struct walrus_t *wlr_handle_get(wlr_handle_t *).
  *
  *     // call this function when you're about to free the walrus;
  *     // it invalidates all handles. (IF YOU DON'T, YOU WILL HAVE
  *     // DANGLING REFERENCES)
- *     void wlr_handles_clear(struct walrus *);
+ *     void wlr_handles_clear(struct walrus_t *);
  *
  * Finally, use HANDLE_IMPL() to define the above functions in some
- * appropriate C file: HANDLE_IMPL(wlr, walrus, [static])
+ * appropriate C file: HANDLE_IMPL(wlr, walrus_t, [static])
  *
  **/
 
@@ -57,12 +57,13 @@
 #define HANDLE_ENTRY(name, structname)         \
   struct name ## _handle_head_t *handle_head
 
-#define HANDLE_DECL(name, structname, linkage)                          \
+#define HANDLE_DECL(name, structname_t, linkage)                        \
   typedef struct name ## _handle_t name ## _handle_t;                   \
-  linkage  name ## _handle_t *name ## _handle_new(struct structname *object); \
+  linkage name ## _handle_t *name ## _handle_new(                       \
+                                          struct structname_t *object); \
   linkage void name ## _handle_free_(name ## _handle_t *);              \
-  linkage struct structname *name ## _handle_get(name ## _handle_t *);  \
-  linkage void name ## _handles_clear(struct structname *object);
+  linkage struct structname_t *name ## _handle_get(name ## _handle_t *); \
+  linkage void name ## _handles_clear(struct structname_t *object);
 
 /*
  * Implementation notes: there are lots of possible implementations here.  We
diff --git a/src/lib/container/include.am b/src/lib/container/include.am
index e6492098b5..00d7b8e587 100644
--- a/src/lib/container/include.am
+++ b/src/lib/container/include.am
@@ -5,10 +5,11 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-container-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_container_a_SOURCES =			\
 	src/lib/container/bloomfilt.c			\
-	src/lib/container/buffers.c			\
 	src/lib/container/map.c				\
+	src/lib/container/namemap.c			\
 	src/lib/container/order.c			\
 	src/lib/container/smartlist.c
 
@@ -17,11 +18,13 @@ src_lib_libtor_container_testing_a_SOURCES = \
 src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/container/bitarray.h			\
 	src/lib/container/bloomfilt.h			\
-	src/lib/container/buffers.h			\
 	src/lib/container/handles.h			\
 	src/lib/container/map.h				\
+	src/lib/container/namemap.h			\
+	src/lib/container/namemap_st.h			\
 	src/lib/container/order.h			\
 	src/lib/container/smartlist.h
diff --git a/src/lib/container/lib_container.md b/src/lib/container/lib_container.md
new file mode 100644
index 0000000000..f4902ca44a
--- /dev/null
+++ b/src/lib/container/lib_container.md
@@ -0,0 +1,49 @@
+@dir /lib/container
+@brief lib/container: Hash tables, dynamic arrays, bit arrays, etc.
+
+### Smartlists: Neither lists, nor especially smart.
+
+For historical reasons, we call our dynamic-allocated array type
+`smartlist_t`.  It can grow or shrink as elements are added and removed.
+
+All smartlists hold an array of `void *`.  Whenever you expose a smartlist
+in an API you *must* document which types its pointers actually hold.
+
+<!-- It would be neat to fix that, wouldn't it? -NM  -->
+
+Smartlists are created empty with `smartlist_new()` and freed with
+`smartlist_free()`.  See the `containers.h` header documentation for more
+information; there are many convenience functions for commonly needed
+operations.
+
+For low-level operations on smartlists, see also
+\refdir{lib/smartlist_core}.
+
+<!-- TODO: WRITE more about what you can do with smartlists. -->
+
+### Digest maps, string maps, and more.
+
+Tor makes frequent use of maps from 160-bit digests, 256-bit digests,
+or nul-terminated strings to `void *`. These types are `digestmap_t`,
+`digest256map_t`, and `strmap_t` respectively.  See the containers.h
+module documentation for more information.
+
+### Intrusive lists and hashtables
+
+For performance-sensitive cases, we sometimes want to use "intrusive"
+collections: ones where the bookkeeping pointers are stuck inside the
+structures that belong to the collection.  If you've used the
+BSD-style sys/queue.h macros, you'll be familiar with these.
+
+Unfortunately, the `sys/queue.h` macros vary significantly between the
+platforms that have them, so we provide our own variants in
+`ext/tor_queue.h`.
+
+We also provide an intrusive hashtable implementation in `ext/ht.h`.
+When you're using it, you'll need to define your own hash
+functions. If attacker-induced collisions are a worry here, use the
+cryptographic siphash24g function to extract hashes.
+
+<!-- TODO: WRITE about bloom filters, namemaps, bit-arrays, order functions.
+-->
+
diff --git a/src/lib/container/map.c b/src/lib/container/map.c
index d213ad50bf..7db84313ea 100644
--- a/src/lib/container/map.c
+++ b/src/lib/container/map.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,7 +21,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "ht.h"
+#include "ext/ht.h"
 
 /** Helper: Declare an entry type and a map type to implement a mapping using
  * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
@@ -85,21 +85,21 @@ digest256map_entry_hash(const digest256map_entry_t *a)
 }
 
 HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq)
+             strmap_entries_eq);
 HT_GENERATE2(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 HT_PROTOTYPE(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq)
+             digestmap_entries_eq);
 HT_GENERATE2(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 HT_PROTOTYPE(digest256map_impl, digest256map_entry_t, node,
              digest256map_entry_hash,
-             digest256map_entries_eq)
+             digest256map_entries_eq);
 HT_GENERATE2(digest256map_impl, digest256map_entry_t, node,
              digest256map_entry_hash,
-             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 #define strmap_entry_free(ent) \
   FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent))
diff --git a/src/lib/container/map.h b/src/lib/container/map.h
index a2d1b01d12..dbc1967247 100644
--- a/src/lib/container/map.h
+++ b/src/lib/container/map.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_MAP_H
@@ -15,36 +15,37 @@
 #include "lib/testsupport/testsupport.h"
 #include "lib/cc/torint.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
-#define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##entry_t *prefix##iter_t;                       \
-  MOCK_DECL(maptype*, prefix##new, (void));                             \
-  void* prefix##set(maptype *map, keytype key, void *val);              \
-  void* prefix##get(const maptype *map, keytype key);                   \
-  void* prefix##remove(maptype *map, keytype key);                      \
-  MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \
-  int prefix##isempty(const maptype *map);                              \
-  int prefix##size(const maptype *map);                                 \
-  prefix##iter_t *prefix##iter_init(maptype *map);                      \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter); \
-  prefix##iter_t *prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter); \
-  void prefix##iter_get(prefix##iter_t *iter, keytype *keyp, void **valp); \
-  int prefix##iter_done(prefix##iter_t *iter);                          \
-  void prefix##assert_ok(const maptype *map)
+#define DECLARE_MAP_FNS(mapname_t, keytype, prefix)                     \
+  typedef struct mapname_t mapname_t;                                   \
+  typedef struct prefix##_entry_t *prefix##_iter_t;                       \
+  MOCK_DECL(mapname_t*, prefix##_new, (void));                           \
+  void* prefix##_set(mapname_t *map, keytype key, void *val);            \
+  void* prefix##_get(const mapname_t *map, keytype key);                 \
+  void* prefix##_remove(mapname_t *map, keytype key);                    \
+  MOCK_DECL(void, prefix##_free_, (mapname_t *map, void (*free_val)(void*))); \
+  int prefix##_isempty(const mapname_t *map);                            \
+  int prefix##_size(const mapname_t *map);                               \
+  prefix##_iter_t *prefix##_iter_init(mapname_t *map);                    \
+  prefix##_iter_t *prefix##_iter_next(mapname_t *map, prefix##_iter_t *iter); \
+  prefix##_iter_t *prefix##_iter_next_rmv(mapname_t *map,                 \
+                                        prefix##_iter_t *iter);          \
+  void prefix##_iter_get(prefix##_iter_t *iter, keytype *keyp, void **valp); \
+  int prefix##_iter_done(prefix##_iter_t *iter);                          \
+  void prefix##_assert_ok(const mapname_t *map)
 
 /* Map from const char * to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(strmap_t, const char *, strmap_);
+DECLARE_MAP_FNS(strmap_t, const char *, strmap);
 /* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_);
+DECLARE_MAP_FNS(digestmap_t, const char *, digestmap);
 /* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash
  * table. */
-DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
+DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map);
 
-#define MAP_FREE_AND_NULL(maptype, map, fn)     \
+#define MAP_FREE_AND_NULL(mapname_t, map, fn)     \
   do {                                          \
-    maptype ## _free_((map), (fn));             \
+    mapname_t ## _free_((map), (fn));             \
     (map) = NULL;                               \
   } while (0)
 
@@ -55,12 +56,12 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
 #undef DECLARE_MAP_FNS
 
 /** Iterates over the key-value pairs in a map <b>map</b> in order.
- * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap_ or digestmap_).
+ * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap or digestmap).
  * The map's keys and values are of type keytype and valtype respectively;
  * each iteration assigns them to keyvar and valvar.
  *
  * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *   MAP_FOREACH(digestmap, m, const char *, k, routerinfo_t *, r) {
  *     // use k and r
  *   } MAP_FOREACH_END.
  */
@@ -80,21 +81,21 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  */
 #define MAP_FOREACH(prefix, map, keytype, keyvar, valtype, valvar)      \
   STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
-         keyvar##_iter = prefix##iter_next(map, keyvar##_iter)) {       \
+    prefix##_iter_t *keyvar##_iter;                                      \
+    for (keyvar##_iter = prefix##_iter_init(map);                        \
+         !prefix##_iter_done(keyvar##_iter);                             \
+         keyvar##_iter = prefix##_iter_next(map, keyvar##_iter)) {       \
       keytype keyvar;                                                   \
       void *valvar##_voidp;                                             \
       valtype valvar;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      prefix##_iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
       valvar = valvar##_voidp;
 
 /** As MAP_FOREACH, except allows members to be removed from the map
  * during the iteration via MAP_DEL_CURRENT.  Example use:
  *
  * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *   MAP_FOREACH(digestmap, m, const char *, k, routerinfo_t *, r) {
  *      if (is_very_old(r))
  *       MAP_DEL_CURRENT(k);
  *   } MAP_FOREACH_END.
@@ -120,18 +121,18 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  */
 #define MAP_FOREACH_MODIFY(prefix, map, keytype, keyvar, valtype, valvar) \
   STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
+    prefix##_iter_t *keyvar##_iter;                                      \
     int keyvar##_del=0;                                                 \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
+    for (keyvar##_iter = prefix##_iter_init(map);                        \
+         !prefix##_iter_done(keyvar##_iter);                             \
          keyvar##_iter = keyvar##_del ?                                 \
-           prefix##iter_next_rmv(map, keyvar##_iter) :                  \
-           prefix##iter_next(map, keyvar##_iter)) {                     \
+           prefix##_iter_next_rmv(map, keyvar##_iter) :                  \
+           prefix##_iter_next(map, keyvar##_iter)) {                     \
       keytype keyvar;                                                   \
       void *valvar##_voidp;                                             \
       valtype valvar;                                                   \
       keyvar##_del=0;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      prefix##_iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
       valvar = valvar##_voidp;
 
 /** Used with MAP_FOREACH_MODIFY to remove the currently-iterated-upon
@@ -151,7 +152,7 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  *   } DIGESTMAP_FOREACH_END.
  */
 #define DIGESTMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(digestmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH(digestmap, map, const char *, keyvar, valtype, valvar)
 
 /** As MAP_FOREACH_MODIFY, but does not require declaration of prefix or
  * keytype.
@@ -162,89 +163,89 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  *   } DIGESTMAP_FOREACH_END.
  */
 #define DIGESTMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(digestmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH_MODIFY(digestmap, map, const char *, keyvar, valtype, valvar)
 /** Used to end a DIGESTMAP_FOREACH() block. */
 #define DIGESTMAP_FOREACH_END MAP_FOREACH_END
 
 #define DIGEST256MAP_FOREACH(map, keyvar, valtype, valvar)               \
-  MAP_FOREACH(digest256map_, map, const uint8_t *, keyvar, valtype, valvar)
+  MAP_FOREACH(digest256map, map, const uint8_t *, keyvar, valtype, valvar)
 #define DIGEST256MAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)       \
-  MAP_FOREACH_MODIFY(digest256map_, map, const uint8_t *,               \
+  MAP_FOREACH_MODIFY(digest256map, map, const uint8_t *,               \
                      keyvar, valtype, valvar)
 #define DIGEST256MAP_FOREACH_END MAP_FOREACH_END
 
 #define STRMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(strmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH(strmap, map, const char *, keyvar, valtype, valvar)
 #define STRMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(strmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH_MODIFY(strmap, map, const char *, keyvar, valtype, valvar)
 #define STRMAP_FOREACH_END MAP_FOREACH_END
 
 void* strmap_set_lc(strmap_t *map, const char *key, void *val);
 void* strmap_get_lc(const strmap_t *map, const char *key);
 void* strmap_remove_lc(strmap_t *map, const char *key);
 
-#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, maptype, valtype)           \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##iter_t *prefix##iter_t;                        \
-  ATTR_UNUSED static inline maptype*                                    \
-  prefix##new(void)                                                     \
+#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, mapname_t, valtype)           \
+  typedef struct mapname_t mapname_t;                                   \
+  typedef struct prefix##_iter_t *prefix##_iter_t;                        \
+  ATTR_UNUSED static inline mapname_t*                                  \
+  prefix##_new(void)                                                     \
   {                                                                     \
-    return (maptype*)digestmap_new();                                   \
+    return (mapname_t*)digestmap_new();                                 \
   }                                                                     \
   ATTR_UNUSED static inline digestmap_t*                                \
-  prefix##to_digestmap(maptype *map)                                    \
+  prefix##_to_digestmap(mapname_t *map)                                  \
   {                                                                     \
     return (digestmap_t*)map;                                           \
   }                                                                     \
   ATTR_UNUSED static inline valtype*                                    \
-  prefix##get(maptype *map, const char *key)     \
+  prefix##_get(mapname_t *map, const char *key)                          \
   {                                                                     \
     return (valtype*)digestmap_get((digestmap_t*)map, key);             \
   }                                                                     \
   ATTR_UNUSED static inline valtype*                                    \
-  prefix##set(maptype *map, const char *key, valtype *val)              \
+  prefix##_set(mapname_t *map, const char *key, valtype *val)            \
   {                                                                     \
     return (valtype*)digestmap_set((digestmap_t*)map, key, val);        \
   }                                                                     \
   ATTR_UNUSED static inline valtype*                                    \
-  prefix##remove(maptype *map, const char *key)                         \
+  prefix##_remove(mapname_t *map, const char *key)                       \
   {                                                                     \
     return (valtype*)digestmap_remove((digestmap_t*)map, key);          \
   }                                                                     \
   ATTR_UNUSED static inline void                                        \
-  prefix##f##ree_(maptype *map, void (*free_val)(void*))                \
+  prefix##_f##ree_(mapname_t *map, void (*free_val)(void*))              \
   {                                                                     \
     digestmap_free_((digestmap_t*)map, free_val);                       \
   }                                                                     \
   ATTR_UNUSED static inline int                                         \
-  prefix##isempty(maptype *map)                                         \
+  prefix##_isempty(mapname_t *map)                                       \
   {                                                                     \
     return digestmap_isempty((digestmap_t*)map);                        \
   }                                                                     \
   ATTR_UNUSED static inline int                                         \
-  prefix##size(maptype *map)                                            \
+  prefix##_size(mapname_t *map)                                          \
   {                                                                     \
     return digestmap_size((digestmap_t*)map);                           \
   }                                                                     \
   ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_init(maptype *map)                       \
+  prefix##_iter_t *prefix##_iter_init(mapname_t *map)                     \
   {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_init((digestmap_t*)map);    \
+    return (prefix##_iter_t*) digestmap_iter_init((digestmap_t*)map);    \
   }                                                                     \
   ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter) \
+  prefix##_iter_t *prefix##_iter_next(mapname_t *map, prefix##_iter_t *iter) \
   {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next(                       \
+    return (prefix##_iter_t*) digestmap_iter_next(                       \
                        (digestmap_t*)map, (digestmap_iter_t*)iter);     \
   }                                                                     \
-  ATTR_UNUSED static inline prefix##iter_t*                             \
-  prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter)             \
+  ATTR_UNUSED static inline prefix##_iter_t*                             \
+  prefix##_iter_next_rmv(mapname_t *map, prefix##_iter_t *iter)           \
   {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next_rmv(                   \
+    return (prefix##_iter_t*) digestmap_iter_next_rmv(                   \
                        (digestmap_t*)map, (digestmap_iter_t*)iter);     \
   }                                                                     \
   ATTR_UNUSED static inline void                                        \
-  prefix##iter_get(prefix##iter_t *iter,                                \
+  prefix##_iter_get(prefix##_iter_t *iter,                                \
                    const char **keyp,                                   \
                    valtype **valp)                                      \
   {                                                                     \
@@ -253,9 +254,9 @@ void* strmap_remove_lc(strmap_t *map, const char *key);
     *valp = v;                                                          \
   }                                                                     \
   ATTR_UNUSED static inline int                                         \
-  prefix##iter_done(prefix##iter_t *iter)                               \
+  prefix##_iter_done(prefix##_iter_t *iter)                               \
   {                                                                     \
     return digestmap_iter_done((digestmap_iter_t*)iter);                \
   }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_MAP_H) */
diff --git a/src/lib/container/namemap.c b/src/lib/container/namemap.c
new file mode 100644
index 0000000000..e286cad947
--- /dev/null
+++ b/src/lib/container/namemap.c
@@ -0,0 +1,189 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file namemap.c
+ * @brief Mappings between identifiers and 16-bit ints.
+ **/
+
+#include "orconfig.h"
+#include "lib/container/smartlist.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "ext/siphash.h"
+
+#include <string.h>
+
+/** Helper for namemap hashtable implementation: compare two entries. */
+static inline int
+mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b)
+{
+  return !strcmp(a->name, b->name);
+}
+
+/** Helper for namemap hashtable implementation: hash an entry. */
+static inline unsigned
+mapped_name_hash(const mapped_name_t *a)
+{
+  return (unsigned) siphash24g(a->name, strlen(a->name));
+}
+
+HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq);
+HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq, 0.6, tor_reallocarray_, tor_free_);
+
+/** Set up an uninitialized <b>map</b>. */
+void
+namemap_init(namemap_t *map)
+{
+  memset(map, 0, sizeof(*map));
+  HT_INIT(namemap_ht, &map->ht);
+  map->names = smartlist_new();
+}
+
+/** Return the name that <b>map</b> associates with a given <b>id</b>, or
+ * NULL if there is no such name. */
+const char *
+namemap_get_name(const namemap_t *map, unsigned id)
+{
+  if (map->names && id < (unsigned)smartlist_len(map->names)) {
+    mapped_name_t *name = smartlist_get(map->names, (int)id);
+    return name->name;
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * Return the name that <b>map</b> associates with a given <b>id</b>, or a
+ * pointer to a statically allocated string describing the value of <b>id</b>
+ * if no such name exists.
+ **/
+const char *
+namemap_fmt_name(const namemap_t *map, unsigned id)
+{
+  static char buf[32];
+
+  const char *name = namemap_get_name(map, id);
+  if (name)
+    return name;
+
+  tor_snprintf(buf, sizeof(buf), "{%u}", id);
+
+  return buf;
+}
+
+/**
+ * Helper: As namemap_get_id(), but requires that <b>name</b> is
+ * <b>namelen</b> charaters long, and that <b>namelen</b> is no more than
+ * MAX_NAMEMAP_NAME_LEN.
+ */
+static unsigned
+namemap_get_id_unchecked(const namemap_t *map,
+                         const char *name,
+                         size_t namelen)
+{
+  union {
+    mapped_name_t n;
+    char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1];
+  } u;
+  memcpy(u.n.name, name, namelen);
+  u.n.name[namelen] = 0;
+  const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n);
+  if (found) {
+    tor_assert(map->names);
+    tor_assert(smartlist_get(map->names, found->intval) == found);
+    return found->intval;
+  }
+
+  return NAMEMAP_ERR;
+}
+
+/**
+ * Return the identifier currently associated by <b>map</b> with the name
+ * <b>name</b>, or NAMEMAP_ERR if no such identifier exists.
+ **/
+unsigned
+namemap_get_id(const namemap_t *map,
+               const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  return namemap_get_id_unchecked(map, name, namelen);
+}
+
+/**
+ * Return the identifier associated by <b>map</b> with the name
+ * <b>name</b>, allocating a new identifier in <b>map</b> if none exists.
+ *
+ * Return NAMEMAP_ERR if <b>name</b> is too long, or if there are no more
+ * identifiers we can allocate.
+ **/
+unsigned
+namemap_get_or_create_id(namemap_t *map,
+                         const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    map->names = smartlist_new();
+
+  unsigned found = namemap_get_id_unchecked(map, name, namelen);
+  if (found != NAMEMAP_ERR)
+    return found;
+
+  unsigned new_id = (unsigned)smartlist_len(map->names);
+  if (new_id == NAMEMAP_ERR)
+    return NAMEMAP_ERR; /* Can't allocate any more. */
+
+  mapped_name_t *insert = tor_malloc_zero(
+                       offsetof(mapped_name_t, name) + namelen + 1);
+  memcpy(insert->name, name, namelen+1);
+  insert->intval = new_id;
+
+  HT_INSERT(namemap_ht, &map->ht, insert);
+  smartlist_add(map->names, insert);
+
+  return new_id;
+}
+
+/** Return the number of entries in 'names' */
+size_t
+namemap_get_size(const namemap_t *map)
+{
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    return 0;
+
+  return smartlist_len(map->names);
+}
+
+/**
+ * Release all storage held in <b>map</b>.
+ */
+void
+namemap_clear(namemap_t *map)
+{
+  if (!map)
+    return;
+
+  HT_CLEAR(namemap_ht, &map->ht);
+  if (map->names) {
+    SMARTLIST_FOREACH(map->names, mapped_name_t *, n,
+                      tor_free(n));
+    smartlist_free(map->names);
+  }
+  memset(map, 0, sizeof(*map));
+}
diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h
new file mode 100644
index 0000000000..b451c18c68
--- /dev/null
+++ b/src/lib/container/namemap.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NAMEMAP_H
+#define TOR_NAMEMAP_H
+
+/**
+ * \file namemap.h
+ *
+ * \brief Header for namemap.c
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+#include <stddef.h>
+
+typedef struct namemap_t namemap_t;
+
+/** Returned in place of an identifier when an error occurs. */
+#define NAMEMAP_ERR UINT_MAX
+
+void namemap_init(namemap_t *map);
+const char *namemap_get_name(const namemap_t *map, unsigned id);
+const char *namemap_fmt_name(const namemap_t *map, unsigned id);
+unsigned namemap_get_id(const namemap_t *map,
+                        const char *name);
+unsigned namemap_get_or_create_id(namemap_t *map,
+                                  const char *name);
+size_t namemap_get_size(const namemap_t *map);
+void namemap_clear(namemap_t *map);
+
+#endif /* !defined(TOR_NAMEMAP_H) */
diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h
new file mode 100644
index 0000000000..39aa85cc09
--- /dev/null
+++ b/src/lib/container/namemap_st.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NAMEMAP_ST_H
+#define NAMEMAP_ST_H
+
+/**
+ * @file namemap_st.h
+ * @brief Internal declarations for namemap structure.
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+struct smartlist_t;
+
+/** Longest allowed name that's allowed in a namemap_t. */
+#define MAX_NAMEMAP_NAME_LEN 128
+
+/** An entry inside a namemap_t. Maps a string to a numeric identifier. */
+typedef struct mapped_name_t {
+  HT_ENTRY(mapped_name_t) node;
+  unsigned intval;
+  char name[FLEXIBLE_ARRAY_MEMBER];
+} mapped_name_t;
+
+/** A structure that allocates small numeric identifiers for names and maps
+ * back and forth between them.  */
+struct namemap_t {
+  HT_HEAD(namemap_ht, mapped_name_t) ht;
+  struct smartlist_t *names;
+};
+
+#ifndef COCCI
+/** Macro to initialize a namemap. */
+#define NAMEMAP_INIT() { HT_INITIALIZER(), NULL }
+#endif
+
+#endif /* !defined(NAMEMAP_ST_H) */
diff --git a/src/lib/container/order.c b/src/lib/container/order.c
index f6503a124e..cac241f027 100644
--- a/src/lib/container/order.c
+++ b/src/lib/container/order.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/container/order.h b/src/lib/container/order.h
index a176d6d8a6..5bca095f35 100644
--- a/src/lib/container/order.h
+++ b/src/lib/container/order.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ORDER_H
@@ -57,4 +57,4 @@ third_quartile_uint32(uint32_t *array, int n_elements)
   return find_nth_uint32(array, n_elements, (n_elements*3)/4);
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_ORDER_H) */
diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c
index 3ab2797d68..7784f83957 100644
--- a/src/lib/container/smartlist.c
+++ b/src/lib/container/smartlist.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -652,7 +652,7 @@ smartlist_sort_pointers(smartlist_t *sl)
 #define LEFT_CHILD(i)  ( 2*(i) + 1 )
 #define RIGHT_CHILD(i) ( 2*(i) + 2 )
 #define PARENT(i)      ( ((i)-1) / 2 )
-/** }@ */
+/** @} */
 
 /** @{ */
 /** Helper macros for heaps: Given a local variable <b>idx_field_offset</b>
@@ -678,7 +678,7 @@ smartlist_sort_pointers(smartlist_t *sl)
 static inline void
 smartlist_heapify(smartlist_t *sl,
                   int (*compare)(const void *a, const void *b),
-                  int idx_field_offset,
+                  ptrdiff_t idx_field_offset,
                   int idx)
 {
   while (1) {
@@ -725,7 +725,7 @@ smartlist_heapify(smartlist_t *sl,
 void
 smartlist_pqueue_add(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset,
+                     ptrdiff_t idx_field_offset,
                      void *item)
 {
   int idx;
@@ -754,7 +754,7 @@ smartlist_pqueue_add(smartlist_t *sl,
 void *
 smartlist_pqueue_pop(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset)
+                     ptrdiff_t idx_field_offset)
 {
   void *top;
   tor_assert(sl->num_used);
@@ -778,7 +778,7 @@ smartlist_pqueue_pop(smartlist_t *sl,
 void
 smartlist_pqueue_remove(smartlist_t *sl,
                         int (*compare)(const void *a, const void *b),
-                        int idx_field_offset,
+                        ptrdiff_t idx_field_offset,
                         void *item)
 {
   int idx = IDX_OF_ITEM(item);
@@ -802,7 +802,7 @@ smartlist_pqueue_remove(smartlist_t *sl,
 void
 smartlist_pqueue_assert_ok(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset)
+                           ptrdiff_t idx_field_offset)
 {
   int i;
   for (i = sl->num_used - 1; i >= 0; --i) {
diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h
index 77682db03e..458d564cd5 100644
--- a/src/lib/container/smartlist.h
+++ b/src/lib/container/smartlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SMARTLIST_H
@@ -13,6 +13,7 @@
  **/
 
 #include <stdarg.h>
+#include <stddef.h>
 
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -72,18 +73,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
 
 void smartlist_pqueue_add(smartlist_t *sl,
                           int (*compare)(const void *a, const void *b),
-                          int idx_field_offset,
+                          ptrdiff_t idx_field_offset,
                           void *item);
 void *smartlist_pqueue_pop(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset);
+                           ptrdiff_t idx_field_offset);
 void smartlist_pqueue_remove(smartlist_t *sl,
                              int (*compare)(const void *a, const void *b),
-                             int idx_field_offset,
+                             ptrdiff_t idx_field_offset,
                              void *item);
 void smartlist_pqueue_assert_ok(smartlist_t *sl,
                                 int (*compare)(const void *a, const void *b),
-                                int idx_field_offset);
+                                ptrdiff_t idx_field_offset);
 
 char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
                              size_t *len_out) ATTR_MALLOC;
@@ -91,6 +92,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
                               size_t join_len, int terminate, size_t *len_out)
   ATTR_MALLOC;
 
+#ifndef COCCI
 /* Helper: Given two lists of items, possibly of different types, such that
  * both lists are sorted on some common field (as determined by a comparison
  * expression <b>cmpexpr</b>), and such that one list (<b>sl1</b>) has no
@@ -164,5 +166,6 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
 #define SMARTLIST_FOREACH_JOIN_END(var1, var2)  \
   }                                             \
   STMT_END
+#endif /* !defined(COCCI) */
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_H) */
diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include
index a0fa4ec05c..810e777271 100644
--- a/src/lib/crypt_ops/.may_include
+++ b/src/lib/crypt_ops/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 lib/arch/*.h
 lib/cc/*.h
+lib/conf/*.h
 lib/container/*.h
 lib/crypt_ops/*.h
 lib/ctime/*.h
@@ -12,13 +13,16 @@ lib/malloc/*.h
 lib/intmath/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/thread/*.h
 lib/log/*.h
 
+lib/crypt_ops/*.inc
+
 trunnel/pwbox.h
 
 keccak-tiny/*.h
 ed25519/*.h
 
-siphash.h
+ext/siphash.h
diff --git a/src/lib/crypt_ops/aes.h b/src/lib/crypt_ops/aes.h
index 7c774062d9..c25417b4e6 100644
--- a/src/lib/crypt_ops/aes.h
+++ b/src/lib/crypt_ops/aes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Implements a minimal interface to counter-mode AES. */
@@ -16,7 +16,7 @@
 #include "lib/cc/torint.h"
 #include "lib/malloc/malloc.h"
 
-typedef struct aes_cnt_cipher aes_cnt_cipher_t;
+typedef struct aes_cnt_cipher_t aes_cnt_cipher_t;
 
 aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv,
                                  int key_bits);
diff --git a/src/lib/crypt_ops/aes_nss.c b/src/lib/crypt_ops/aes_nss.c
index 4eda5e5902..71d2f01449 100644
--- a/src/lib/crypt_ops/aes_nss.c
+++ b/src/lib/crypt_ops/aes_nss.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,10 +15,10 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/log/util_bug.h"
 
-DISABLE_GCC_WARNING(strict-prototypes)
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
 #include <pk11pub.h>
 #include <secerr.h>
-ENABLE_GCC_WARNING(strict-prototypes)
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
 
 aes_cnt_cipher_t *
 aes_new_cipher(const uint8_t *key, const uint8_t *iv,
diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c
index 2f985d4512..502f7703bd 100644
--- a/src/lib/crypt_ops/aes_openssl.c
+++ b/src/lib/crypt_ops/aes_openssl.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,7 +28,7 @@
 #error "We require OpenSSL >= 1.0.0"
 #endif
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <stdlib.h>
 #include <string.h>
@@ -37,9 +37,8 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/engine.h>
 #include <openssl/modes.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
-#include "lib/crypt_ops/aes.h"
 #include "lib/log/log.h"
 #include "lib/ctime/di_ops.h"
 
@@ -148,13 +147,13 @@ evaluate_ctr_for_aes(void)
 {
   return 0;
 }
-#else /* !(defined(USE_EVP_AES_CTR)) */
+#else /* !defined(USE_EVP_AES_CTR) */
 
 /*======================================================================*/
 /* Interface to AES code, and counter implementation */
 
 /** Implements an AES counter-mode cipher. */
-struct aes_cnt_cipher {
+struct aes_cnt_cipher_t {
 /** This next element (however it's defined) is the AES key. */
   union {
     EVP_CIPHER_CTX evp;
diff --git a/src/lib/crypt_ops/certs.md b/src/lib/crypt_ops/certs.md
new file mode 100644
index 0000000000..f3bd8c2c96
--- /dev/null
+++ b/src/lib/crypt_ops/certs.md
@@ -0,0 +1,29 @@
+
+@page certificates Certificates in Tor
+
+We have, alas, several certificate types in Tor.
+
+The tor_x509_cert_t type represents an X.509 certificate. This document
+won't explain X.509 to you -- possibly, no document can. (OTOH, Peter
+Gutmann's "x.509 style guide", though severely dated, does a good job of
+explaining how awful x.509 can be.)  Do not introduce any new usages of
+X.509. Right now we only use it in places where TLS forces us to do so.
+See x509.c for more information about using this type.
+
+
+The authority_cert_t type is used only for directory authority keys. It
+has a medium-term signing key (which the authorities actually keep
+online) signed by a long-term identity key (which the authority operator
+had really better be keeping offline).  Don't use it for any new kind of
+certificate.
+
+For new places where you need a certificate, consider tor_cert_t: it
+represents a typed and dated _something_ signed by an Ed25519 key.  The
+format is described in tor-spec. Unlike x.509, you can write it on a
+napkin.  The torcert.c file is used for manipulating these certificates and
+their associated keys.
+
+(Additionally, the Tor directory design uses a fairly wide variety of
+documents that include keys and which are signed by keys. You can
+consider these documents to be an additional kind of certificate if you
+want.)
diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h
index 6605d01045..c2e1459078 100644
--- a/src/lib/crypt_ops/compat_openssl.h
+++ b/src/lib/crypt_ops/compat_openssl.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_OPENSSL_H
@@ -41,7 +41,7 @@
    ((st) == SSL3_ST_SW_SRVR_HELLO_B))
 #define OSSL_HANDSHAKE_STATE int
 #define CONST_IF_OPENSSL_1_1_API
-#else /* !(!defined(OPENSSL_1_1_API)) */
+#else /* defined(OPENSSL_1_1_API) */
 #define STATE_IS_SW_SERVER_HELLO(st) \
   ((st) == TLS_ST_SW_SRVR_HELLO)
 #define CONST_IF_OPENSSL_1_1_API const
diff --git a/src/lib/crypt_ops/crypto_cipher.c b/src/lib/crypt_ops/crypto_cipher.c
index 7bc2edad54..0a24a580ae 100644
--- a/src/lib/crypt_ops/crypto_cipher.c
+++ b/src/lib/crypt_ops/crypto_cipher.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h
index cc4fbf7a41..1e22a7c138 100644
--- a/src/lib/crypt_ops/crypto_cipher.h
+++ b/src/lib/crypt_ops/crypto_cipher.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
 /** Length of our symmetric cipher's keys of 256-bit. */
 #define CIPHER256_KEY_LEN 32
 
-typedef struct aes_cnt_cipher crypto_cipher_t;
+typedef struct aes_cnt_cipher_t crypto_cipher_t;
 
 /* environment setup */
 crypto_cipher_t *crypto_cipher_new(const char *key);
@@ -54,4 +54,4 @@ int crypto_cipher_decrypt_with_iv(const char *key,
                                   char *to, size_t tolen,
                                   const char *from, size_t fromlen);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_CIPHER_H) */
diff --git a/src/lib/crypt_ops/crypto_curve25519.c b/src/lib/crypt_ops/crypto_curve25519.c
index de4e17a296..2a2589f07d 100644
--- a/src/lib/crypt_ops/crypto_curve25519.c
+++ b/src/lib/crypt_ops/crypto_curve25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h
index 061a7a3505..154a0b94bc 100644
--- a/src/lib/crypt_ops/crypto_curve25519.h
+++ b/src/lib/crypt_ops/crypto_curve25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -76,8 +76,8 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret);
 
 int curve25519_public_from_base64(curve25519_public_key_t *pkey,
                                   const char *input);
-int curve25519_public_to_base64(char *output,
-                                const curve25519_public_key_t *pkey);
+void curve25519_public_to_base64(char *output,
+                                 const curve25519_public_key_t *pkey);
 
 void curve25519_set_impl_params(int use_ed);
 void curve25519_init(void);
diff --git a/src/lib/crypt_ops/crypto_dh.c b/src/lib/crypt_ops/crypto_dh.c
index 4be7948761..086aceed6f 100644
--- a/src/lib/crypt_ops/crypto_dh.c
+++ b/src/lib/crypt_ops/crypto_dh.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_dh.h b/src/lib/crypt_ops/crypto_dh.h
index 850d50c7ae..2a0e1f6912 100644
--- a/src/lib/crypt_ops/crypto_dh.h
+++ b/src/lib/crypt_ops/crypto_dh.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_dh_nss.c b/src/lib/crypt_ops/crypto_dh_nss.c
index 379eb84a4f..018db8bf43 100644
--- a/src/lib/crypt_ops/crypto_dh_nss.c
+++ b/src/lib/crypt_ops/crypto_dh_nss.c
@@ -1,11 +1,11 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file crypto_dh_nss.h
+ * \file crypto_dh_nss.c
  *
  * \brief NSS implementation of Diffie-Hellman over Z_p.
  **/
diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c
index 8c6388fd5d..c5f7271596 100644
--- a/src/lib/crypt_ops/crypto_dh_openssl.c
+++ b/src/lib/crypt_ops/crypto_dh_openssl.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,11 +17,11 @@
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/dh.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/bn.h>
 #include <string.h>
@@ -34,7 +34,7 @@ static int tor_check_dh_key(int severity, const BIGNUM *bn);
 struct crypto_dh_t {
   DH *dh; /**< The openssl DH object */
 };
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g);
 
@@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     goto out;
   if (!DH_set0_pqg(dh, dh_p, NULL, dh_g))
     goto out;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (!(dh->p = BN_dup(p)))
     goto out;
   if (!(dh->g = BN_dup(g)))
@@ -100,10 +100,10 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     DH_free(dh);
   return ret;
 }
-#endif
+#endif /* 0 */
 
 /**
- * Helper: convert <b>hex<b> to a bignum, and return it.  Assert that the
+ * Helper: convert <b>hex</b> to a bignum, and return it.  Assert that the
  * operation was successful.
  */
 static BIGNUM *
@@ -202,7 +202,7 @@ crypto_dh_new(int dh_type)
     tor_free(res); // sets res to NULL.
   return res;
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 /** Create and return a new openssl DH from a given prime and generator. */
 static DH *
@@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g)
 
   if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS))
     goto err;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   res_dh->p = dh_p;
   res_dh->g = dh_g;
   res_dh->length = DH_PRIVATE_KEY_BITS;
@@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh)
              "the-universe chances really do happen.  Treating as a failure.");
     return -1;
   }
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) {
     /* LCOV_EXCL_START
      * If this happens, then openssl's DH implementation is busted. */
@@ -461,7 +461,7 @@ crypto_dh_free_(crypto_dh_t *dh)
   DH_free(dh->dh);
   tor_free(dh);
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 void
 crypto_dh_free_all_openssl(void)
diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c
index de81b87b7e..7775e69410 100644
--- a/src/lib/crypt_ops/crypto_digest.c
+++ b/src/lib/crypt_ops/crypto_digest.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,171 +23,6 @@
 
 #include "lib/arch/bytes.h"
 
-#ifdef ENABLE_NSS
-DISABLE_GCC_WARNING(strict-prototypes)
-#include <pk11pub.h>
-ENABLE_GCC_WARNING(strict-prototypes)
-#else
-
-#include "lib/crypt_ops/crypto_openssl_mgt.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-
-#include <openssl/hmac.h>
-#include <openssl/sha.h>
-
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
-
-#ifdef ENABLE_NSS
-/**
- * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
- * On failure, return SEC_OID_UNKNOWN. */
-static SECOidTag
-digest_alg_to_nss_oid(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: return SEC_OID_SHA1;
-    case DIGEST_SHA256: return SEC_OID_SHA256;
-    case DIGEST_SHA512: return SEC_OID_SHA512;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return SEC_OID_UNKNOWN;
-  }
-}
-
-/* Helper: get an unkeyed digest via pk11wrap */
-static int
-digest_nss_internal(SECOidTag alg,
-                    char *digest, unsigned len_out,
-                    const char *msg, size_t msg_len)
-{
-  if (alg == SEC_OID_UNKNOWN)
-    return -1;
-  tor_assert(msg_len <= UINT_MAX);
-
-  int rv = -1;
-  SECStatus s;
-  PK11Context *ctx = PK11_CreateDigestContext(alg);
-  if (!ctx)
-    return -1;
-
-  s = PK11_DigestBegin(ctx);
-  if (s != SECSuccess)
-    goto done;
-
-  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-
-  unsigned int len = 0;
-  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
-  if (s != SECSuccess)
-    goto done;
-
-  rv = 0;
- done:
-  PK11_DestroyContext(ctx, PR_TRUE);
-  return rv;
-}
-
-/** True iff alg is implemented in our crypto library, and we want to use that
- * implementation */
-static bool
-library_supports_digest(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return true;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return false;
-  }
-}
-#endif
-
-/* Crypto digest functions */
-
-/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
- * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, -1 on failure.
- */
-MOCK_IMPL(int,
-crypto_digest,(char *digest, const char *m, size_t len))
-{
-  tor_assert(m);
-  tor_assert(digest);
-#ifdef ENABLE_NSS
-  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
-#else
-  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
-    return -1;
-  }
-#endif
-  return 0;
-}
-
-/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest256(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA256) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
-#else
-    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
-#endif
-  } else {
-    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
-/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest512(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA512) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
-#else
-    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-           != NULL);
-#endif
-  } else {
-    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
 /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
  * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
  * success, -1 on failure. */
@@ -267,485 +102,6 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg)
   }
 }
 
-/** Intermediate information about the digest of a stream of data. */
-struct crypto_digest_t {
-  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
-   /** State for the digest we're using.  Only one member of the
-    * union is usable, depending on the value of <b>algorithm</b>. Note also
-    * that space for other members might not even be allocated!
-    */
-  union {
-#ifdef ENABLE_NSS
-    PK11Context *ctx;
-#else
-    SHA_CTX sha1; /**< state for SHA1 */
-    SHA256_CTX sha2; /**< state for SHA256 */
-    SHA512_CTX sha512; /**< state for SHA512 */
-#endif
-    keccak_state sha3; /**< state for SHA3-[256,512] */
-  } d;
-};
-
-#ifdef TOR_UNIT_TESTS
-
-digest_algorithm_t
-crypto_digest_get_algorithm(crypto_digest_t *digest)
-{
-  tor_assert(digest);
-
-  return digest->algorithm;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/**
- * Return the number of bytes we need to malloc in order to get a
- * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
- * when we free one.
- */
-static size_t
-crypto_digest_alloc_bytes(digest_algorithm_t alg)
-{
-  /* Helper: returns the number of bytes in the 'f' field of 'st' */
-#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
-  /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
-                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
-  switch (alg) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.ctx);
-#else
-    case DIGEST_SHA1:
-      return END_OF_FIELD(d.sha1);
-    case DIGEST_SHA256:
-      return END_OF_FIELD(d.sha2);
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.sha512);
-#endif
-    case DIGEST_SHA3_256:
-    case DIGEST_SHA3_512:
-      return END_OF_FIELD(d.sha3);
-    default:
-      tor_assert(0); // LCOV_EXCL_LINE
-      return 0;      // LCOV_EXCL_LINE
-  }
-#undef END_OF_FIELD
-#undef STRUCT_FIELD_SIZE
-}
-
-/**
- * Internal function: create and return a new digest object for 'algorithm'.
- * Does not typecheck the algorithm.
- */
-static crypto_digest_t *
-crypto_digest_new_internal(digest_algorithm_t algorithm)
-{
-  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
-  r->algorithm = algorithm;
-
-  switch (algorithm)
-    {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
-      if (BUG(!r->d.ctx)) {
-        tor_free(r);
-        return NULL;
-      }
-      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
-        crypto_digest_free(r);
-        return NULL;
-      }
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Init(&r->d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Init(&r->d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Init(&r->d.sha512);
-      break;
-#endif
-    case DIGEST_SHA3_256:
-      keccak_digest_init(&r->d.sha3, 256);
-      break;
-    case DIGEST_SHA3_512:
-      keccak_digest_init(&r->d.sha3, 512);
-      break;
-    default:
-      tor_assert_unreached();
-    }
-
-  return r;
-}
-
-/** Allocate and return a new digest object to compute SHA1 digests.
- */
-crypto_digest_t *
-crypto_digest_new(void)
-{
-  return crypto_digest_new_internal(DIGEST_SHA1);
-}
-
-/** Allocate and return a new digest object to compute 256-bit digests
- * using <b>algorithm</b>.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
- * C_RUST_COUPLED: `crypto::digest::Sha256::default`
- */
-crypto_digest_t *
-crypto_digest256_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Allocate and return a new digest object to compute 512-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest512_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Deallocate a digest object.
- */
-void
-crypto_digest_free_(crypto_digest_t *digest)
-{
-  if (!digest)
-    return;
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
-  }
-#endif
-  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memwipe(digest, 0, bytes);
-  tor_free(digest);
-}
-
-/** Add <b>len</b> bytes from <b>data</b> to the digest object.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
- * C_RUST_COUPLED: `crypto::digest::Sha256::process`
- */
-void
-crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                        size_t len)
-{
-  tor_assert(digest);
-  tor_assert(data);
-  /* Using the SHA*_*() calls directly means we don't support doing
-   * SHA in hardware. But so far the delay of getting the question
-   * to the hardware, and hearing the answer, is likely higher than
-   * just doing it ourselves. Hashes are fast.
-   */
-  switch (digest->algorithm) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      tor_assert(len <= UINT_MAX);
-      SECStatus s = PK11_DigestOp(digest->d.ctx,
-                                  (const unsigned char *)data,
-                                  (unsigned int)len);
-      tor_assert(s == SECSuccess);
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Update(&digest->d.sha1, (void*)data, len);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Update(&digest->d.sha2, (void*)data, len);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Update(&digest->d.sha512, (void*)data, len);
-      break;
-#endif
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
-      break;
-    default:
-      /* LCOV_EXCL_START */
-      tor_fragile_assert();
-      break;
-      /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Compute the hash of the data that has been passed to the digest
- * object; write the first out_len bytes of the result to <b>out</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
- * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
- */
-void
-crypto_digest_get_digest(crypto_digest_t *digest,
-                         char *out, size_t out_len)
-{
-  unsigned char r[DIGEST512_LEN];
-  tor_assert(digest);
-  tor_assert(out);
-  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
-
-  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
-   * short output buffers by truncating appropriately. */
-  if (digest->algorithm == DIGEST_SHA3_256 ||
-      digest->algorithm == DIGEST_SHA3_512) {
-    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
-    return;
-  }
-
-#ifdef ENABLE_NSS
-  /* Copy into a temporary buffer since DigestFinal (alters) the context */
-  unsigned char buf[1024];
-  unsigned int saved_len = 0;
-  unsigned rlen;
-  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
-                                               buf, sizeof(buf),
-                                               &saved_len);
-  tor_assert(saved);
-  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
-  tor_assert(s == SECSuccess);
-  tor_assert(rlen >= out_len);
-  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
-  tor_assert(s == SECSuccess);
-  if (saved != buf) {
-    PORT_ZFree(saved, saved_len);
-  }
-#else
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t tmpenv;
-  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
-  memcpy(&tmpenv, digest, alloc_bytes);
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Final(r, &tmpenv.d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Final(r, &tmpenv.d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Final(r, &tmpenv.d.sha512);
-      break;
-//LCOV_EXCL_START
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-    default:
-      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
-      /* This is fatal, because it should never happen. */
-      tor_assert_unreached();
-      break;
-//LCOV_EXCL_STOP
-  }
-#endif
-  memcpy(out, r, out_len);
-  memwipe(r, 0, sizeof(r));
-}
-
-/** Allocate and return a new digest object with the same state as
- * <b>digest</b>
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
- * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
- */
-crypto_digest_t *
-crypto_digest_dup(const crypto_digest_t *digest)
-{
-  tor_assert(digest);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    result->d.ctx = PK11_CloneContext(digest->d.ctx);
-  }
-#endif
-  return result;
-}
-
-/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object.
- */
-void
-crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
-                         const crypto_digest_t *digest)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  tor_assert(bytes <= sizeof(checkpoint->mem));
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    unsigned char *allocated;
-    allocated = PK11_SaveContextAlloc(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      sizeof(checkpoint->mem),
-                                      &checkpoint->bytes_used);
-    /* No allocation is allowed here. */
-    tor_assert(allocated == checkpoint->mem);
-    return;
-  }
-#endif
-  memcpy(checkpoint->mem, digest, bytes);
-}
-
-/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
- * state was previously stored with crypto_digest_checkpoint() */
-void
-crypto_digest_restore(crypto_digest_t *digest,
-                      const crypto_digest_checkpoint_t *checkpoint)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    SECStatus s = PK11_RestoreContext(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      checkpoint->bytes_used);
-    tor_assert(s == SECSuccess);
-    return;
-  }
-#endif
-  memcpy(digest, checkpoint->mem, bytes);
-}
-
-/** Replace the state of the digest object <b>into</b> with the state
- * of the digest object <b>from</b>.  Requires that 'into' and 'from'
- * have the same digest type.
- */
-void
-crypto_digest_assign(crypto_digest_t *into,
-                     const crypto_digest_t *from)
-{
-  tor_assert(into);
-  tor_assert(from);
-  tor_assert(into->algorithm == from->algorithm);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(from->algorithm)) {
-    PK11_DestroyContext(into->d.ctx, PR_TRUE);
-    into->d.ctx = PK11_CloneContext(from->d.ctx);
-    return;
-  }
-#endif
-  memcpy(into,from,alloc_bytes);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of those strings,
- * plus the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist(char *digest_out, size_t len_out,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of: the
- * optional string <b>prepend</b>, those strings,
- * and the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>len_out</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                        const char *prepend,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_t *d = crypto_digest_new_internal(alg);
-  if (prepend)
-    crypto_digest_add_bytes(d, prepend, strlen(prepend));
-  SMARTLIST_FOREACH(lst, const char *, cp,
-                    crypto_digest_add_bytes(d, cp, strlen(cp)));
-  if (append)
-    crypto_digest_add_bytes(d, append, strlen(append));
-  crypto_digest_get_digest(d, digest_out, len_out);
-  crypto_digest_free(d);
-}
-
-/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
- * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
- * result in <b>hmac_out</b>. Asserts on failure.
- */
-void
-crypto_hmac_sha256(char *hmac_out,
-                   const char *key, size_t key_len,
-                   const char *msg, size_t msg_len)
-{
-  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
-  tor_assert(key_len < INT_MAX);
-  tor_assert(msg_len < INT_MAX);
-  tor_assert(hmac_out);
-#ifdef ENABLE_NSS
-  PK11SlotInfo *slot = NULL;
-  PK11SymKey *symKey = NULL;
-  PK11Context *hmac = NULL;
-
-  int ok = 0;
-  SECStatus s;
-  SECItem keyItem, paramItem;
-  keyItem.data = (unsigned char *)key;
-  keyItem.len = (unsigned)key_len;
-  paramItem.type = siBuffer;
-  paramItem.data = NULL;
-  paramItem.len = 0;
-
-  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
-  if (!slot)
-    goto done;
-  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
-                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
-  if (!symKey)
-    goto done;
-
-  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
-                                    &paramItem);
-  if (!hmac)
-    goto done;
-  s = PK11_DigestBegin(hmac);
-  if (s != SECSuccess)
-    goto done;
-  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-  unsigned int len=0;
-  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
-  if (s != SECSuccess || len != DIGEST256_LEN)
-    goto done;
-  ok = 1;
-
- done:
-  if (hmac)
-    PK11_DestroyContext(hmac, PR_TRUE);
-  if (symKey)
-    PK11_FreeSymKey(symKey);
-  if (slot)
-    PK11_FreeSlot(slot);
-
-  tor_assert(ok);
-#else
-  unsigned char *rv = NULL;
-  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
-            (unsigned char*)hmac_out, NULL);
-  tor_assert(rv);
-#endif
-}
-
 /** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
  * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
  * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
@@ -779,7 +135,26 @@ crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
 
 /** Internal state for a eXtendable-Output Function (XOF). */
 struct crypto_xof_t {
+#ifdef OPENSSL_HAS_SHAKE3_EVP
+  /* XXXX We can't enable this yet, because OpenSSL's
+   * DigestFinalXOF function can't be called repeatedly on the same
+   * XOF.
+   *
+   * We could in theory use the undocumented SHA3_absorb and SHA3_squeeze
+   * functions, but let's not mess with undocumented OpenSSL internals any
+   * more than we have to.
+   *
+   * We could also revise our XOF code so that it only allows a single
+   * squeeze operation; we don't require streaming squeeze operations
+   * outside the tests yet.
+   */
+  EVP_MD_CTX *ctx;
+#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */
+  /**
+   * State of the Keccak sponge for the SHAKE-256 computation.
+   **/
   keccak_state s;
+#endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */
 };
 
 /** Allocate a new XOF object backed by SHAKE-256.  The security level
@@ -792,7 +167,14 @@ crypto_xof_new(void)
 {
   crypto_xof_t *xof;
   xof = tor_malloc(sizeof(crypto_xof_t));
+#ifdef OPENSSL_HAS_SHAKE256
+  xof->ctx = EVP_MD_CTX_new();
+  tor_assert(xof->ctx);
+  int r = EVP_DigestInit(xof->ctx, EVP_shake256());
+  tor_assert(r == 1);
+#else /* !defined(OPENSSL_HAS_SHAKE256) */
   keccak_xof_init(&xof->s, 256);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
   return xof;
 }
 
@@ -803,8 +185,13 @@ crypto_xof_new(void)
 void
 crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestUpdate(xof->ctx, data, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_absorb(&xof->s, data, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Squeeze bytes out of a XOF object.  Calling this routine will render
@@ -813,8 +200,13 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 void
 crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestFinalXOF(xof->ctx, out, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_squeeze(&xof->s, out, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Cleanse and deallocate a XOF object. */
@@ -823,6 +215,34 @@ crypto_xof_free_(crypto_xof_t *xof)
 {
   if (!xof)
     return;
+#ifdef OPENSSL_HAS_SHAKE256
+  if (xof->ctx)
+    EVP_MD_CTX_free(xof->ctx);
+#endif
   memwipe(xof, 0, sizeof(crypto_xof_t));
   tor_free(xof);
 }
+
+/** Compute the XOF (SHAKE256) of a <b>input_len</b> bytes at <b>input</b>,
+ * putting <b>output_len</b> bytes at <b>output</b>. */
+void
+crypto_xof(uint8_t *output, size_t output_len,
+           const uint8_t *input, size_t input_len)
+{
+#ifdef OPENSSL_HAS_SHA3
+  EVP_MD_CTX *ctx = EVP_MD_CTX_new();
+  tor_assert(ctx);
+  int r = EVP_DigestInit(ctx, EVP_shake256());
+  tor_assert(r == 1);
+  r = EVP_DigestUpdate(ctx, input, input_len);
+  tor_assert(r == 1);
+  r = EVP_DigestFinalXOF(ctx, output, output_len);
+  tor_assert(r == 1);
+  EVP_MD_CTX_free(ctx);
+#else /* !defined(OPENSSL_HAS_SHA3) */
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, input, input_len);
+  crypto_xof_squeeze_bytes(xof, output, output_len);
+  crypto_xof_free(xof);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+}
diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h
index 47e60ce617..eefd2e3f0a 100644
--- a/src/lib/crypt_ops/crypto_digest.h
+++ b/src/lib/crypt_ops/crypto_digest.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,6 +38,9 @@
 /** Length of hex encoding of SHA512 digest, not including final NUL. */
 #define HEX_DIGEST512_LEN 128
 
+/**
+ * An identifier for a cryptographic digest algorithm.
+ **/
 typedef enum {
   DIGEST_SHA1 = 0,
   DIGEST_SHA256 = 1,
@@ -45,16 +48,31 @@ typedef enum {
   DIGEST_SHA3_256 = 3,
   DIGEST_SHA3_512 = 4,
 } digest_algorithm_t;
+/** Number of digest algorithms that we know */
 #define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
+/** Number of digest algorithms to compute when computing "all the
+ * commonly used digests."
+ *
+ * (This is used in common_digests_t and related functions.)
+ */
 #define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
 
+/**
+ * Bytes of storage needed to record the state of an in-progress SHA-1 digest.
+ *
+ * This is a deliberate overestimate.
+ **/
 #define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512)
+
 /** Structure used to temporarily save the a digest object. Only implemented
  * for SHA1 digest for now. */
 typedef struct crypto_digest_checkpoint_t {
 #ifdef ENABLE_NSS
+  /** The number of bytes used in <b>mem</b>. */
   unsigned int bytes_used;
 #endif
+  /** A buffer to store the SHA1 state. Its contents are unspecified, and
+   * are managed by the underlying crypto library.*/
   uint8_t mem[DIGEST_CHECKPOINT_BYTES];
 } crypto_digest_checkpoint_t;
 
@@ -67,10 +85,19 @@ typedef struct crypto_digest_checkpoint_t {
  * once.
  **/
 typedef struct {
+  /** An array of digest outputs, one for each "common" digest algorithm. */
   char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
 } common_digests_t;
 
+/**
+ * State for computing a digest over a stream of data.
+ **/
 typedef struct crypto_digest_t crypto_digest_t;
+
+/**
+ * State for computing an "extendable-output function" (like SHAKE) over a
+ * stream of data, and/or streaming the output.
+ **/
 typedef struct crypto_xof_t crypto_xof_t;
 
 struct smartlist_t;
@@ -97,6 +124,9 @@ crypto_digest_t *crypto_digest_new(void);
 crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
 crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
 void crypto_digest_free_(crypto_digest_t *digest);
+/**
+ * Release all storage held in <b>d</b>, and set it to NULL.
+ **/
 #define crypto_digest_free(d) \
   FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d))
 void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
@@ -122,8 +152,13 @@ crypto_xof_t *crypto_xof_new(void);
 void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
 void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
 void crypto_xof_free_(crypto_xof_t *xof);
+/**
+ * Release all storage held in <b>xof</b>, and set it to NULL.
+ **/
 #define crypto_xof_free(xof) \
   FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
+void crypto_xof(uint8_t *output, size_t output_len,
+                const uint8_t *input, size_t input_len);
 
 #ifdef TOR_UNIT_TESTS
 digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
diff --git a/src/lib/crypt_ops/crypto_digest_nss.c b/src/lib/crypt_ops/crypto_digest_nss.c
new file mode 100644
index 0000000000..92c20fe9e8
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_nss.c
@@ -0,0 +1,563 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_nss.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (NSS specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
+#include <pk11pub.h>
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
+
+/**
+ * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
+ * On failure, return SEC_OID_UNKNOWN. */
+static SECOidTag
+digest_alg_to_nss_oid(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: return SEC_OID_SHA1;
+    case DIGEST_SHA256: return SEC_OID_SHA256;
+    case DIGEST_SHA512: return SEC_OID_SHA512;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return SEC_OID_UNKNOWN;
+  }
+}
+
+/** Helper: Compute an unkeyed digest of the <b>msg_len</b> bytes at
+ * <b>msg</b>, using the digest algorithm specified by <b>alg</b>.
+ * Store the result in the <b>len_out</b>-byte buffer at <b>digest</b>.
+ * Return the number of bytes written on success, and -1 on failure.
+ **/
+static int
+digest_nss_internal(SECOidTag alg,
+                    char *digest, unsigned len_out,
+                    const char *msg, size_t msg_len)
+{
+  if (alg == SEC_OID_UNKNOWN)
+    return -1;
+  tor_assert(msg_len <= UINT_MAX);
+
+  int rv = -1;
+  SECStatus s;
+  PK11Context *ctx = PK11_CreateDigestContext(alg);
+  if (!ctx)
+    return -1;
+
+  s = PK11_DigestBegin(ctx);
+  if (s != SECSuccess)
+    goto done;
+
+  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+
+  unsigned int len = 0;
+  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
+  if (s != SECSuccess)
+    goto done;
+
+  rv = 0;
+ done:
+  PK11_DestroyContext(ctx, PR_TRUE);
+  return rv;
+}
+
+/** True iff alg is implemented in our crypto library, and we want to use that
+ * implementation */
+static bool
+library_supports_digest(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return true;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return false;
+  }
+}
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
+  } else {
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
+  } else {
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    PK11Context *ctx;
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.ctx);
+    case DIGEST_SHA3_256:
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
+      if (BUG(!r->d.ctx)) {
+        tor_free(r);
+        return NULL;
+      }
+      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+  if (library_supports_digest(digest->algorithm)) {
+    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
+  }
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      tor_assert(len <= UINT_MAX);
+      SECStatus s = PK11_DigestOp(digest->d.ctx,
+                                  (const unsigned char *)data,
+                                  (unsigned int)len);
+      tor_assert(s == SECSuccess);
+      break;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+  }
+
+  /* Copy into a temporary buffer since DigestFinal (alters) the context */
+  unsigned char buf[1024];
+  unsigned int saved_len = 0;
+  unsigned rlen;
+  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
+                                               buf, sizeof(buf),
+                                               &saved_len);
+  tor_assert(saved);
+  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
+  tor_assert(s == SECSuccess);
+  tor_assert(rlen >= out_len);
+  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
+  tor_assert(s == SECSuccess);
+
+  if (saved != buf) {
+    PORT_ZFree(saved, saved_len);
+  }
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+  if (library_supports_digest(digest->algorithm)) {
+    result->d.ctx = PK11_CloneContext(digest->d.ctx);
+  }
+
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  if (library_supports_digest(digest->algorithm)) {
+    unsigned char *allocated;
+    allocated = PK11_SaveContextAlloc(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      sizeof(checkpoint->mem),
+                                      &checkpoint->bytes_used);
+    /* No allocation is allowed here. */
+    tor_assert(allocated == checkpoint->mem);
+    return;
+  }
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  if (library_supports_digest(digest->algorithm)) {
+    SECStatus s = PK11_RestoreContext(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      checkpoint->bytes_used);
+    tor_assert(s == SECSuccess);
+    return;
+  }
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+  if (library_supports_digest(from->algorithm)) {
+    PK11_DestroyContext(into->d.ctx, PR_TRUE);
+    into->d.ctx = PK11_CloneContext(from->d.ctx);
+    return;
+  }
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+
+  PK11SlotInfo *slot = NULL;
+  PK11SymKey *symKey = NULL;
+  PK11Context *hmac = NULL;
+
+  int ok = 0;
+  SECStatus s;
+  SECItem keyItem, paramItem;
+  keyItem.data = (unsigned char *)key;
+  keyItem.len = (unsigned)key_len;
+  paramItem.type = siBuffer;
+  paramItem.data = NULL;
+  paramItem.len = 0;
+
+  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
+  if (!slot)
+    goto done;
+  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
+                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
+  if (!symKey)
+    goto done;
+
+  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
+                                    &paramItem);
+  if (!hmac)
+    goto done;
+  s = PK11_DigestBegin(hmac);
+  if (s != SECSuccess)
+    goto done;
+  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+  unsigned int len=0;
+  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
+  if (s != SECSuccess || len != DIGEST256_LEN)
+    goto done;
+  ok = 1;
+
+ done:
+  if (hmac)
+    PK11_DestroyContext(hmac, PR_TRUE);
+  if (symKey)
+    PK11_FreeSymKey(symKey);
+  if (slot)
+    PK11_FreeSlot(slot);
+
+  tor_assert(ok);
+}
diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c
new file mode 100644
index 0000000000..11189c7fb2
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_openssl.c
@@ -0,0 +1,521 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_openssl.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (OpenSSL specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+
+DISABLE_GCC_WARNING("-Wredundant-decls")
+
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+ENABLE_GCC_WARNING("-Wredundant-decls")
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
+    return -1;
+  }
+  return 0;
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST256_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL);
+#else
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
+           != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST512_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL);
+#else
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    SHA_CTX sha1; /**< state for SHA1 */
+    SHA256_CTX sha2; /**< state for SHA256 */
+    SHA512_CTX sha512; /**< state for SHA512 */
+#ifdef OPENSSL_HAS_SHA3
+    EVP_MD_CTX *md;
+#else
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+#endif
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /** Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /** Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1:
+      return END_OF_FIELD(d.sha1);
+    case DIGEST_SHA256:
+      return END_OF_FIELD(d.sha2);
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.sha512);
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.md);
+#else
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1:
+      SHA1_Init(&r->d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Init(&r->d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Init(&r->d.sha512);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_512:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    if (digest->d.md) {
+      EVP_MD_CTX_free(digest->d.md);
+    }
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Update(&digest->d.sha1, (void*)data, len);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Update(&digest->d.sha2, (void*)data, len);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Update(&digest->d.sha512, (void*)data, len);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: {
+      int r = EVP_DigestUpdate(digest->d.md, data, len);
+      tor_assert(r);
+  }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned dlen = (unsigned)
+      crypto_digest_algorithm_get_length(digest->algorithm);
+    EVP_MD_CTX *tmp = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(tmp, digest->d.md);
+    memset(r, 0xff, sizeof(r));
+    int res = EVP_DigestFinal(tmp, r, &dlen);
+    EVP_MD_CTX_free(tmp);
+    tor_assert(res == 1);
+    goto done;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    /* Tiny-Keccak handles copying into a temporary ctx, and also can handle
+     * short output buffers by truncating appropriately. */
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t tmpenv;
+  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
+  memcpy(&tmpenv, digest, alloc_bytes);
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Final(r, &tmpenv.d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Final(r, &tmpenv.d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Final(r, &tmpenv.d.sha512);
+      break;
+//LCOV_EXCL_START
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+    default:
+      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
+      /* This is fatal, because it should never happen. */
+      tor_assert_unreached();
+      break;
+//LCOV_EXCL_STOP
+  }
+#ifdef OPENSSL_HAS_SHA3
+ done:
+#endif
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    result->d.md = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(result->d.md, digest->d.md);
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (from->algorithm == DIGEST_SHA3_256 ||
+      from->algorithm == DIGEST_SHA3_512) {
+    EVP_MD_CTX_copy(into->d.md, from->d.md);
+    return;
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+  unsigned char *rv = NULL;
+  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
+            (unsigned char*)hmac_out, NULL);
+  tor_assert(rv);
+}
diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c
index 0a442bb739..f242c7011e 100644
--- a/src/lib/crypt_ops/crypto_ed25519.c
+++ b/src/lib/crypt_ops/crypto_ed25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -226,7 +226,7 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong)
 int
 ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey)
 {
-  return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
+  return safe_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
 }
 
 /* Return a heap-allocated array that contains <b>msg</b> prefixed by the
diff --git a/src/lib/crypt_ops/crypto_ed25519.h b/src/lib/crypt_ops/crypto_ed25519.h
index 325b28244d..346de464e3 100644
--- a/src/lib/crypt_ops/crypto_ed25519.h
+++ b/src/lib/crypt_ops/crypto_ed25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c
index 84f73e5272..92b8b9372e 100644
--- a/src/lib/crypt_ops/crypto_format.c
+++ b/src/lib/crypt_ops/crypto_format.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -104,7 +104,7 @@ crypto_read_tagged_contents_from_file(const char *fname,
   prefix[32] = 0;
   /* Check type, extract tag. */
   if (strcmpstart(prefix, "== ") || strcmpend(prefix, " ==") ||
-      ! tor_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
+      ! fast_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
     saved_errno = EINVAL;
     goto end;
   }
@@ -131,20 +131,27 @@ crypto_read_tagged_contents_from_file(const char *fname,
   return r;
 }
 
-/** Encode <b>pkey</b> as a base64-encoded string, without trailing "="
+/** Encode <b>pkey</b> as a base64-encoded string, including trailing "="
  * characters, in the buffer <b>output</b>, which must have at least
- * CURVE25519_BASE64_PADDED_LEN+1 bytes available.  Return 0 on success, -1 on
- * failure. */
-int
+ * CURVE25519_BASE64_PADDED_LEN+1 bytes available.
+ * Can not fail.
+ *
+ * Careful! CURVE25519_BASE64_PADDED_LEN is one byte longer than
+ * ED25519_BASE64_LEN.
+ */
+void
 curve25519_public_to_base64(char *output,
                             const curve25519_public_key_t *pkey)
 {
   char buf[128];
-  base64_encode(buf, sizeof(buf),
-                (const char*)pkey->public_key, CURVE25519_PUBKEY_LEN, 0);
-  buf[CURVE25519_BASE64_PADDED_LEN] = '\0';
+  int n = base64_encode(buf, sizeof(buf),
+                        (const char*)pkey->public_key,
+                        CURVE25519_PUBKEY_LEN, 0);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode(). */
+  tor_assert(n == CURVE25519_BASE64_PADDED_LEN);
+  tor_assert(buf[CURVE25519_BASE64_PADDED_LEN] == '\0');
   memcpy(output, buf, CURVE25519_BASE64_PADDED_LEN+1);
-  return 0;
 }
 
 /** Try to decode a base64-encoded curve25519 public key from <b>input</b>
@@ -181,8 +188,7 @@ ed25519_fmt(const ed25519_public_key_t *pkey)
     if (ed25519_public_key_is_zero(pkey)) {
       strlcpy(formatted, "<unset>", sizeof(formatted));
     } else {
-      int r = ed25519_public_to_base64(formatted, pkey);
-      tor_assert(!r);
+      ed25519_public_to_base64(formatted, pkey);
     }
   } else {
     strlcpy(formatted, "<null>", sizeof(formatted));
@@ -202,28 +208,35 @@ ed25519_public_from_base64(ed25519_public_key_t *pkey,
 
 /** Encode the public key <b>pkey</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_BASE64_LEN bytes of encoded key,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
+ *
+ * Careful! ED25519_BASE64_LEN is one byte shorter than
+ * CURVE25519_BASE64_PADDED_LEN.
  */
-int
+void
 ed25519_public_to_base64(char *output,
                          const ed25519_public_key_t *pkey)
 {
-  return digest256_to_base64(output, (const char *)pkey->pubkey);
+  digest256_to_base64(output, (const char *)pkey->pubkey);
 }
 
 /** Encode the signature <b>sig</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_SIG_BASE64_LEN bytes of encoded signature,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
  */
-int
+void
 ed25519_signature_to_base64(char *output,
                             const ed25519_signature_t *sig)
 {
   char buf[256];
   int n = base64_encode_nopad(buf, sizeof(buf), sig->sig, ED25519_SIG_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
   tor_assert(n == ED25519_SIG_BASE64_LEN);
+  tor_assert(buf[ED25519_SIG_BASE64_LEN] == '\0');
   memcpy(output, buf, ED25519_SIG_BASE64_LEN+1);
-  return 0;
 }
 
 /** Try to decode the string <b>input</b> into an ed25519 signature. On
@@ -233,16 +246,11 @@ int
 ed25519_signature_from_base64(ed25519_signature_t *sig,
                               const char *input)
 {
-
   if (strlen(input) != ED25519_SIG_BASE64_LEN)
     return -1;
-  char buf[ED25519_SIG_BASE64_LEN+3];
-  memcpy(buf, input, ED25519_SIG_BASE64_LEN);
-  buf[ED25519_SIG_BASE64_LEN+0] = '=';
-  buf[ED25519_SIG_BASE64_LEN+1] = '=';
-  buf[ED25519_SIG_BASE64_LEN+2] = 0;
   char decoded[128];
-  int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf));
+  int n = base64_decode(decoded, sizeof(decoded), input,
+                        ED25519_SIG_BASE64_LEN);
   if (n < 0 || n != ED25519_SIG_LEN)
     return -1;
   memcpy(sig->sig, decoded, ED25519_SIG_LEN);
@@ -250,24 +258,26 @@ ed25519_signature_from_base64(ed25519_signature_t *sig,
   return 0;
 }
 
-/** Base64 encode DIGEST_LINE bytes from <b>digest</b>, remove the trailing =
+/** Base64 encode DIGEST_LEN bytes from <b>digest</b>, remove the trailing =
  * characters, and store the nul-terminated result in the first
- * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.  */
-/* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST_LEN, 0);
-  buf[BASE64_DIGEST_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST_LEN);
+  tor_assert(buf[BASE64_DIGEST_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest_from_base64(char *digest, const char *d64)
 {
@@ -279,22 +289,24 @@ digest_from_base64(char *digest, const char *d64)
 
 /** Base64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the
  * trailing = characters, and store the nul-terminated result in the first
- * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */
- /* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest256_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN, 0);
-  buf[BASE64_DIGEST256_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST256_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST256_LEN);
+  tor_assert(buf[BASE64_DIGEST256_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST256_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST256_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest256_from_base64(char *digest, const char *d64)
 {
diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h
index fe852e6a61..91da137e1c 100644
--- a/src/lib/crypt_ops/crypto_format.h
+++ b/src/lib/crypt_ops/crypto_format.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,18 +33,18 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname,
 
 int ed25519_public_from_base64(struct ed25519_public_key_t *pkey,
                                const char *input);
-int ed25519_public_to_base64(char *output,
-                             const struct ed25519_public_key_t *pkey);
+void ed25519_public_to_base64(char *output,
+                              const struct ed25519_public_key_t *pkey);
 const char *ed25519_fmt(const struct ed25519_public_key_t *pkey);
 
 int ed25519_signature_from_base64(struct ed25519_signature_t *sig,
                                   const char *input);
-int ed25519_signature_to_base64(char *output,
-                                const struct ed25519_signature_t *sig);
+void ed25519_signature_to_base64(char *output,
+                                 const struct ed25519_signature_t *sig);
 
-int digest_to_base64(char *d64, const char *digest);
+void digest_to_base64(char *d64, const char *digest);
 int digest_from_base64(char *digest, const char *d64);
-int digest256_to_base64(char *d64, const char *digest);
+void digest256_to_base64(char *d64, const char *digest);
 int digest256_from_base64(char *digest, const char *d64);
 
 #endif /* !defined(TOR_CRYPTO_FORMAT_H) */
diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c
index fd2e701651..7b02820087 100644
--- a/src/lib/crypt_ops/crypto_hkdf.c
+++ b/src/lib/crypt_ops/crypto_hkdf.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
 #include <openssl/kdf.h>
 #define HAVE_OPENSSL_HKDF 1
 #endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <string.h>
 
@@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl(
   return 0;
 }
 
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
 
 /**
  * Perform RFC5869 HKDF computation using our own legacy implementation.
@@ -166,7 +166,7 @@ crypto_expand_key_material_rfc5869_sha256_legacy(
   memwipe(mac, 0, sizeof(mac));
   return 0;
 }
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 
 /** Expand some secret key material according to RFC5869, using SHA256 as the
  * underlying hash.  The <b>key_in_len</b> bytes at <b>key_in</b> are the
@@ -191,11 +191,11 @@ crypto_expand_key_material_rfc5869_sha256(
                                              salt_in_len, info_in,
                                              info_in_len,
                                              key_out, key_out_len);
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
   return crypto_expand_key_material_rfc5869_sha256_legacy(key_in,
                                                key_in_len, salt_in,
                                                salt_in_len, info_in,
                                                info_in_len,
                                                key_out, key_out_len);
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 }
diff --git a/src/lib/crypt_ops/crypto_hkdf.h b/src/lib/crypt_ops/crypto_hkdf.h
index 2994d18e3d..404f548774 100644
--- a/src/lib/crypt_ops/crypto_hkdf.h
+++ b/src/lib/crypt_ops/crypto_hkdf.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c
index 329c264af6..a836bd8645 100644
--- a/src/lib/crypt_ops/crypto_init.c
+++ b/src/lib/crypt_ops/crypto_init.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,8 @@
 
 #include "orconfig.h"
 
+#define CRYPTO_PRIVATE
+
 #include "lib/crypt_ops/crypto_init.h"
 
 #include "lib/crypt_ops/crypto_curve25519.h"
@@ -20,8 +22,14 @@
 #include "lib/crypt_ops/crypto_openssl_mgt.h"
 #include "lib/crypt_ops/crypto_nss_mgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_sys.h"
+#include "lib/crypt_ops/crypto_options_st.h"
+#include "lib/conf/conftypes.h"
+#include "lib/log/util_bug.h"
+
+#include "lib/subsys/subsys.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** Boolean: has our crypto library been initialized? (early phase) */
 static int crypto_early_initialized_ = 0;
@@ -66,6 +74,8 @@ crypto_early_init(void)
     if (crypto_init_siphash_key() < 0)
       return -1;
 
+    crypto_rand_fast_init();
+
     curve25519_init();
     ed25519_init();
   }
@@ -92,7 +102,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
     (void)useAccel;
     (void)accelName;
     (void)accelDir;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
     if (crypto_nss_late_init() < 0)
       return -1;
@@ -108,6 +118,7 @@ crypto_thread_cleanup(void)
 #ifdef ENABLE_OPENSSL
   crypto_openssl_thread_cleanup();
 #endif
+  destroy_thread_fast_rng();
 }
 
 /**
@@ -126,6 +137,8 @@ crypto_global_cleanup(void)
   crypto_nss_global_cleanup();
 #endif
 
+  crypto_rand_fast_shutdown();
+
   crypto_early_initialized_ = 0;
   crypto_global_initialized_ = 0;
   have_seeded_siphash = 0;
@@ -142,6 +155,12 @@ crypto_prefork(void)
 #ifdef ENABLE_NSS
   crypto_nss_prefork();
 #endif
+  /* It is not safe to share a fast_rng object across a fork boundary unless
+   * we actually have zero-on-fork support in map_anon.c.  If we have
+   * drop-on-fork support, we will crash; if we have neither, we will yield
+   * a copy of the parent process's rng, which is scary and insecure.
+   */
+  destroy_thread_fast_rng();
 }
 
 /** Run operations that the crypto library requires to be happy again
@@ -202,3 +221,111 @@ tor_is_using_nss(void)
   return 0;
 #endif
 }
+
+static int
+subsys_crypto_initialize(void)
+{
+  if (crypto_early_init() < 0)
+    return -1;
+  crypto_dh_init();
+  return 0;
+}
+
+static void
+subsys_crypto_shutdown(void)
+{
+  crypto_global_cleanup();
+}
+
+static void
+subsys_crypto_prefork(void)
+{
+  crypto_prefork();
+}
+
+static void
+subsys_crypto_postfork(void)
+{
+  crypto_postfork();
+}
+
+static void
+subsys_crypto_thread_cleanup(void)
+{
+  crypto_thread_cleanup();
+}
+
+/** Magic number for crypto_options_t. */
+#define CRYPTO_OPTIONS_MAGIC 0x68757368
+
+/**
+ * Return 0 if <b>arg</b> is a valid crypto_options_t.  Otherwise return -1
+ * and set *<b>msg_out</b> to a freshly allocated error string.
+ **/
+static int
+crypto_options_validate(const void *arg, char **msg_out)
+{
+  const crypto_options_t *opt = arg;
+  tor_assert(opt->magic == CRYPTO_OPTIONS_MAGIC);
+  tor_assert(msg_out);
+
+  if (opt->AccelDir && !opt->AccelName) {
+    *msg_out = tor_strdup("Can't use hardware crypto accelerator dir "
+                          "without engine name.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/* Declare the options field table for crypto_options */
+#define CONF_CONTEXT LL_TABLE
+#include "lib/crypt_ops/crypto_options.inc"
+#undef CONF_CONTEXT
+
+/**
+ * Declares the configuration options for this module.
+ **/
+static const config_format_t crypto_options_fmt = {
+  .size = sizeof(crypto_options_t),
+  .magic = { "crypto_options_t",
+             CRYPTO_OPTIONS_MAGIC,
+             offsetof(crypto_options_t, magic) },
+  .vars = crypto_options_t_vars,
+  .validate_fn = crypto_options_validate,
+};
+
+/**
+ * Invoked from subsysmgr.c when a new set of options arrives.
+ **/
+static int
+crypto_set_options(void *arg)
+{
+  const crypto_options_t *options = arg;
+  const bool hardware_accel = options->HardwareAccel || options->AccelName;
+
+  // This call already checks for crypto_global_initialized_, so it
+  // will only initialize the subsystem the first time it's called.
+  if (crypto_global_init(hardware_accel,
+                         options->AccelName,
+                         options->AccelDir)) {
+    log_err(LD_BUG, "Unable to initialize the crypto subsystem. Exiting.");
+    return -1;
+  }
+  return 0;
+}
+
+const struct subsys_fns_t sys_crypto = {
+  .name = "crypto",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -60,
+  .initialize = subsys_crypto_initialize,
+  .shutdown = subsys_crypto_shutdown,
+  .prefork = subsys_crypto_prefork,
+  .postfork = subsys_crypto_postfork,
+  .thread_cleanup = subsys_crypto_thread_cleanup,
+
+  .options_format = &crypto_options_fmt,
+  .set_options = crypto_set_options,
+};
diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h
index 540d08eb56..b11e2e34bf 100644
--- a/src/lib/crypt_ops/crypto_init.h
+++ b/src/lib/crypt_ops/crypto_init.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,4 +33,4 @@ const char *crypto_get_header_version_string(void);
 
 int tor_is_using_nss(void);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_INIT_H) */
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.c b/src/lib/crypt_ops/crypto_nss_mgt.c
index 0179126e38..d82e51249c 100644
--- a/src/lib/crypt_ops/crypto_nss_mgt.c
+++ b/src/lib/crypt_ops/crypto_nss_mgt.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/string/printf.h"
 
-DISABLE_GCC_WARNING(strict-prototypes)
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
 #include <nss.h>
 #include <pk11func.h>
 #include <ssl.h>
@@ -24,7 +24,7 @@ DISABLE_GCC_WARNING(strict-prototypes)
 #include <prerror.h>
 #include <prtypes.h>
 #include <prinit.h>
-ENABLE_GCC_WARNING(strict-prototypes)
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
 
 const char *
 crypto_nss_get_version_str(void)
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h
index 72fd2a1229..8686b1b8aa 100644
--- a/src/lib/crypt_ops/crypto_nss_mgt.h
+++ b/src/lib/crypt_ops/crypto_nss_mgt.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,6 @@ void crypto_nss_global_cleanup(void);
 
 void crypto_nss_prefork(void);
 void crypto_nss_postfork(void);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
-#endif /* !defined(TOR_CRYPTO_NSS_H) */
+#endif /* !defined(TOR_CRYPTO_NSS_MGT_H) */
diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c
index 2186d2a939..4cacb3dd98 100644
--- a/src/lib/crypt_ops/crypto_ope.c
+++ b/src/lib/crypt_ops/crypto_ope.c
@@ -1,8 +1,9 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * A rudimentary order-preserving encryption scheme.
+ * @file crypto_ope.c
+ * @brief A rudimentary order-preserving encryption scheme.
  *
  * To compute the encryption of N, this scheme uses an AES-CTR stream to
  * generate M-byte values, and adds the first N of them together. (+1 each to
@@ -57,9 +58,9 @@ ope_val_from_le(ope_val_t x)
     ((x) >> 8) |
     (((x)&0xff) << 8);
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
 #define ope_val_from_le(x) (x)
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
 /**
  * Return a new AES256-CTR stream cipher object for <b>ope</b>, ready to yield
@@ -143,7 +144,7 @@ crypto_ope_new(const uint8_t *key)
   return ope;
 }
 
-/** Free all storage held in <>ope</b>. */
+/** Free all storage held in <b>ope</b>. */
 void
 crypto_ope_free_(crypto_ope_t *ope)
 {
diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h
index 610d956335..7498ea6a2e 100644
--- a/src/lib/crypt_ops/crypto_ope.h
+++ b/src/lib/crypt_ops/crypto_ope.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file crypto_ope.h
+ * @brief header for crypto_ope.c
+ **/
+
 #ifndef CRYPTO_OPE_H
 #define CRYPTO_OPE_H
 
@@ -37,10 +42,10 @@ void crypto_ope_free_(crypto_ope_t *ope);
 uint64_t crypto_ope_encrypt(const crypto_ope_t *ope, int plaintext);
 
 #ifdef CRYPTO_OPE_PRIVATE
-struct aes_cnt_cipher;
-STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope,
+struct aes_cnt_cipher_t;
+STATIC struct aes_cnt_cipher_t *ope_get_cipher(const crypto_ope_t *ope,
                                               uint32_t initial_idx);
-STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n);
-#endif
+STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher_t *c, size_t n);
+#endif /* defined(CRYPTO_OPE_PRIVATE) */
 
-#endif
+#endif /* !defined(CRYPTO_OPE_H) */
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c
index e7d6084f6c..065cbca1cc 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.c
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,7 +21,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "lib/thread/threads.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/err.h>
 #include <openssl/rsa.h>
@@ -36,7 +36,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/crypto.h>
 #include <openssl/ssl.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <string.h>
 
@@ -130,10 +130,12 @@ crypto_openssl_get_header_version_str(void)
   return crypto_openssl_header_version_str;
 }
 
+#ifndef COCCI
 #ifndef OPENSSL_THREADS
-#error OpenSSL has been built without thread support. Tor requires an \
- OpenSSL library with thread support enabled.
+#error "OpenSSL has been built without thread support. Tor requires an \
+ OpenSSL library with thread support enabled."
 #endif
+#endif /* !defined(COCCI) */
 
 #ifndef NEW_THREAD_API
 /** Helper: OpenSSL uses this callback to manipulate mutexes. */
@@ -185,6 +187,10 @@ crypto_openssl_free_all(void)
   tor_free(crypto_openssl_version_str);
   tor_free(crypto_openssl_header_version_str);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
 #ifndef NEW_THREAD_API
   if (n_openssl_mutexes_) {
     int n = n_openssl_mutexes_;
@@ -209,10 +215,10 @@ crypto_openssl_early_init(void)
                      OPENSSL_INIT_LOAD_CRYPTO_STRINGS |
                      OPENSSL_INIT_ADD_ALL_CIPHERS |
                      OPENSSL_INIT_ADD_ALL_DIGESTS, NULL);
-#else
+#else /* !defined(OPENSSL_1_1_API) */
     ERR_load_crypto_strings();
     OpenSSL_add_all_algorithms();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
     setup_openssl_threading();
 
@@ -278,8 +284,14 @@ log_engine(const char *fn, ENGINE *e)
 }
 #endif /* !defined(DISABLE_ENGINES) */
 
-/** Initialize engines for openssl (if enabled). */
-static void
+/** Initialize engines for openssl (if enabled).  Load all the built-in
+ * engines, along with the one called <b>accelName</b> (which may be NULL).
+ * If <b>accelName</b> is prefixed with "!", then it is required: return -1
+ * if it can't be loaded.  Otherwise return 0.
+ *
+ * If <b>accelDir</b> is not NULL, it is the path from which the engine should
+ * be loaded. */
+static int
 crypto_openssl_init_engines(const char *accelName,
                             const char *accelDir)
 {
@@ -287,7 +299,13 @@ crypto_openssl_init_engines(const char *accelName,
   (void)accelName;
   (void)accelDir;
   log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled.");
-#else
+  if (accelName && accelName[0] == '!') {
+    log_warn(LD_CRYPTO, "Unable to load required dynamic OpenSSL engine "
+             "\"%s\".", accelName+1);
+    return -1;
+  }
+  return 0;
+#else /* !defined(DISABLE_ENGINES) */
   ENGINE *e = NULL;
 
   log_info(LD_CRYPTO, "Initializing OpenSSL engine support.");
@@ -295,6 +313,9 @@ crypto_openssl_init_engines(const char *accelName,
   ENGINE_register_all_complete();
 
   if (accelName) {
+    const bool required = accelName[0] == '!';
+    if (required)
+      ++accelName;
     if (accelDir) {
       log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\""
                " via path \"%s\".", accelName, accelDir);
@@ -305,8 +326,11 @@ crypto_openssl_init_engines(const char *accelName,
       e = ENGINE_by_id(accelName);
     }
     if (!e) {
-      log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".",
+      log_warn(LD_CRYPTO, "Unable to load %sdynamic OpenSSL engine \"%s\".",
+               required?"required ":"",
                accelName);
+      if (required)
+        return -1;
     } else {
       log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".",
                accelName);
@@ -343,6 +367,7 @@ crypto_openssl_init_engines(const char *accelName,
 #ifdef NID_aes_256_gcm
   log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm));
 #endif
+  return 0;
 
 #endif /* defined(DISABLE_ENGINES) */
 }
@@ -353,7 +378,8 @@ crypto_openssl_late_init(int useAccel, const char *accelName,
                          const char *accelDir)
 {
   if (useAccel > 0) {
-    crypto_openssl_init_engines(accelName, accelDir);
+    if (crypto_openssl_init_engines(accelName, accelDir) < 0)
+      return -1;
   } else {
     log_info(LD_CRYPTO, "NOT using OpenSSL engine support.");
   }
@@ -382,7 +408,7 @@ crypto_openssl_thread_cleanup(void)
 void
 crypto_openssl_global_cleanup(void)
 {
-  #ifndef OPENSSL_1_1_API
+#ifndef OPENSSL_1_1_API
   EVP_cleanup();
 #endif
 #ifndef NEW_THREAD_API
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h
index 8dbadfc9d2..c67ab6467c 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.h
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -83,6 +83,6 @@ int crypto_openssl_late_init(int useAccel, const char *accelName,
 void crypto_openssl_thread_cleanup(void);
 void crypto_openssl_global_cleanup(void);
 
-#endif /* ENABLE_OPENSSL */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #endif /* !defined(TOR_CRYPTO_OPENSSL_H) */
diff --git a/src/lib/crypt_ops/crypto_options.inc b/src/lib/crypt_ops/crypto_options.inc
new file mode 100644
index 0000000000..5bee0daacd
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_options.inc
@@ -0,0 +1,19 @@
+
+/**
+ * @file crypto_options.inc
+ * @brief Declare configuration options for the crypto_ops module.
+ **/
+
+/** Holds configuration about our cryptography options. */
+BEGIN_CONF_STRUCT(crypto_options_t)
+
+/** Should we enable extra OpenSSL hardware acceleration (where available)?  */
+CONF_VAR(HardwareAccel, BOOL, CFLG_IMMUTABLE, "0")
+
+/** Optional OpenSSL hardware-acceleration engine name */
+CONF_VAR(AccelName, STRING, CFLG_IMMUTABLE, NULL)
+
+/** Optional OpenSSL hardware-acceleration engine search directory. */
+CONF_VAR(AccelDir, FILENAME, CFLG_IMMUTABLE, NULL)
+
+END_CONF_STRUCT(crypto_options_t)
diff --git a/src/lib/crypt_ops/crypto_options_st.h b/src/lib/crypt_ops/crypto_options_st.h
new file mode 100644
index 0000000000..a453c451fe
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_options_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file crypto_options_st.h
+ * @brief Header for lib/crypt_ops/crypto_options_st.c
+ **/
+
+#ifndef TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H
+#define TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H
+
+#include "lib/conf/confdecl.h"
+
+#define CONF_CONTEXT STRUCT
+#include "lib/crypt_ops/crypto_options.inc"
+#undef CONF_CONTEXT
+
+typedef struct crypto_options_t crypto_options_t;
+
+#endif /* !defined(TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H) */
diff --git a/src/lib/crypt_ops/crypto_pwbox.c b/src/lib/crypt_ops/crypto_pwbox.c
index a8db08f7b7..bfad27d9fc 100644
--- a/src/lib/crypt_ops/crypto_pwbox.c
+++ b/src/lib/crypt_ops/crypto_pwbox.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_pwbox.h b/src/lib/crypt_ops/crypto_pwbox.h
index 5a26889fb2..0a85b1230a 100644
--- a/src/lib/crypt_ops/crypto_pwbox.h
+++ b/src/lib/crypt_ops/crypto_pwbox.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c
index 206929d6b3..ce6f21dbb4 100644
--- a/src/lib/crypt_ops/crypto_rand.c
+++ b/src/lib/crypt_ops/crypto_rand.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,6 @@
  * number generators, and working with randomness.
  **/
 
-#ifndef CRYPTO_RAND_PRIVATE
 #define CRYPTO_RAND_PRIVATE
 
 #include "lib/crypt_ops/crypto_rand.h"
@@ -37,17 +36,18 @@
 
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/crypto_digest.h"
+#include "lib/ctime/di_ops.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
 #endif
 
 #ifdef ENABLE_OPENSSL
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 #include <openssl/rand.h>
 #include <openssl/sha.h>
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 #include <pk11pub.h>
@@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
   (void)out;
   (void)out_len;
   return -1;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   static const char *filenames[] = {
     "/dev/srandom", "/dev/urandom", "/dev/random", NULL
   };
@@ -315,7 +315,7 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
       }
     }
 
-    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
+    if ((out_len < sanity_min_size) || !safe_mem_is_zero((char*)out, out_len))
       return 0;
   }
 
@@ -419,7 +419,7 @@ crypto_seed_openssl_rng(void)
   else
     return -1;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 /**
@@ -442,7 +442,7 @@ crypto_seed_nss_rng(void)
 
   return load_entropy_ok ? 0 : -1;
 }
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 /**
  * Seed the RNG for any and all crypto libraries that we're using with bytes
@@ -520,7 +520,7 @@ crypto_rand_unmocked(char *to, size_t n)
 
 #undef BUFLEN
   }
-#else
+#else /* !defined(ENABLE_NSS) */
   int r = RAND_bytes((unsigned char*)to, (int)n);
   /* We consider a PRNG failure non-survivable. Let's assert so that we get a
    * stack trace about where it happened.
@@ -530,111 +530,14 @@ crypto_rand_unmocked(char *to, size_t n)
 }
 
 /**
- * Return a pseudorandom integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
- * INT_MAX+1, inclusive.
+ * Draw an unsigned 32-bit integer uniformly at random.
  */
-int
-crypto_rand_int(unsigned int max)
-{
-  unsigned int val;
-  unsigned int cutoff;
-  tor_assert(max <= ((unsigned int)INT_MAX)+1);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT_MAX - (UINT_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom integer, chosen uniformly from the values i such
- * that min <= i < max.
- *
- * <b>min</b> MUST be in range [0, <b>max</b>).
- * <b>max</b> MUST be in range (min, INT_MAX].
- **/
-int
-crypto_rand_int_range(unsigned int min, unsigned int max)
-{
-  tor_assert(min < max);
-  tor_assert(max <= INT_MAX);
-
-  /* The overflow is avoided here because crypto_rand_int() returns a value
-   * between 0 and (max - min) inclusive. */
-  return min + crypto_rand_int(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports uint64_t.
- **/
-uint64_t
-crypto_rand_uint64_range(uint64_t min, uint64_t max)
+uint32_t
+crypto_rand_u32(void)
 {
-  tor_assert(min < max);
-  return min + crypto_rand_uint64(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports time_t.
- **/
-time_t
-crypto_rand_time_range(time_t min, time_t max)
-{
-  tor_assert(min < max);
-  return min + (time_t)crypto_rand_uint64(max - min);
-}
-
-/**
- * Return a pseudorandom 64-bit integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.
- **/
-uint64_t
-crypto_rand_uint64(uint64_t max)
-{
-  uint64_t val;
-  uint64_t cutoff;
-  tor_assert(max < UINT64_MAX);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT64_MAX - (UINT64_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom double d, chosen uniformly from the range
- * 0.0 <= d < 1.0.
- **/
-double
-crypto_rand_double(void)
-{
-  /* We just use an unsigned int here; we don't really care about getting
-   * more than 32 bits of resolution */
-  unsigned int u;
-  crypto_rand((char*)&u, sizeof(u));
-#if SIZEOF_INT == 4
-#define UINT_MAX_AS_DOUBLE 4294967296.0
-#elif SIZEOF_INT == 8
-#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
-#else
-#error SIZEOF_INT is neither 4 nor 8
-#endif /* SIZEOF_INT == 4 || ... */
-  return ((double)u) / UINT_MAX_AS_DOUBLE;
+  uint32_t rand;
+  crypto_rand((void*)&rand, sizeof(rand));
+  return rand;
 }
 
 /**
@@ -724,8 +627,6 @@ crypto_force_rand_ssleay(void)
     RAND_set_rand_method(default_method);
     return 1;
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   return 0;
 }
-
-#endif /* !defined(CRYPTO_RAND_PRIVATE) */
diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h
index 86fa20faa3..99aff5d4a9 100644
--- a/src/lib/crypt_ops/crypto_rand.h
+++ b/src/lib/crypt_ops/crypto_rand.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,7 @@
 #include "lib/cc/compat_compiler.h"
 #include "lib/cc/torint.h"
 #include "lib/testsupport/testsupport.h"
+#include "lib/malloc/malloc.h"
 
 /* random numbers */
 int crypto_seed_rng(void) ATTR_WUR;
@@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n);
 void crypto_strongest_rand(uint8_t *out, size_t out_len);
 MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len));
 int crypto_rand_int(unsigned int max);
+unsigned crypto_rand_uint(unsigned limit);
 int crypto_rand_int_range(unsigned int min, unsigned int max);
 uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
 time_t crypto_rand_time_range(time_t min, time_t max);
+uint32_t crypto_rand_u32(void);
 uint64_t crypto_rand_uint64(uint64_t max);
 double crypto_rand_double(void);
 struct tor_weak_rng_t;
@@ -40,6 +43,61 @@ void *smartlist_choose(const struct smartlist_t *sl);
 void smartlist_shuffle(struct smartlist_t *sl);
 int crypto_force_rand_ssleay(void);
 
+/**
+ * A fast PRNG, for use when the PRNG provided by our crypto library isn't
+ * fast enough.  This one _should_ be cryptographically strong, but
+ * has seen less auditing than the PRNGs in OpenSSL and NSS.  Use with
+ * caution.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+typedef struct crypto_fast_rng_t crypto_fast_rng_t;
+/**
+ * Number of bytes used to seed a crypto_rand_fast_t.
+ **/
+crypto_fast_rng_t *crypto_fast_rng_new(void);
+#define CRYPTO_FAST_RNG_SEED_LEN 48
+crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed);
+void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n);
+void crypto_fast_rng_free_(crypto_fast_rng_t *);
+#define crypto_fast_rng_free(c)                                 \
+  FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c))
+
+unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit);
+uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit);
+uint32_t crypto_fast_rng_get_u32(crypto_fast_rng_t *rng);
+uint64_t crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                                      uint64_t min, uint64_t max);
+double crypto_fast_rng_get_double(crypto_fast_rng_t *rng);
+
+/**
+ * Using the fast_rng <b>rng</b>, yield true with probability
+ * 1/<b>n</b>. Otherwise yield false.
+ *
+ * <b>n</b> must not be zero.
+ **/
+#define crypto_fast_rng_one_in_n(rng, n)        \
+  (0 == (crypto_fast_rng_get_uint((rng), (n))))
+
+crypto_fast_rng_t *get_thread_fast_rng(void);
+
+#ifdef CRYPTO_PRIVATE
+/* These are only used from crypto_init.c */
+void destroy_thread_fast_rng(void);
+void crypto_rand_fast_init(void);
+void crypto_rand_fast_shutdown(void);
+#endif /* defined(CRYPTO_PRIVATE) */
+
+#if defined(TOR_UNIT_TESTS)
+/* Used for white-box testing */
+size_t crypto_fast_rng_get_bytes_used_per_stream(void);
+/* For deterministic prng implementations */
+void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng);
+/* To override the prng for testing. */
+crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng);
+#endif /* defined(TOR_UNIT_TESTS) */
+
 #ifdef CRYPTO_RAND_PRIVATE
 
 STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c
new file mode 100644
index 0000000000..172ea48bdb
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_fast.c
@@ -0,0 +1,438 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand_fast.c
+ *
+ * \brief A fast strong PRNG for use when our underlying cryptographic
+ *   library's PRNG isn't fast enough.
+ **/
+
+/* This library is currently implemented to use the same implementation
+ * technique as libottery, using AES-CTR-256 as our underlying stream cipher.
+ * It's backtracking-resistant immediately, and prediction-resistant after
+ * a while.
+ *
+ * Here's how it works:
+ *
+ * We generate pseudorandom bytes using AES-CTR-256.  We generate BUFLEN bytes
+ * at a time.  When we do this, we keep the first SEED_LEN bytes as the key
+ * and the IV for our next invocation of AES_CTR, and yield the remaining
+ * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG.  As we yield
+ * bytes to the user, we clear them from the buffer.
+ *
+ * After we have refilled the buffer RESEED_AFTER times, we mix in an
+ * additional SEED_LEN bytes from our strong PRNG into the seed.
+ *
+ * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN
+ * bytes from the PRNG and use them with our stream cipher to fill the user's
+ * request.
+ */
+
+#define CRYPTO_PRIVATE
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/intmath/cmp.h"
+#include "lib/cc/ctassert.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/thread/threads.h"
+
+#include "lib/log/util_bug.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <string.h>
+
+#ifdef NOINHERIT_CAN_FAIL
+#define CHECK_PID
+#endif
+
+#ifdef CHECK_PID
+#define PID_FIELD_LEN sizeof(pid_t)
+#else
+#define PID_FIELD_LEN 0
+#endif
+
+/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter.
+ */
+#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN)
+
+/* The amount of space that we mmap for a crypto_fast_rng_t.
+ */
+#define MAPLEN 4096
+
+/* The number of random bytes that we can yield to the user after each
+ * time we fill a crypto_fast_rng_t's buffer.
+ */
+#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN - PID_FIELD_LEN)
+
+/* The number of buffer refills after which we should fetch more
+ * entropy from crypto_strongest_rand().
+ */
+#define RESEED_AFTER 16
+
+/* The length of the stream cipher key we will use for the PRNG, in bytes.
+ */
+#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN)
+/* The length of the stream cipher key we will use for the PRNG, in bits.
+ */
+#define KEY_BITS (KEY_LEN * 8)
+
+/* Make sure that we have a key length we can actually use with AES. */
+CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256);
+
+struct crypto_fast_rng_t {
+  /** How many more fills does this buffer have before we should mix
+   * in the output of crypto_strongest_rand()?
+   *
+   * This value may be negative if unit tests are enabled.  If so, it
+   * indicates that we should never mix in extra data from
+   * crypto_strongest_rand().
+   */
+  int16_t n_till_reseed;
+  /** How many bytes are remaining in cbuf_t.bytes? */
+  uint16_t bytes_left;
+#ifdef CHECK_PID
+  /** Which process owns this fast_rng?  If this value is zero, we do not
+   * need to test the owner. */
+  pid_t owner;
+#endif
+  struct cbuf_t {
+    /** The seed (key and IV) that we will use the next time that we refill
+     * cbuf_t. */
+    uint8_t seed[SEED_LEN];
+    /**
+     * Bytes that we are yielding to the user.  The next byte to be
+     * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this
+     * array are set to zero.
+     */
+    uint8_t bytes[BUFLEN];
+  } buf;
+};
+
+/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf_t.
+ */
+CTASSERT(sizeof(struct cbuf_t) == BUFLEN+SEED_LEN);
+/* We're trying to fit all of the RNG state into a nice mmapable chunk.
+ */
+CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN);
+
+/**
+ * Initialize and return a new fast PRNG, using a strong random seed.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new(void)
+{
+  uint8_t seed[SEED_LEN];
+  crypto_strongest_rand(seed, sizeof(seed));
+  crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed);
+  memwipe(seed, 0, sizeof(seed));
+  return result;
+}
+
+/**
+ * Initialize and return a new fast PRNG, using a seed value specified
+ * in <b>seed</b>.  This value must be CRYPTO_FAST_RNG_SEED_LEN bytes
+ * long.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, you should probably look at get_thread_fast_rng().  Alternatively,
+ * use crypto_rand(), wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new_from_seed(const uint8_t *seed)
+{
+  unsigned inherit = INHERIT_RES_KEEP;
+  /* We try to allocate this object as securely as we can, to avoid
+   * having it get dumped, swapped, or shared after fork.
+   */
+  crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result),
+                                ANONMAP_PRIVATE | ANONMAP_NOINHERIT,
+                                &inherit);
+  memcpy(result->buf.seed, seed, SEED_LEN);
+  /* Causes an immediate refill once the user asks for data. */
+  result->bytes_left = 0;
+  result->n_till_reseed = RESEED_AFTER;
+#ifdef CHECK_PID
+  if (inherit == INHERIT_RES_KEEP) {
+    /* This value will neither be dropped nor zeroed after fork, so we need to
+     * check our pid to make sure we are not sharing it across a fork.  This
+     * can be expensive if the pid value isn't cached, sadly.
+     */
+    result->owner = getpid();
+  }
+#elif defined(_WIN32)
+  /* Windows can't fork(), so there's no need to noinherit. */
+#else
+  /* We decided above that noinherit would always do _something_. Assert here
+   * that we were correct. */
+  tor_assertf(inherit != INHERIT_RES_KEEP,
+              "We failed to create a non-inheritable memory region, even "
+              "though we believed such a failure to be impossible! This is "
+              "probably a bug in Tor support for your platform; please report "
+              "it.");
+#endif /* defined(CHECK_PID) || ... */
+  return result;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more
+ * entropy.
+ */
+void
+crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed = -1;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Helper: create a crypto_cipher_t object from SEED_LEN bytes of
+ * input.  The first KEY_LEN bytes are used as the stream cipher's key,
+ * and the remaining CIPHER_IV_LEN bytes are used as its IV.
+ **/
+static inline crypto_cipher_t *
+cipher_from_seed(const uint8_t *seed)
+{
+  return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS);
+}
+
+/**
+ * Helper: mix additional entropy into <b>rng</b> by using our XOF to mix the
+ * old value for the seed with some additional bytes from
+ * crypto_strongest_rand().
+ **/
+static void
+crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng)
+{
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN);
+  {
+    uint8_t seedbuf[SEED_LEN];
+    crypto_strongest_rand(seedbuf, SEED_LEN);
+    crypto_xof_add_bytes(xof, seedbuf, SEED_LEN);
+    memwipe(seedbuf, 0, SEED_LEN);
+  }
+  crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN);
+  crypto_xof_free(xof);
+}
+
+/**
+ * Helper: refill the seed bytes and output buffer of <b>rng</b>, using
+ * the input seed bytes as input (key and IV) for the stream cipher.
+ *
+ * If the n_till_reseed counter has reached zero, mix more random bytes into
+ * the seed before refilling the buffer.
+ **/
+static void
+crypto_fast_rng_refill(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed--;
+  if (rng->n_till_reseed == 0) {
+    /* It's time to reseed the RNG. */
+    crypto_fast_rng_add_entopy(rng);
+    rng->n_till_reseed = RESEED_AFTER;
+  } else if (rng->n_till_reseed < 0) {
+#ifdef TOR_UNIT_TESTS
+    /* Reseeding is disabled for testing; never do it on this prng. */
+    rng->n_till_reseed = -1;
+#else
+    /* If testing is disabled, this shouldn't be able to become negative. */
+    tor_assert_unreached();
+#endif /* defined(TOR_UNIT_TESTS) */
+  }
+  /* Now fill rng->buf with output from our stream cipher, initialized from
+   * that seed value. */
+  crypto_cipher_t *c = cipher_from_seed(rng->buf.seed);
+  memset(&rng->buf, 0, sizeof(rng->buf));
+  crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf));
+  crypto_cipher_free(c);
+
+  rng->bytes_left = sizeof(rng->buf.bytes);
+}
+
+/**
+ * Release all storage held by <b>rng</b>.
+ **/
+void
+crypto_fast_rng_free_(crypto_fast_rng_t *rng)
+{
+  if (!rng)
+    return;
+  memwipe(rng, 0, sizeof(*rng));
+  tor_munmap_anonymous(rng, sizeof(*rng));
+}
+
+/**
+ * Helper: extract bytes from the PRNG, refilling it as necessary. Does not
+ * optimize the case when the user has asked for a huge output.
+ **/
+static void
+crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out,
+                              const size_t n)
+{
+#ifdef CHECK_PID
+  if (rng->owner) {
+    /* Note that we only need to do this check when we have owner set: that
+     * is, when our attempt to block inheriting failed, and the result was
+     * INHERIT_RES_KEEP.
+     *
+     * If the result was INHERIT_RES_DROP, then any attempt to access the rng
+     * memory after forking will crash.
+     *
+     * If the result was INHERIT_RES_ZERO, then forking will set the bytes_left
+     * and n_till_reseed fields to zero.  This function will call
+     * crypto_fast_rng_refill(), which will in turn reseed the PRNG.
+     *
+     * So we only need to do this test in the case when mmap_anonymous()
+     * returned INHERIT_KEEP.  We avoid doing it needlessly, since getpid() is
+     * often a system call, and that can be slow.
+     */
+    tor_assert(rng->owner == getpid());
+  }
+#endif /* defined(CHECK_PID) */
+
+  size_t bytes_to_yield = n;
+
+  while (bytes_to_yield) {
+    if (rng->bytes_left == 0)
+      crypto_fast_rng_refill(rng);
+
+    const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield);
+
+    tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left);
+    uint8_t *copy_from = rng->buf.bytes +
+      (sizeof(rng->buf.bytes) - rng->bytes_left);
+    memcpy(out, copy_from, to_copy);
+    memset(copy_from, 0, to_copy);
+
+    out += to_copy;
+    bytes_to_yield -= to_copy;
+    rng->bytes_left -= to_copy;
+  }
+}
+
+/**
+ * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>.
+ **/
+void
+crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n)
+{
+  if (PREDICT_UNLIKELY(n > BUFLEN)) {
+    /* The user has asked for a lot of output; generate it from a stream
+     * cipher seeded by the PRNG rather than by pulling it out of the PRNG
+     * directly.
+     */
+    uint8_t seed[SEED_LEN];
+    crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN);
+    crypto_cipher_t *c = cipher_from_seed(seed);
+    memset(out, 0, n);
+    crypto_cipher_crypt_inplace(c, (char*)out, n);
+    crypto_cipher_free(c);
+    memwipe(seed, 0, sizeof(seed));
+    return;
+  }
+
+  crypto_fast_rng_getbytes_impl(rng, out, n);
+}
+
+#if defined(TOR_UNIT_TESTS)
+/** for white-box testing: return the number of bytes that are returned from
+ * the user for each invocation of the stream cipher in this RNG. */
+size_t
+crypto_fast_rng_get_bytes_used_per_stream(void)
+{
+  return BUFLEN;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Thread-local instance for our fast RNG.
+ **/
+static tor_threadlocal_t thread_rng;
+
+/**
+ * Return a per-thread fast RNG, initializing it if necessary.
+ *
+ * You do not need to free this yourself.
+ *
+ * It is NOT safe to share this value across threads.
+ **/
+crypto_fast_rng_t *
+get_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+
+  if (PREDICT_UNLIKELY(rng == NULL)) {
+    rng = crypto_fast_rng_new();
+    tor_threadlocal_set(&thread_rng, rng);
+  }
+
+  return rng;
+}
+
+/**
+ * Used when a thread is exiting: free the per-thread fast RNG if needed.
+ * Invoked from the crypto subsystem's thread-cleanup code.
+ **/
+void
+destroy_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+  if (!rng)
+    return;
+  crypto_fast_rng_free(rng);
+  tor_threadlocal_set(&thread_rng, NULL);
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Replace the current thread's rng with <b>rng</b>. For use by the
+ * unit tests only.  Returns the previous thread rng.
+ **/
+crypto_fast_rng_t *
+crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng)
+{
+  crypto_fast_rng_t *old_rng =  tor_threadlocal_get(&thread_rng);
+  tor_threadlocal_set(&thread_rng, rng);
+  return old_rng;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * initialization code.
+ **/
+void
+crypto_rand_fast_init(void)
+{
+  tor_threadlocal_init(&thread_rng);
+}
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * shutdown code.
+ **/
+void
+crypto_rand_fast_shutdown(void)
+{
+  destroy_thread_fast_rng();
+  tor_threadlocal_destroy(&thread_rng);
+}
diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c
new file mode 100644
index 0000000000..b2516c4bdc
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_numeric.c
@@ -0,0 +1,194 @@
+/**
+ * \file crypto_rand_numeric.c
+ *
+ * \brief Functions for retrieving uniformly distributed numbers
+ *   from our PRNGs.
+ **/
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/log/util_bug.h"
+
+/**
+ * Implementation macro: yields code that returns a uniform unbiased
+ * random number between 0 and limit.  "type" is the type of the number to
+ * return; "maxval" is the largest possible value of "type"; and "fill_stmt"
+ * is a code snippet that fills an object named "val" with random bits.
+ **/
+#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt)         \
+  do {                                                                  \
+    type val;                                                           \
+    type cutoff;                                                        \
+    tor_assert((limit) > 0);                                            \
+                                                                        \
+    /* We ignore any values that are >= 'cutoff,' to avoid biasing */   \
+    /* the distribution with clipping at the upper end of the type's */ \
+    /* range. */                                                        \
+    cutoff = (maxval) - ((maxval)%(limit));                             \
+    while (1) {                                                         \
+      fill_stmt;                                                        \
+      if (val < cutoff)                                                 \
+        return val % (limit);                                           \
+    }                                                                   \
+  } while (0)
+
+/**
+ * Return a pseudorandom integer chosen uniformly from the values between 0
+ * and <b>limit</b>-1 inclusive. limit must be strictly greater than 0, and
+ * less than UINT_MAX. */
+unsigned
+crypto_rand_uint(unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
+ * INT_MAX+1, inclusive.
+ */
+int
+crypto_rand_int(unsigned int max)
+{
+  /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying
+   * to return a signed type. Instead we make sure that the range is
+   * reasonable for a nonnegative int, use crypto_rand_uint(), and cast.
+   */
+  tor_assert(max <= ((unsigned int)INT_MAX)+1);
+
+  return (int)crypto_rand_uint(max);
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values i such
+ * that min <= i < max.
+ *
+ * <b>min</b> MUST be in range [0, <b>max</b>).
+ * <b>max</b> MUST be in range (min, INT_MAX].
+ **/
+int
+crypto_rand_int_range(unsigned int min, unsigned int max)
+{
+  tor_assert(min < max);
+  tor_assert(max <= INT_MAX);
+
+  /* The overflow is avoided here because crypto_rand_int() returns a value
+   * between 0 and (max - min) inclusive. */
+  return min + crypto_rand_int(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports uint64_t.
+ **/
+uint64_t
+crypto_rand_uint64_range(uint64_t min, uint64_t max)
+{
+  tor_assert(min < max);
+  return min + crypto_rand_uint64(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports time_t.
+ **/
+time_t
+crypto_rand_time_range(time_t min, time_t max)
+{
+  tor_assert(min < max);
+  return min + (time_t)crypto_rand_uint64(max - min);
+}
+
+/**
+ * Return a pseudorandom 64-bit integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.
+ **/
+uint64_t
+crypto_rand_uint64(uint64_t max)
+{
+  tor_assert(max < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+#if SIZEOF_INT == 4
+#define UINT_MAX_AS_DOUBLE 4294967296.0
+#elif SIZEOF_INT == 8
+#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
+#else
+#error SIZEOF_INT is neither 4 nor 8
+#endif /* SIZEOF_INT == 4 || ... */
+
+/**
+ * Return a pseudorandom double d, chosen uniformly from the range
+ * 0.0 <= d < 1.0.
+ **/
+double
+crypto_rand_double(void)
+{
+  /* We just use an unsigned int here; we don't really care about getting
+   * more than 32 bits of resolution */
+  unsigned int u;
+  crypto_rand((char*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
+/**
+ * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t
+ */
+unsigned
+crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit)
+{
+  tor_assert(limit < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_u32, but extract the result from a crypto_fast_rng_t.
+ */
+uint32_t
+crypto_fast_rng_get_u32(crypto_fast_rng_t *rng)
+{
+  uint32_t val;
+  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val));
+  return val;
+}
+
+/**
+ * As crypto_rand_uint64_range(), but extract the result from a
+ * crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                             uint64_t min, uint64_t max)
+{
+  /* Handle corrupted input */
+  if (BUG(min >= max)) {
+    return min;
+  }
+
+  return min + crypto_fast_rng_get_uint64(rng, max - min);
+}
+
+/**
+ * As crypto_rand_get_double() but extract the result from a crypto_fast_rng_t.
+ */
+double
+crypto_fast_rng_get_double(crypto_fast_rng_t *rng)
+{
+  unsigned int u;
+  crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c
index 8fd8a8aa7b..195e4bbaf9 100644
--- a/src/lib/crypt_ops/crypto_rsa.c
+++ b/src/lib/crypt_ops/crypto_rsa.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -59,7 +59,7 @@ crypto_get_rsa_padding(int padding)
     default: tor_assert(0); return -1; // LCOV_EXCL_LINE
     }
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Compare the public-key components of a and b.  Return non-zero iff
  * a==b.  A NULL key is considered to be distinct from all non-NULL
diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h
index 6d9cc8d30e..ab2e9db80d 100644
--- a/src/lib/crypt_ops/crypto_rsa.h
+++ b/src/lib/crypt_ops/crypto_rsa.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -122,7 +122,7 @@ struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env);
 crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa);
 MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,(
                                  crypto_pk_t *env,int private));
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 struct SECKEYPublicKeyStr;
@@ -132,7 +132,7 @@ const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey(
                                            const crypto_pk_t *key);
 const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey(
                                            const crypto_pk_t *key);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src);
 void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src);
@@ -143,6 +143,6 @@ struct SECItemStr;
 STATIC int secitem_uint_cmp(const struct SECItemStr *a,
                             const struct SECItemStr *b);
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_CRYPTO_RSA_H) */
diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c
index 7abf6716f0..66f325e868 100644
--- a/src/lib/crypt_ops/crypto_rsa_nss.c
+++ b/src/lib/crypt_ops/crypto_rsa_nss.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -156,7 +156,7 @@ crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private))
   tor_free(buf);
   return result;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Allocate and return storage for a public key.  The key itself will not yet
  * be set.
@@ -645,7 +645,7 @@ crypto_pk_asn1_decode(const char *str, size_t len)
   return result;
 }
 
-DISABLE_GCC_WARNING(unused-parameter)
+DISABLE_GCC_WARNING("-Wunused-parameter")
 
 /** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the Base64
  * encoding of the DER representation of the private key into the
diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c
index 17eae24cc2..c96ee81fd3 100644
--- a/src/lib/crypt_ops/crypto_rsa_openssl.c
+++ b/src/lib/crypt_ops/crypto_rsa_openssl.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/fs/files.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/err.h>
 #include <openssl/rsa.h>
@@ -27,7 +27,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/bn.h>
 #include <openssl/conf.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "lib/log/log.h"
 #include "lib/encoding/binascii.h"
@@ -54,7 +54,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k)
   const BIGNUM *p, *q;
   RSA_get0_factors(k->key, &p, &q);
   return p != NULL; /* XXX/yawning: Should we check q? */
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   return k && k->key && k->key->p;
 #endif /* defined(OPENSSL_1_1_API) */
 }
@@ -288,7 +288,7 @@ crypto_pk_num_bits(crypto_pk_t *env)
   tor_assert(n != NULL);
 
   return RSA_bits(env->key);
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   tor_assert(env->key->n);
   return BN_num_bits(env->key->n);
 #endif /* defined(OPENSSL_1_1_API) */
@@ -583,15 +583,15 @@ rsa_private_key_too_long(RSA *rsa, int max_bits)
   dmp1 = RSA_get0_dmp1(rsa);
   dmq1 = RSA_get0_dmq1(rsa);
   iqmp = RSA_get0_iqmp(rsa);
-#else
+#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,1)) */
   /* The accessors above did not exist in openssl 1.1.0. */
   p = q = dmp1 = dmq1 = iqmp = NULL;
   RSA_get0_key(rsa, &n, &e, &d);
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,1) */
 
   if (RSA_bits(rsa) > max_bits)
     return true;
-#else
+#else /* !defined(OPENSSL_1_1_API) */
   n = rsa->n;
   e = rsa->e;
   p = rsa->p;
@@ -600,7 +600,7 @@ rsa_private_key_too_long(RSA *rsa, int max_bits)
   dmp1 = rsa->dmp1;
   dmq1 = rsa->dmq1;
   iqmp = rsa->iqmp;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
   if (n && BN_num_bits(n) > max_bits)
     return true;
diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c
index 42276597d4..3a9ed5ef58 100644
--- a/src/lib/crypt_ops/crypto_s2k.c
+++ b/src/lib/crypt_ops/crypto_s2k.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv < 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
       SECItem passItem = { .type = siBuffer,
                            .data = (unsigned char *) secret,
                            .len = (int)secret_len };
@@ -325,7 +325,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (alg)
         SECOID_DestroyAlgorithmID(alg, PR_TRUE);
       return rv;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
     }
 
     case S2K_TYPE_SCRYPT: {
@@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv != 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else /* !(defined(HAVE_SCRYPT)) */
+#else /* !defined(HAVE_SCRYPT) */
       return S2K_NO_SCRYPT_SUPPORT;
 #endif /* defined(HAVE_SCRYPT) */
     }
@@ -380,7 +380,7 @@ secret_to_key_derivekey(uint8_t *key_out, size_t key_out_len,
 #ifndef HAVE_SCRYPT
   if (type == S2K_TYPE_SCRYPT)
     return S2K_NO_SCRYPT_SUPPORT;
- #endif
+#endif
 
   if (! legacy_format) {
     ++spec;
diff --git a/src/lib/crypt_ops/crypto_s2k.h b/src/lib/crypt_ops/crypto_s2k.h
index a16a3d781e..181a17acb1 100644
--- a/src/lib/crypt_ops/crypto_s2k.h
+++ b/src/lib/crypt_ops/crypto_s2k.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h
new file mode 100644
index 0000000000..2115d4fc99
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_sys.h
+ * \brief Declare subsystem object for the crypto module.
+ **/
+
+#ifndef TOR_CRYPTO_SYS_H
+#define TOR_CRYPTO_SYS_H
+
+extern const struct subsys_fns_t sys_crypto;
+
+#endif /* !defined(TOR_CRYPTO_SYS_H) */
diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c
index 67a1a9eb92..7ebb860d09 100644
--- a/src/lib/crypt_ops/crypto_util.c
+++ b/src/lib/crypt_ops/crypto_util.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,8 +10,6 @@
  * \brief Common cryptographic utilities.
  **/
 
-#define CRYPTO_UTIL_PRIVATE
-
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/cc/compat_compiler.h"
 
@@ -26,11 +24,11 @@
 #include <stdlib.h>
 
 #ifdef ENABLE_OPENSSL
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 #include <openssl/err.h>
 #include <openssl/crypto.h>
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -109,3 +107,17 @@ memwipe(void *mem, uint8_t byte, size_t sz)
    **/
   memset(mem, byte, sz);
 }
+
+/**
+ * Securely all memory in <b>str</b>, then free it.
+ *
+ * As tor_free(), tolerates null pointers.
+ **/
+void
+tor_str_wipe_and_free_(char *str)
+{
+  if (!str)
+    return;
+  memwipe(str, 0, strlen(str));
+  tor_free_(str);
+}
diff --git a/src/lib/crypt_ops/crypto_util.h b/src/lib/crypt_ops/crypto_util.h
index 613a1bd0dd..36ee230176 100644
--- a/src/lib/crypt_ops/crypto_util.h
+++ b/src/lib/crypt_ops/crypto_util.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,8 +14,18 @@
 #define TOR_CRYPTO_UTIL_H
 
 #include "lib/cc/torint.h"
+#include "lib/malloc/malloc.h"
 
 /** OpenSSL-based utility functions. */
 void memwipe(void *mem, uint8_t byte, size_t sz);
 
+void tor_str_wipe_and_free_(char *str);
+/**
+ * Securely all memory in <b>str</b>, then free it.
+ *
+ * As tor_free(), tolerates null pointers, and sets <b>str</b> to NULL.
+ **/
+#define tor_str_wipe_and_free(str)                      \
+  FREE_AND_NULL(char, tor_str_wipe_and_free_, (str))
+
 #endif /* !defined(TOR_CRYPTO_UTIL_H) */
diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c
index 0dba64d595..5162d6d8bd 100644
--- a/src/lib/crypt_ops/digestset.c
+++ b/src/lib/crypt_ops/digestset.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/digestset.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /* Wrap our hash function to have the signature that the bloom filter
  * needs. */
diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h
index 91d53a0542..505ac10395 100644
--- a/src/lib/crypt_ops/digestset.h
+++ b/src/lib/crypt_ops/digestset.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,4 +26,4 @@ void digestset_add(digestset_t *set, const char *addr);
 int digestset_probably_contains(const digestset_t *set,
                                 const char *addr);
 
-#endif
+#endif /* !defined(TOR_DIGESTSET_H) */
diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am
index 1022096fdc..7644cab412 100644
--- a/src/lib/crypt_ops/include.am
+++ b/src/lib/crypt_ops/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_cipher.c		\
 	src/lib/crypt_ops/crypto_curve25519.c		\
@@ -17,6 +18,8 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_ope.c          	\
 	src/lib/crypt_ops/crypto_pwbox.c		\
 	src/lib/crypt_ops/crypto_rand.c			\
+	src/lib/crypt_ops/crypto_rand_fast.c		\
+	src/lib/crypt_ops/crypto_rand_numeric.c		\
 	src/lib/crypt_ops/crypto_rsa.c			\
 	src/lib/crypt_ops/crypto_s2k.c			\
 	src/lib/crypt_ops/crypto_util.c                 \
@@ -25,12 +28,14 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 if USE_NSS
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_nss.c			\
+	src/lib/crypt_ops/crypto_digest_nss.c		\
 	src/lib/crypt_ops/crypto_dh_nss.c		\
 	src/lib/crypt_ops/crypto_nss_mgt.c		\
 	src/lib/crypt_ops/crypto_rsa_nss.c
 else
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_openssl.c			\
+	src/lib/crypt_ops/crypto_digest_openssl.c	\
 	src/lib/crypt_ops/crypto_rsa_openssl.c
 endif
 
@@ -48,6 +53,7 @@ src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_crypt_ops_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/crypt_ops/aes.h				\
 	src/lib/crypt_ops/compat_openssl.h		\
@@ -62,9 +68,12 @@ noinst_HEADERS +=					\
 	src/lib/crypt_ops/crypto_nss_mgt.h		\
 	src/lib/crypt_ops/crypto_openssl_mgt.h		\
 	src/lib/crypt_ops/crypto_ope.h          	\
+	src/lib/crypt_ops/crypto_options.inc		\
+	src/lib/crypt_ops/crypto_options_st.h		\
 	src/lib/crypt_ops/crypto_pwbox.h		\
 	src/lib/crypt_ops/crypto_rand.h			\
 	src/lib/crypt_ops/crypto_rsa.h			\
 	src/lib/crypt_ops/crypto_s2k.h			\
+	src/lib/crypt_ops/crypto_sys.h			\
 	src/lib/crypt_ops/crypto_util.h                 \
 	src/lib/crypt_ops/digestset.h
diff --git a/src/lib/crypt_ops/lib_crypt_ops.md b/src/lib/crypt_ops/lib_crypt_ops.md
new file mode 100644
index 0000000000..4e675e4871
--- /dev/null
+++ b/src/lib/crypt_ops/lib_crypt_ops.md
@@ -0,0 +1,137 @@
+@dir /lib/crypt_ops
+@brief lib/crypt_ops: Cryptographic operations.
+
+This module contains wrappers around the cryptographic libraries that we
+support, and implementations for some higher-level cryptographic
+constructions that we use.
+
+It wraps our two major cryptographic backends (OpenSSL or NSS, as configured
+by the user), and also wraps other cryptographic code in src/ext.
+
+Generally speaking, Tor code shouldn't be calling OpenSSL or NSS
+(or any other crypto library) directly.  Instead, we should indirect through
+one of the functions in this directory, or through \refdir{lib/tls}.
+
+Cryptography functionality that's available is described below.
+
+### RNG facilities ###
+
+The most basic RNG capability in Tor is the crypto_rand() family of
+functions. These currently use OpenSSL's RAND_() backend, but may use
+something faster in the future.
+
+In addition to crypto_rand(), which fills in a buffer with random
+bytes, we also have functions to produce random integers in certain
+ranges; to produce random hostnames; to produce random doubles, etc.
+
+When you're creating a long-term cryptographic secret, you might want
+to use crypto_strongest_rand() instead of crypto_rand().  It takes the
+operating system's entropy source and combines it with output from
+crypto_rand().  This is a pure paranoia measure, but it might help us
+someday.
+
+You can use smartlist_choose() to pick a random element from a smartlist
+and smartlist_shuffle() to randomize the order of a smartlist.  Both are
+potentially a bit slow.
+
+### Cryptographic digests and related functions ###
+
+We treat digests as separate types based on the length of their
+outputs.  We support one 160-bit digest (SHA1), two 256-bit digests
+(SHA256 and SHA3-256), and two 512-bit digests (SHA512 and SHA3-512).
+
+You should not use SHA1 for anything new.
+
+The crypto_digest\*() family of functions manipulates digests.  You
+can either compute a digest of a chunk of memory all at once using
+crypto_digest(), crypto_digest256(), or crypto_digest512().  Or you
+can create a crypto_digest_t object with
+crypto_digest{,256,512}_new(), feed information to it in chunks using
+crypto_digest_add_bytes(), and then extract the final digest using
+crypto_digest_get_digest().  You can copy the state of one of these
+objects using crypto_digest_dup() or crypto_digest_assign().
+
+We support the HMAC hash-based message authentication code
+instantiated using SHA256. See crypto_hmac_sha256.  (You should not
+add any HMAC users with SHA1, and HMAC is not necessary with SHA3.)
+
+We also support the SHA3 cousins, SHAKE128 and SHAKE256.  Unlike
+digests, these are extendable output functions (or XOFs) where you can
+get any amount of output.  Use the crypto_xof_\*() functions to access
+these.
+
+We have several ways to derive keys from cryptographically strong secret
+inputs (like diffie-hellman outputs). The old
+crypto_expand_key_material_TAP() performs an ad-hoc KDF based on SHA1 -- you
+shouldn't use it for implementing anything but old versions of the Tor
+protocol.  You can use HKDF-SHA256 (as defined in RFC5869) for more modern
+protocols.  Also consider SHAKE256.
+
+If your input is potentially weak, like a password or passphrase, use a salt
+along with the secret_to_key() functions as defined in crypto_s2k.c.  Prefer
+scrypt over other hashing methods when possible.  If you're using a password
+to encrypt something, see the "boxed file storage" section below.
+
+Finally, in order to store objects in hash tables, Tor includes the
+randomized SipHash 2-4 function.  Call it via the siphash24g() function in
+src/ext/siphash.h whenever you're creating a hashtable whose keys may be
+manipulated by an attacker in order to DoS you with collisions.
+
+
+### Stream ciphers ###
+
+You can create instances of a stream cipher using crypto_cipher_new().
+These are stateful objects of type crypto_cipher_t.  Note that these
+objects only support AES-128 right now; a future version should add
+support for AES-128 and/or ChaCha20.
+
+You can encrypt/decrypt with crypto_cipher_encrypt or
+crypto_cipher_decrypt. The crypto_cipher_crypt_inplace function performs
+an encryption without a copy.
+
+Note that sensible people should not use raw stream ciphers; they should
+probably be using some kind of AEAD. Sorry.
+
+### Public key functionality ###
+
+We support four public key algorithms: DH1024, RSA, Curve25519, and
+Ed25519.
+
+We support DH1024 over two prime groups.  You access these via the
+crypto_dh_\*() family of functions.
+
+We support RSA in many bit sizes for signing and encryption.  You access
+it via the crypto_pk_*() family of functions.  Note that a crypto_pk_t
+may or may not include a private key.  See the crypto_pk_* functions in
+crypto.c for a full list of functions here.
+
+For Curve25519 functionality, see the functions and types in
+crypto_curve25519.c. Curve25519 is generally suitable for when you need
+a secure fast elliptic-curve diffie hellman implementation. When
+designing new protocols, prefer it over DH in Z_p.
+
+For Ed25519 functionality, see the functions and types in
+crypto_ed25519.c. Ed25519 is a generally suitable as a secure fast
+elliptic curve signature method. For new protocols, prefer it over RSA
+signatures.
+
+### Metaformats for storage ###
+
+When OpenSSL manages the storage of some object, we use whatever format
+OpenSSL provides -- typically, some kind of PEM-wrapped base 64 encoding
+that starts with "----- BEGIN CRYPTOGRAPHIC OBJECT ----".
+
+When we manage the storage of some cryptographic object, we prefix the
+object with 32-byte NUL-padded prefix in order to avoid accidental
+object confusion; see the crypto_read_tagged_contents_from_file() and
+crypto_write_tagged_contents_to_file() functions for manipulating
+these. The prefix is "== type: tag ==", where type describes the object
+and its encoding, and tag indicates which one it is.
+
+### Boxed-file storage ###
+
+When managing keys, you frequently want to have some way to write a
+secret object to disk, encrypted with a passphrase.  The crypto_pwbox
+and crypto_unpwbox functions do so in a way that's likely to be
+readable by future versions of Tor.
+
diff --git a/src/lib/ctime/di_ops.c b/src/lib/ctime/di_ops.c
index 89e0837ae9..d57d286990 100644
--- a/src/lib/ctime/di_ops.c
+++ b/src/lib/ctime/di_ops.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -72,10 +72,10 @@ tor_memcmp(const void *a, const void *b, size_t len)
      * actually implementation-defined in standard C.  So how do we
      * get away with assuming it?  Easy.  We check.) */
 #if ((-60 >> 8) != -1)
-#error "According to cpp, right-shift doesn't perform sign-extension."
+#error "cpp says right-shift doesn't perform sign-extension."
 #endif
 #ifndef RSHIFT_DOES_SIGN_EXTEND
-#error "According to configure, right-shift doesn't perform sign-extension."
+#error "configure says right-shift doesn't perform sign-extension."
 #endif
 
     /* If v1 == v2, equal_p is ~0, so this will leave retval
@@ -145,8 +145,11 @@ tor_memeq(const void *a, const void *b, size_t sz)
 
 /* Implement di_digest256_map_t as a linked list of entries. */
 struct di_digest256_map_t {
+  /** Pointer to the next entry in the list. */
   struct di_digest256_map_t *next;
+  /** Key for this entry. */
   uint8_t key[32];
+  /** Value for this entry. */
   void *val;
 };
 
@@ -276,3 +279,30 @@ select_array_member_cumulative_timei(const uint64_t *entries, int n_entries,
 
   return i_chosen;
 }
+
+/**
+ * If <b>s</b> is true, then copy <b>n</b> bytes from <b>src</b> to
+ * <b>dest</b>.  Otherwise leave <b>dest</b> alone.
+ *
+ * This function behaves the same as
+ *
+ *      if (s)
+ *         memcpy(dest, src, n);
+ *
+ * except that it tries to run in the same amount of time whether <b>s</b> is
+ * true or not.
+ **/
+void
+memcpy_if_true_timei(bool s, void *dest, const void *src, size_t n)
+{
+  // If s is true, mask will be ~0.  If s is false, mask will be 0.
+  const char mask = (char) -(signed char)s;
+
+  char *destp = dest;
+  const char *srcp = src;
+  for (size_t i = 0; i < n; ++i) {
+    *destp = (*destp & ~mask) | (*srcp & mask);
+    ++destp;
+    ++srcp;
+  }
+}
diff --git a/src/lib/ctime/di_ops.h b/src/lib/ctime/di_ops.h
index 264b56a8c1..9fe2884ecc 100644
--- a/src/lib/ctime/di_ops.h
+++ b/src/lib/ctime/di_ops.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,8 @@
 
 int tor_memcmp(const void *a, const void *b, size_t sz);
 int tor_memeq(const void *a, const void *b, size_t sz);
+/** Perform a constant-time comparison of the <b>sz</b> bytes at <b>a</b> and
+ * <b>b</b>, yielding true if they are different, and false otherwise. */
 #define tor_memneq(a,b,sz) (!tor_memeq((a),(b),(sz)))
 
 /** Alias for the platform's memcmp() function.  This function is
@@ -24,7 +26,19 @@ int tor_memeq(const void *a, const void *b, size_t sz);
  * implementation.
  */
 #define fast_memcmp(a,b,c) (memcmp((a),(b),(c)))
+/** Alias for the platform's memcmp() function, for use in testing equality.
+ *
+ * This function is <em>not</em> data-independent: we define this alias so
+ * that we can mark cases where we are deliberately using a data-dependent
+ * memcmp() implementation.
+ */
 #define fast_memeq(a,b,c)  (0==memcmp((a),(b),(c)))
+/** Alias for the platform's memcmp() function, for use in testing inequality.
+ *
+ * This function is <em>not</em> data-independent: we define this alias so
+ * that we can mark cases where we are deliberately using a data-dependent
+ * memcmp() implementation.
+ */
 #define fast_memneq(a,b,c) (0!=memcmp((a),(b),(c)))
 
 int safe_mem_is_zero(const void *mem, size_t sz);
@@ -35,9 +49,17 @@ int safe_mem_is_zero(const void *mem, size_t sz);
  *
  * Not efficient for large maps! */
 typedef struct di_digest256_map_t di_digest256_map_t;
+/**
+ * Type for a function used to free members of a di_digest256_map_t.
+ **/
 typedef void (*dimap_free_fn)(void *);
 
 void dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn);
+/**
+ * @copydoc dimap_free_
+ *
+ * Additionally, set the pointer <b>map</b> to NULL.
+ **/
 #define dimap_free(map, free_fn)                \
   do {                                          \
     dimap_free_((map), (free_fn));              \
@@ -51,5 +73,6 @@ int select_array_member_cumulative_timei(const uint64_t *entries,
                                          int n_entries,
                                          uint64_t total, uint64_t rand_val);
 
-#endif /* !defined(TOR_DI_OPS_H) */
+void memcpy_if_true_timei(bool s, void *dest, const void *src, size_t n);
 
+#endif /* !defined(TOR_DI_OPS_H) */
diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am
index b46c43ba0c..83942ca4e0 100644
--- a/src/lib/ctime/include.am
+++ b/src/lib/ctime/include.am
@@ -11,6 +11,7 @@ else
 mulodi4_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_ctime_a_SOURCES =			\
 	$(mulodi4_source) 				\
 	src/ext/csiphash.c   				\
@@ -21,5 +22,6 @@ src_lib_libtor_ctime_testing_a_SOURCES = \
 src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@
 src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/ctime/di_ops.h
diff --git a/src/lib/ctime/lib_ctime.md b/src/lib/ctime/lib_ctime.md
new file mode 100644
index 0000000000..913199f6a5
--- /dev/null
+++ b/src/lib/ctime/lib_ctime.md
@@ -0,0 +1,14 @@
+@dir /lib/ctime
+@brief lib/ctime: Constant-time code to avoid side-channels.
+
+This module contains constant-time implementations of various
+data comparison and table lookup functions.  We use these in preference to
+memcmp() and so forth, since memcmp() can leak information about its inputs
+based on how fast it returns.  In general, your code should call tor_memeq()
+and tor_memneq(), not memcmp().
+
+We also define some _non_-constant-time wrappers for memcmp() here: Since we
+consider calls to memcmp() to be in error, we require that code that actually
+doesn't need to be constant-time to use the fast_memeq() / fast_memneq() /
+fast_memcmp() aliases instead.
+
diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h
index a2ffbc51c2..bc2707b36f 100644
--- a/src/lib/defs/dh_sizes.h
+++ b/src/lib/defs/dh_sizes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@
 /** Length of our legacy DH keys. */
 #define DH1024_KEY_LEN (1024/8)
 
-#endif
+#endif /* !defined(TOR_DH_SIZES_H) */
diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h
index 525e5209d6..7eef1747db 100644
--- a/src/lib/defs/digest_sizes.h
+++ b/src/lib/defs/digest_sizes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_DIGEST_SIZES_H
@@ -24,4 +24,4 @@
 /** Length of the output of our 64-bit optimized message digests (SHA512). */
 #define DIGEST512_LEN 64
 
-#endif
+#endif /* !defined(TOR_DIGEST_SIZES_H) */
diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am
index 48ee7f29fc..84ee403771 100644
--- a/src/lib/defs/include.am
+++ b/src/lib/defs/include.am
@@ -1,5 +1,8 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += 			\
 	src/lib/defs/dh_sizes.h 	\
 	src/lib/defs/digest_sizes.h	\
+	src/lib/defs/logging_types.h	\
+	src/lib/defs/time.h      	\
 	src/lib/defs/x25519_sizes.h
diff --git a/src/lib/defs/lib_defs.md b/src/lib/defs/lib_defs.md
new file mode 100644
index 0000000000..5762e4550b
--- /dev/null
+++ b/src/lib/defs/lib_defs.md
@@ -0,0 +1,2 @@
+@dir /lib/defs
+@brief lib/defs: Lowest-level constants, used in many places.
diff --git a/src/lib/defs/logging_types.h b/src/lib/defs/logging_types.h
new file mode 100644
index 0000000000..33aa46186b
--- /dev/null
+++ b/src/lib/defs/logging_types.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file logging_types.h
+ *
+ * \brief Global definition for types used by logging systems.
+ **/
+
+#ifndef TOR_LOGGING_TYPES_H
+#define TOR_LOGGING_TYPES_H
+
+/* We define this here so that it can be used both by backtrace.h and
+ * log.h.
+ */
+
+/** Mask of zero or more log domains, OR'd together. */
+typedef uint64_t log_domain_mask_t;
+
+#endif /* !defined(TOR_LOGGING_TYPES_H) */
diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h
new file mode 100644
index 0000000000..5707330795
--- /dev/null
+++ b/src/lib/defs/time.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TIME_DEFS_H
+#define TOR_TIME_DEFS_H
+
+/**
+ * \file time.h
+ *
+ * \brief Definitions for timing-related constants.
+ **/
+
+/** How many microseconds per second */
+#define TOR_USEC_PER_SEC (1000000)
+/** How many nanoseconds per microsecond */
+#define TOR_NSEC_PER_USEC (1000)
+/** How many nanoseconds per millisecond */
+#define TOR_NSEC_PER_MSEC (1000*1000)
+
+#endif /* !defined(TOR_TIME_DEFS_H) */
diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h
index 8933a8866b..acb08c5e6a 100644
--- a/src/lib/defs/x25519_sizes.h
+++ b/src/lib/defs/x25519_sizes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,14 +23,22 @@
 /** Length of the result of a curve25519 handshake. */
 #define CURVE25519_OUTPUT_LEN 32
 
+/** Length of an Ed25519 public key */
 #define ED25519_PUBKEY_LEN 32
+/** Length of an Ed25519 secret key */
 #define ED25519_SECKEY_LEN 64
+/** Length of the seed that is ordinarily expanded to an Ed25519 secret
+ * key. */
 #define ED25519_SECKEY_SEED_LEN 32
+/** Length of an Ed25519 signature. */
 #define ED25519_SIG_LEN 64
 
+/** Length of a Curve25519 key when encoded in base 64, with padding. */
 #define CURVE25519_BASE64_PADDED_LEN 44
 
+/** Length of a Ed25519 key when encoded in base 64, without padding. */
 #define ED25519_BASE64_LEN 43
+/** Length of a Ed25519 signature when encoded in base 64, without padding. */
 #define ED25519_SIG_BASE64_LEN 86
 
-#endif
+#endif /* !defined(TOR_X25519_SIZES_H) */
diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include
new file mode 100644
index 0000000000..884f4c0dbc
--- /dev/null
+++ b/src/lib/dispatch/.may_include
@@ -0,0 +1,11 @@
+orconfig.h
+
+ext/tor_queue.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
\ No newline at end of file
diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h
new file mode 100644
index 0000000000..9c7c4833c2
--- /dev/null
+++ b/src/lib/dispatch/dispatch.h
@@ -0,0 +1,114 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_H
+#define TOR_DISPATCH_H
+
+#include "lib/dispatch/msgtypes.h"
+
+/**
+ * \file dispatch.h
+ * \brief Low-level APIs for message-passing system.
+ *
+ * This module implements message dispatch based on a set of short integer
+ * identifiers.  For a higher-level interface, see pubsub.h.
+ *
+ * Each message is represented as a generic msg_t object, and is discriminated
+ * by its message_id_t.  Messages are delivered by a dispatch_t object, which
+ * delivers each message to its recipients by a configured "channel".
+ *
+ * A "channel" is a means of delivering messages.  Every message_id_t must
+ * be associated with exactly one channel, identified by channel_id_t.
+ * When a channel receives messages, a callback is invoked to either process
+ * the messages immediately, or to cause them to be processed later.
+ *
+ * Every message_id_t has zero or more associated receiver functions set up in
+ * the dispatch_t object.  Once the dispatch_t object is created, receivers
+ * can be enabled or disabled [TODO], but not added or removed.
+ *
+ * Every message_id_t has an associated datatype, identified by a
+ * msg_type_id_t.  These datatypes can be associated with functions to
+ * (for example) free them, or format them for debugging.
+ *
+ * To setup a dispatch_t object, first create a dispatch_cfg_t object, and
+ * configure messages with their types, channels, and receivers.  Then, use
+ * dispatch_new() with that dispatch_cfg_t to create the dispatch_t object.
+ *
+ * (We use a two-phase contruction procedure here to enable better static
+ * reasoning about publish/subscribe relationships.)
+ *
+ * Once you have a dispatch_t, you can queue messages on it with
+ * dispatch_send*(), and cause those messages to be delivered with
+ * dispatch_flush().
+ **/
+
+/**
+ * A "dispatcher" is the highest-level object; it handles making sure that
+ * messages are received and delivered properly.  Only the mainloop
+ * should handle this type directly.
+ */
+typedef struct dispatch_t dispatch_t;
+
+struct dispatch_cfg_t;
+
+dispatch_t *dispatch_new(const struct dispatch_cfg_t *cfg);
+
+/**
+ * Free a dispatcher.  Tor does this at exit.
+ */
+#define dispatch_free(d) \
+  FREE_AND_NULL(dispatch_t, dispatch_free_, (d))
+
+void dispatch_free_(dispatch_t *);
+
+int dispatch_send(dispatch_t *d,
+                  subsys_id_t sender,
+                  channel_id_t channel,
+                  message_id_t msg,
+                  msg_type_id_t type,
+                  msg_aux_data_t auxdata);
+
+int dispatch_send_msg(dispatch_t *d, msg_t *m);
+
+int dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m);
+
+/* Flush up to <b>max_msgs</b> currently pending messages from the
+ * dispatcher.  Messages that are not pending when this function are
+ * called, are not flushed by this call.  Return 0 on success, -1 on
+ * unrecoverable error.
+ */
+int dispatch_flush(dispatch_t *, channel_id_t chan, int max_msgs);
+
+/**
+ * Function callback type used to alert some other module when a channel's
+ * queue changes from empty to nonempty.
+ *
+ * Ex 1: To cause messages to be processed immediately on-stack, this callback
+ * should invoke dispatch_flush() directly.
+ *
+ * Ex 2: To cause messages to be processed very soon, from the event queue,
+ * this callback should schedule an event callback to run dispatch_flush().
+ *
+ * Ex 3: To cause messages to be processed periodically, this function should
+ * do nothing, and a periodic event should invoke dispatch_flush().
+ **/
+typedef void (*dispatch_alertfn_t)(struct dispatch_t *,
+                                   channel_id_t, void *);
+
+int dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                          dispatch_alertfn_t fn, void *userdata);
+
+#define dispatch_free_msg(d,msg)                                \
+  STMT_BEGIN {                                                  \
+    msg_t **msg_tmp_ptr__ = &(msg);                             \
+    dispatch_free_msg_((d), *msg_tmp_ptr__);                    \
+    *msg_tmp_ptr__= NULL;                                       \
+  } STMT_END
+void dispatch_free_msg_(const dispatch_t *d, msg_t *msg);
+
+char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg);
+
+#endif /* !defined(TOR_DISPATCH_H) */
diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c
new file mode 100644
index 0000000000..a54188dcaa
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.c
@@ -0,0 +1,141 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_cfg.c
+ * \brief Create and configure a dispatch_cfg_t.
+ *
+ * A dispatch_cfg_t object is used to configure a set of messages and
+ * associated information before creating a dispatch_t.
+ */
+
+#define DISPATCH_PRIVATE
+
+#include "orconfig.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/malloc/malloc.h"
+
+/**
+ * Create and return a new dispatch_cfg_t.
+ **/
+dispatch_cfg_t *
+dcfg_new(void)
+{
+  dispatch_cfg_t *cfg = tor_malloc(sizeof(dispatch_cfg_t));
+  cfg->type_by_msg = smartlist_new();
+  cfg->chan_by_msg = smartlist_new();
+  cfg->fns_by_type = smartlist_new();
+  cfg->recv_by_msg = smartlist_new();
+  return cfg;
+}
+
+/**
+ * Associate a message with a datatype.  Return 0 on success, -1 if a
+ * different type was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                  msg_type_id_t type)
+{
+  smartlist_grow(cfg->type_by_msg, msg+1);
+  msg_type_id_t *oldval = smartlist_get(cfg->type_by_msg, msg);
+  if (oldval != NULL && *oldval != type) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->type_by_msg, msg, tor_memdup(&type, sizeof(type)));
+  return 0;
+}
+
+/**
+ * Associate a message with a channel.  Return 0 on success, -1 if a
+ * different channel was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                  channel_id_t chan)
+{
+  smartlist_grow(cfg->chan_by_msg, msg+1);
+  channel_id_t *oldval = smartlist_get(cfg->chan_by_msg, msg);
+  if (oldval != NULL && *oldval != chan) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->chan_by_msg, msg, tor_memdup(&chan, sizeof(chan)));
+  return 0;
+}
+
+/**
+ * Associate a set of functions with a datatype. Return 0 on success, -1 if
+ * different functions were previously associated with the type.
+ **/
+int
+dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                  const dispatch_typefns_t *fns)
+{
+  smartlist_grow(cfg->fns_by_type, type+1);
+  dispatch_typefns_t *oldfns = smartlist_get(cfg->fns_by_type, type);
+  if (oldfns && (oldfns->free_fn != fns->free_fn ||
+                 oldfns->fmt_fn != fns->fmt_fn))
+    return -1;
+  if (!oldfns)
+    smartlist_set(cfg->fns_by_type, type, tor_memdup(fns, sizeof(*fns)));
+  return 0;
+}
+
+/**
+ * Associate a receiver with a message ID.  Multiple receivers may be
+ * associated with a single messasge ID.
+ *
+ * Return 0 on success, on failure.
+ **/
+int
+dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+              subsys_id_t sys, recv_fn_t fn)
+{
+  smartlist_grow(cfg->recv_by_msg, msg+1);
+  smartlist_t *receivers = smartlist_get(cfg->recv_by_msg, msg);
+  if (!receivers) {
+    receivers = smartlist_new();
+    smartlist_set(cfg->recv_by_msg, msg, receivers);
+  }
+
+  dispatch_rcv_t *rcv = tor_malloc(sizeof(dispatch_rcv_t));
+  rcv->sys = sys;
+  rcv->enabled = true;
+  rcv->fn = fn;
+  smartlist_add(receivers, (void*)rcv);
+  return 0;
+}
+
+/** Helper: release all storage held by <b>cfg</b>. */
+void
+dcfg_free_(dispatch_cfg_t *cfg)
+{
+  if (!cfg)
+    return;
+
+  SMARTLIST_FOREACH(cfg->type_by_msg, msg_type_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->chan_by_msg, channel_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->fns_by_type, dispatch_typefns_t *, f, tor_free(f));
+  smartlist_free(cfg->type_by_msg);
+  smartlist_free(cfg->chan_by_msg);
+  smartlist_free(cfg->fns_by_type);
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, receivers) {
+    if (!receivers)
+      continue;
+    SMARTLIST_FOREACH(receivers, dispatch_rcv_t *, rcv, tor_free(rcv));
+    smartlist_free(receivers);
+  } SMARTLIST_FOREACH_END(receivers);
+  smartlist_free(cfg->recv_by_msg);
+
+  tor_free(cfg);
+}
diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h
new file mode 100644
index 0000000000..a4f1948eac
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_CFG_H
+#define TOR_DISPATCH_CFG_H
+
+/**
+ * @file dispatch_cfg.h
+ * @brief Header for distpach_cfg.c
+ **/
+
+#include "lib/dispatch/msgtypes.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * A "dispatch_cfg" is the configuration used to set up a dispatcher.
+ * It is created and accessed with a set of dcfg_* functions, and then
+ * used with dispatcher_new() to make the dispatcher.
+ */
+typedef struct dispatch_cfg_t dispatch_cfg_t;
+
+dispatch_cfg_t *dcfg_new(void);
+
+int dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                      msg_type_id_t type);
+
+int dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                      channel_id_t chan);
+
+int dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                      const dispatch_typefns_t *fns);
+
+int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+                  subsys_id_t sys, recv_fn_t fn);
+
+/** Free a dispatch_cfg_t. */
+#define dcfg_free(cfg) \
+  FREE_AND_NULL(dispatch_cfg_t, dcfg_free_, (cfg))
+
+void dcfg_free_(dispatch_cfg_t *cfg);
+
+#ifdef DISPATCH_NEW_PRIVATE
+struct smartlist_t;
+STATIC int max_in_u16_sl(const struct smartlist_t *sl, int dflt);
+#endif
+
+#endif /* !defined(TOR_DISPATCH_CFG_H) */
diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h
new file mode 100644
index 0000000000..3c99adf2f7
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg_st.h
@@ -0,0 +1,33 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dispatch_cfg_st.h
+ * @brief Declarations for dispatch-configuration types.
+ **/
+
+#ifndef TOR_DISPATCH_CFG_ST_H
+#define TOR_DISPATCH_CFG_ST_H
+
+struct smartlist_t;
+
+/** Information needed to create a dispatcher, but in a less efficient, more
+ * mutable format.
+ *
+ * Nearly everybody should use the \refdir{lib/pubsub} module to configure
+ * dispatchers, instead of using this. */
+struct dispatch_cfg_t {
+  /** A list of msg_type_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *type_by_msg;
+  /** A list of channel_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *chan_by_msg;
+  /** A list of dispatch_rcv_t, indexed by msg_type_id_t. */
+  struct smartlist_t *fns_by_type;
+  /** A list of dispatch_typefns_t, indexed by msg_t. */
+  struct smartlist_t *recv_by_msg;
+};
+
+#endif /* !defined(TOR_DISPATCH_CFG_ST_H) */
diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c
new file mode 100644
index 0000000000..3d51c876a7
--- /dev/null
+++ b/src/lib/dispatch/dispatch_core.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_core.c
+ * \brief Core module for sending and receiving messages.
+ */
+
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_naming.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Use <b>d</b> to drop all storage held for <b>msg</b>.
+ *
+ * (We need the dispatcher so we know how to free the auxiliary data.)
+ **/
+void
+dispatch_free_msg_(const dispatch_t *d, msg_t *msg)
+{
+  if (!msg)
+    return;
+
+  d->typefns[msg->type].free_fn(msg->aux_data__);
+  tor_free(msg);
+}
+
+/**
+ * Format the auxiliary data held by msg.
+ **/
+char *
+dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg)
+{
+  if (!msg)
+    return NULL;
+
+  return d->typefns[msg->type].fmt_fn(msg->aux_data__);
+}
+
+/**
+ * Release all storage held by <b>d</b>.
+ **/
+void
+dispatch_free_(dispatch_t *d)
+{
+  if (d == NULL)
+    return;
+
+  size_t n_queues = d->n_queues;
+  for (size_t i = 0; i < n_queues; ++i) {
+    msg_t *m, *mtmp;
+    TOR_SIMPLEQ_FOREACH_SAFE(m, &d->queues[i].queue, next, mtmp) {
+      dispatch_free_msg(d, m);
+    }
+  }
+
+  size_t n_msgs = d->n_msgs;
+
+  for (size_t i = 0; i < n_msgs; ++i) {
+    tor_free(d->table[i]);
+  }
+  tor_free(d->table);
+  tor_free(d->typefns);
+  tor_free(d->queues);
+
+  // This is the only time we will treat d->cfg as non-const.
+  //dispatch_cfg_free_((dispatch_items_t *) d->cfg);
+
+  tor_free(d);
+}
+
+/**
+ * Tell the dispatcher to call <b>fn</b> with <b>userdata</b> whenever
+ * <b>chan</b> becomes nonempty.  Return 0 on success, -1 on error.
+ **/
+int
+dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                      dispatch_alertfn_t fn, void *userdata)
+{
+  if (BUG(chan >= d->n_queues))
+    return -1;
+
+  dqueue_t *q = &d->queues[chan];
+  q->alert_fn = fn;
+  q->alert_fn_arg = userdata;
+  return 0;
+}
+
+/**
+ * Send a message on the appropriate channel notifying that channel if
+ * necessary.
+ *
+ * This function takes ownership of the auxiliary data; it can't be static or
+ * stack-allocated, and the caller is not allowed to use it afterwards.
+ *
+ * This function does not check the various vields of the message object for
+ * consistency.
+ **/
+int
+dispatch_send(dispatch_t *d,
+              subsys_id_t sender,
+              channel_id_t channel,
+              message_id_t msg,
+              msg_type_id_t type,
+              msg_aux_data_t auxdata)
+{
+  if (!d->table[msg]) {
+    /* Fast path: nobody wants this data. */
+
+    d->typefns[type].free_fn(auxdata);
+    return 0;
+  }
+
+  msg_t *m = tor_malloc(sizeof(msg_t));
+
+  m->sender = sender;
+  m->channel = channel;
+  m->msg = msg;
+  m->type = type;
+  memcpy(&m->aux_data__, &auxdata, sizeof(msg_aux_data_t));
+
+  return dispatch_send_msg(d, m);
+}
+
+int
+dispatch_send_msg(dispatch_t *d, msg_t *m)
+{
+  if (BUG(!d))
+    goto err;
+  if (BUG(!m))
+    goto err;
+  if (BUG(m->channel >= d->n_queues))
+    goto err;
+  if (BUG(m->msg >= d->n_msgs))
+    goto err;
+
+  dtbl_entry_t *ent = d->table[m->msg];
+  if (ent) {
+    if (BUG(m->type != ent->type))
+      goto err;
+    if (BUG(m->channel != ent->channel))
+      goto err;
+  }
+
+  return dispatch_send_msg_unchecked(d, m);
+ err:
+  /* Probably it isn't safe to free m, since type could be wrong. */
+  return -1;
+}
+
+/**
+ * Send a message on the appropriate queue, notifying that queue if necessary.
+ *
+ * This function takes ownership of the message object and its auxiliary data;
+ * it can't be static or stack-allocated, and the caller isn't allowed to use
+ * it afterwards.
+ *
+ * This function does not check the various fields of the message object for
+ * consistency, and can crash if they are out of range.  Only functions that
+ * have already constructed the message in a safe way, or checked it for
+ * correctness themselves, should call this function.
+ **/
+int
+dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m)
+{
+  /* Find the right queue. */
+  dqueue_t *q = &d->queues[m->channel];
+  bool was_empty = TOR_SIMPLEQ_EMPTY(&q->queue);
+
+  /* Append the message. */
+  TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next);
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Queued: %s (%s) from %s, on %s.",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  /* If we just made the queue nonempty for the first time, call the alert
+   * function. */
+  if (was_empty) {
+    q->alert_fn(d, m->channel, q->alert_fn_arg);
+  }
+
+  return 0;
+}
+
+/**
+ * Run all of the callbacks on <b>d</b> associated with <b>m</b>.
+ **/
+static void
+dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m)
+{
+  tor_assert(m->msg <= d->n_msgs);
+  dtbl_entry_t *ent = d->table[m->msg];
+  int n_fns = ent->n_fns;
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Delivering: %s (%s) from %s, on %s:",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  int i;
+  for (i=0; i < n_fns; ++i) {
+    if (ent->rcv[i].enabled) {
+      log_debug(LD_MESG, "  Delivering to %s.",
+                get_subsys_id_name(ent->rcv[i].sys));
+      ent->rcv[i].fn(m);
+    }
+  }
+}
+
+/**
+ * Run up to <b>max_msgs</b> callbacks for messages on the channel <b>ch</b>
+ * on the given dispatcher.  Return 0 on success or recoverable failure,
+ * -1 on unrecoverable error.
+ **/
+int
+dispatch_flush(dispatch_t *d, channel_id_t ch, int max_msgs)
+{
+  if (BUG(ch >= d->n_queues))
+    return 0;
+
+  int n_flushed = 0;
+  dqueue_t *q = &d->queues[ch];
+
+  while (n_flushed < max_msgs) {
+    msg_t *m = TOR_SIMPLEQ_FIRST(&q->queue);
+    if (!m)
+      break;
+    TOR_SIMPLEQ_REMOVE_HEAD(&q->queue, next);
+    dispatcher_run_msg_cbs(d, m);
+    dispatch_free_msg(d, m);
+    ++n_flushed;
+  }
+
+  return 0;
+}
diff --git a/src/lib/dispatch/dispatch_naming.c b/src/lib/dispatch/dispatch_naming.c
new file mode 100644
index 0000000000..bb49343712
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dispatch_naming.c
+ * @brief Name-to-ID maps for our message dispatch system.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/cc/compat_compiler.h"
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+
+#include "lib/log/util_bug.h"
+#include "lib/log/log.h"
+
+#include <stdlib.h>
+
+/** Global namemap for message IDs. */
+static namemap_t message_id_map = NAMEMAP_INIT();
+/** Global namemap for subsystem IDs. */
+static namemap_t subsys_id_map = NAMEMAP_INIT();
+/** Global namemap for channel IDs. */
+static namemap_t channel_id_map = NAMEMAP_INIT();
+/** Global namemap for message type IDs. */
+static namemap_t msg_type_id_map = NAMEMAP_INIT();
+
+void
+dispatch_naming_init(void)
+{
+}
+
+#ifndef COCCI
+/* Helper macro: declare functions to map IDs to and from names for a given
+ * type in a namemap_t.
+ */
+#define DECLARE_ID_MAP_FNS(type)                                        \
+  type##_id_t                                                           \
+  get_##type##_id(const char *name)                                     \
+  {                                                                     \
+    unsigned u = namemap_get_or_create_id(&type##_id_map, name);        \
+    tor_assert(u != NAMEMAP_ERR);                                       \
+    tor_assert(u != ERROR_ID);                                          \
+    return (type##_id_t) u;                                             \
+  }                                                                     \
+  const char *                                                          \
+  get_##type##_id_name(type##_id_t id)                                  \
+  {                                                                     \
+    return namemap_fmt_name(&type##_id_map, id);                        \
+  }                                                                     \
+  size_t                                                                \
+  get_num_##type##_ids(void)                                            \
+  {                                                                     \
+    return namemap_get_size(&type##_id_map);                            \
+  }                                                                     \
+  EAT_SEMICOLON
+#endif /* !defined(COCCI) */
+
+DECLARE_ID_MAP_FNS(message);
+DECLARE_ID_MAP_FNS(channel);
+DECLARE_ID_MAP_FNS(subsys);
+DECLARE_ID_MAP_FNS(msg_type);
diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h
new file mode 100644
index 0000000000..72206d3ed5
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.h
@@ -0,0 +1,51 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dispatch_naming.h
+ * @brief Header for dispatch_naming.c
+ **/
+
+#ifndef TOR_DISPATCH_NAMING_H
+#define TOR_DISPATCH_NAMING_H
+
+#include "lib/dispatch/msgtypes.h"
+#include <stddef.h>
+
+/**
+ * Return an existing channel ID by name, allocating the channel ID if
+ * if necessary.  Returns ERROR_ID if we have run out of
+ * channels
+ */
+channel_id_t get_channel_id(const char *);
+/**
+ * Return the name corresponding to a given channel ID.
+ **/
+const char *get_channel_id_name(channel_id_t);
+/**
+ * Return the total number of _named_ channel IDs.
+ **/
+size_t get_num_channel_ids(void);
+
+/* As above, but for messages. */
+message_id_t get_message_id(const char *);
+const char *get_message_id_name(message_id_t);
+size_t get_num_message_ids(void);
+
+/* As above, but for subsystems */
+subsys_id_t get_subsys_id(const char *);
+const char *get_subsys_id_name(subsys_id_t);
+size_t get_num_subsys_ids(void);
+
+/* As above, but for types. Note that types additionally must be
+ * "defined", if any message is to use them. */
+msg_type_id_t get_msg_type_id(const char *);
+const char *get_msg_type_id_name(msg_type_id_t);
+size_t get_num_msg_type_ids(void);
+
+void dispatch_naming_init(void);
+
+#endif /* !defined(TOR_DISPATCH_NAMING_H) */
diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c
new file mode 100644
index 0000000000..e1dbb1c4b8
--- /dev/null
+++ b/src/lib/dispatch/dispatch_new.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_new.c
+ * \brief Code to construct a dispatch_t from a dispatch_cfg_t.
+ **/
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+
+#include "lib/cc/ctassert.h"
+#include "lib/intmath/cmp.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/** Given a smartlist full of (possibly NULL) pointers to uint16_t values,
+ * return the largest value, or dflt if the list is empty. */
+STATIC int
+max_in_u16_sl(const smartlist_t *sl, int dflt)
+{
+  uint16_t *maxptr = NULL;
+  SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) {
+    if (!maxptr)
+      maxptr = u;
+    else if (u && *u > *maxptr)
+      maxptr = u;
+  } SMARTLIST_FOREACH_END(u);
+
+  return maxptr ? *maxptr : dflt;
+}
+
+/* The above function is only safe to call if we are sure that channel_id_t
+ * and msg_type_id_t are really uint16_t.  They should be so defined in
+ * msgtypes.h, but let's be extra cautious.
+ */
+CTASSERT(sizeof(uint16_t) == sizeof(msg_type_id_t));
+CTASSERT(sizeof(uint16_t) == sizeof(channel_id_t));
+
+/** Helper: Format an unformattable message auxiliary data item: just return a
+* copy of the string <>. */
+static char *
+type_fmt_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+  return tor_strdup("<>");
+}
+
+/** Helper: Free an unfreeable message auxiliary data item: do nothing. */
+static void
+type_free_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+}
+
+/** Type functions to use when no type functions are provided. */
+static dispatch_typefns_t nop_typefns = {
+  .free_fn = type_free_nop,
+  .fmt_fn = type_fmt_nop
+};
+
+/**
+ * Alert function to use when none is configured: do nothing.
+ **/
+static void
+alert_fn_nop(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)d;
+  (void)ch;
+  (void)arg;
+}
+
+/**
+ * Given a list of recvfn_t, create and return a new dtbl_entry_t mapping
+ * to each of those functions.
+ **/
+static dtbl_entry_t *
+dtbl_entry_from_lst(smartlist_t *receivers)
+{
+  if (!receivers)
+    return NULL;
+
+  size_t n_recv = smartlist_len(receivers);
+  dtbl_entry_t *ent;
+  ent = tor_malloc_zero(offsetof(dtbl_entry_t, rcv) +
+                        sizeof(dispatch_rcv_t) * n_recv);
+
+  ent->n_fns = n_recv;
+
+  SMARTLIST_FOREACH_BEGIN(receivers, const dispatch_rcv_t *, rcv) {
+    memcpy(&ent->rcv[rcv_sl_idx], rcv, sizeof(*rcv));
+    if (rcv->enabled) {
+      ++ent->n_enabled;
+    }
+  } SMARTLIST_FOREACH_END(rcv);
+
+  return ent;
+}
+
+/** Create and return a new dispatcher from a given dispatch_cfg_t. */
+dispatch_t *
+dispatch_new(const dispatch_cfg_t *cfg)
+{
+  dispatch_t *d = tor_malloc_zero(sizeof(dispatch_t));
+
+  /* Any message that has a type or a receiver counts towards our messages */
+  const size_t n_msgs = MAX(smartlist_len(cfg->type_by_msg),
+                            smartlist_len(cfg->recv_by_msg)) + 1;
+
+  /* Any channel that any message has counts towards the number of channels. */
+  const size_t n_chans = (size_t)
+    MAX(1, max_in_u16_sl(cfg->chan_by_msg,0)) + 1;
+
+  /* Any type that a message has, or that has functions, counts towards
+   * the number of types. */
+  const size_t n_types = (size_t) MAX(max_in_u16_sl(cfg->type_by_msg,0),
+                                      smartlist_len(cfg->fns_by_type)) + 1;
+
+  d->n_msgs = n_msgs;
+  d->n_queues = n_chans;
+  d->n_types = n_types;
+
+  /* Initialize the array of type-functions. */
+  d->typefns = tor_calloc(n_types, sizeof(dispatch_typefns_t));
+  for (size_t i = 0; i < n_types; ++i) {
+    /* Default to no-op for everything... */
+    memcpy(&d->typefns[i], &nop_typefns, sizeof(dispatch_typefns_t));
+  }
+  SMARTLIST_FOREACH_BEGIN(cfg->fns_by_type, dispatch_typefns_t *, fns) {
+    /* Set the functions if they are provided. */
+    if (fns) {
+      if (fns->free_fn)
+        d->typefns[fns_sl_idx].free_fn = fns->free_fn;
+      if (fns->fmt_fn)
+        d->typefns[fns_sl_idx].fmt_fn = fns->fmt_fn;
+    }
+  } SMARTLIST_FOREACH_END(fns);
+
+  /* Initialize the message queues: one for each channel. */
+  d->queues = tor_calloc(d->n_queues, sizeof(dqueue_t));
+  for (size_t i = 0; i < d->n_queues; ++i) {
+    TOR_SIMPLEQ_INIT(&d->queues[i].queue);
+    d->queues[i].alert_fn = alert_fn_nop;
+  }
+
+  /* Build the dispatch tables mapping message IDs to receivers. */
+  d->table = tor_calloc(d->n_msgs, sizeof(dtbl_entry_t *));
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, rcv) {
+    d->table[rcv_sl_idx] = dtbl_entry_from_lst(rcv);
+  } SMARTLIST_FOREACH_END(rcv);
+
+  /* Fill in the empty entries in the dispatch tables:
+   * types and channels for each message. */
+  SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, msg_type_id_t *, type) {
+    if (d->table[type_sl_idx])
+      d->table[type_sl_idx]->type = *type;
+  } SMARTLIST_FOREACH_END(type);
+
+  SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, channel_id_t *, chan) {
+    if (d->table[chan_sl_idx])
+      d->table[chan_sl_idx]->channel = *chan;
+  } SMARTLIST_FOREACH_END(chan);
+
+  return d;
+}
diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h
new file mode 100644
index 0000000000..ad5b4efc40
--- /dev/null
+++ b/src/lib/dispatch/dispatch_st.h
@@ -0,0 +1,108 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_st.h
+ *
+ * \brief private structures used for the dispatcher module
+ */
+
+#ifndef TOR_DISPATCH_ST_H
+#define TOR_DISPATCH_ST_H
+
+#ifdef DISPATCH_PRIVATE
+
+#include "lib/container/smartlist.h"
+
+/**
+ * Information about the recipient of a message.
+ **/
+typedef struct dispatch_rcv_t {
+  /** The subsystem receiving a message. */
+  subsys_id_t sys;
+  /** True iff this recipient is enabled. */
+  bool enabled;
+  /** The function that will handle the message. */
+  recv_fn_t fn;
+} dispatch_rcv_t;
+
+/**
+ * Information used by a dispatcher to handle and dispatch a single message
+ * ID.  It maps that message ID to its type, channel, and list of receiver
+ * functions.
+ *
+ * This structure is used when the dispatcher is running.
+ **/
+typedef struct dtbl_entry_t {
+  /** The number of enabled non-stub subscribers for this message.
+   *
+   * Note that for now, this will be the same as <b>n_fns</b>, since there is
+   * no way to turn these subscribers on an off yet. */
+  uint16_t n_enabled;
+  /** The channel that handles this message. */
+  channel_id_t channel;
+  /** The associated C type for this message. */
+  msg_type_id_t type;
+  /**
+   * The number of functions pointers for subscribers that receive this
+   * message, in rcv. */
+  uint16_t n_fns;
+  /**
+   * The recipients for this message.
+   */
+  dispatch_rcv_t rcv[FLEXIBLE_ARRAY_MEMBER];
+} dtbl_entry_t;
+
+/**
+ * A queue of messages for a given channel, used by a live dispatcher.
+ */
+typedef struct dqueue_t {
+  /** The queue of messages itself. */
+  TOR_SIMPLEQ_HEAD( , msg_t) queue;
+  /** A function to be called when the queue becomes nonempty. */
+  dispatch_alertfn_t alert_fn;
+  /** An argument for the alert_fn. */
+  void *alert_fn_arg;
+} dqueue_t ;
+
+/**
+ * A single dispatcher for cross-module messages.
+ */
+struct dispatch_t {
+  /**
+   * The length of <b>table</b>: the number of message IDs that this
+   * dispatcher can handle.
+   */
+  size_t n_msgs;
+  /**
+   * The length of <b>queues</b>: the number of channels that this dispatcher
+   * has configured.
+   */
+  size_t n_queues;
+  /**
+   * The length of <b>typefns</b>: the number of C type IDs that this
+   * dispatcher has configured.
+   */
+  size_t n_types;
+  /**
+   * An array of message queues, indexed by channel ID.
+   */
+  dqueue_t *queues;
+  /**
+   * An array of entries about how to handle particular message types, indexed
+   * by message ID.
+   */
+  dtbl_entry_t **table;
+  /**
+   * An array of function tables for manipulating types, index by message
+   * type ID.
+   **/
+  dispatch_typefns_t *typefns;
+};
+
+#endif /* defined(DISPATCH_PRIVATE) */
+
+#endif /* !defined(TOR_DISPATCH_ST_H) */
diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am
new file mode 100644
index 0000000000..4a0e0dfd90
--- /dev/null
+++ b/src/lib/dispatch/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-dispatch.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_dispatch_a_SOURCES =			\
+	src/lib/dispatch/dispatch_cfg.c			\
+	src/lib/dispatch/dispatch_core.c		\
+	src/lib/dispatch/dispatch_naming.c		\
+	src/lib/dispatch/dispatch_new.c
+
+src_lib_libtor_dispatch_testing_a_SOURCES = \
+	$(src_lib_libtor_dispatch_a_SOURCES)
+src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/dispatch/dispatch.h			\
+	src/lib/dispatch/dispatch_cfg.h 		\
+	src/lib/dispatch/dispatch_cfg_st.h 		\
+	src/lib/dispatch/dispatch_naming.h		\
+	src/lib/dispatch/dispatch_st.h			\
+	src/lib/dispatch/msgtypes.h
diff --git a/src/lib/dispatch/lib_dispatch.md b/src/lib/dispatch/lib_dispatch.md
new file mode 100644
index 0000000000..153ca50080
--- /dev/null
+++ b/src/lib/dispatch/lib_dispatch.md
@@ -0,0 +1,14 @@
+@dir /lib/dispatch
+@brief lib/dispatch: In-process message delivery.
+
+This module provides a general in-process "message dispatch" system in which
+typed messages are sent on channels. The dispatch.h header has far more
+information.
+
+It is used by by \refdir{lib/pubsub} to implement our general
+inter-module publish/subscribe system.
+
+This is not a fancy multi-threaded many-to-many dispatcher as you may be used
+to from more sophisticated architectures: this dispatcher is intended only
+for use in improving Tor's architecture.
+
diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h
new file mode 100644
index 0000000000..01d969dcb5
--- /dev/null
+++ b/src/lib/dispatch/msgtypes.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file msgtypes.h
+ * \brief Types used for messages in the dispatcher code.
+ **/
+
+#ifndef TOR_DISPATCH_MSGTYPES_H
+#define TOR_DISPATCH_MSGTYPES_H
+
+#include <stdint.h>
+
+#include "ext/tor_queue.h"
+
+/**
+ * These types are aliases for subsystems, channels, and message IDs.
+ **/
+typedef uint16_t subsys_id_t;
+typedef uint16_t channel_id_t;
+typedef uint16_t message_id_t;
+
+/**
+ * This identifies a C type that can be sent along with a message.
+ **/
+typedef uint16_t msg_type_id_t;
+
+/**
+ * An ID value returned for *_type_t when none exists.
+ */
+#define ERROR_ID 65535
+
+/**
+ * Auxiliary (untyped) data sent along with a message.
+ *
+ * We define this as a union of a pointer and a u64, so that the integer
+ * types will have the same range across platforms.
+ **/
+typedef union {
+  void *ptr;
+  uint64_t u64;
+} msg_aux_data_t;
+
+/**
+ * Structure of a received message.
+ **/
+typedef struct msg_t {
+  TOR_SIMPLEQ_ENTRY(msg_t) next;
+  subsys_id_t sender;
+  channel_id_t channel;
+  message_id_t msg;
+  /** We could omit this field, since it is implicit in the message type, but
+   * IMO let's leave it in for safety. */
+  msg_type_id_t type;
+  /** Untyped auxiliary data. You shouldn't have to mess with this
+   * directly. */
+  msg_aux_data_t aux_data__;
+} msg_t;
+
+/**
+ * A function that a subscriber uses to receive a message.
+ **/
+typedef void (*recv_fn_t)(const msg_t *m);
+
+/**
+ * Table of functions to use for a given C type.  Any omitted (NULL) functions
+ * will be treated as no-ops.
+ **/
+typedef struct dispatch_typefns_t {
+  /** Release storage held for the auxiliary data of this type. */
+  void (*free_fn)(msg_aux_data_t);
+  /** Format and return a newly allocated string describing the contents
+   * of this data element. */
+  char *(*fmt_fn)(msg_aux_data_t);
+} dispatch_typefns_t;
+
+#endif /* !defined(TOR_DISPATCH_MSGTYPES_H) */
diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include
index 7c2ef36929..c9bf4b1786 100644
--- a/src/lib/encoding/.may_include
+++ b/src/lib/encoding/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/container/*.h
 lib/ctime/*.h
 lib/encoding/*.h
 lib/intmath/*.h
diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c
index bd063440d6..3e549eb8e3 100644
--- a/src/lib/encoding/binascii.c
+++ b/src/lib/encoding/binascii.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -84,7 +84,7 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
 }
 
 /** Implements base32 decoding as in RFC 4648.
- * Returns 0 if successful, -1 otherwise.
+ * Return the number of bytes decoded if successful; -1 otherwise.
  */
 int
 base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
@@ -147,7 +147,7 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
   memset(tmp, 0, srclen); /* on the heap, this should be safe */
   tor_free(tmp);
   tmp = NULL;
-  return 0;
+  return i;
 }
 
 #define BASE64_OPENSSL_LINELEN 64
@@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags)
   return enclen;
 }
 
+/** Return an upper bound on the number of bytes that might be needed to hold
+ * the data from decoding the base64 string <b>srclen</b>.  This is only an
+ * upper bound, since some part of the base64 string might be padding or
+ * space. */
+size_t
+base64_decode_maxsize(size_t srclen)
+{
+  tor_assert(srclen < INT_MAX / 3);
+
+  return CEIL_DIV(srclen * 3, 4);
+}
+
 /** Internal table mapping 6 bit values to the Base64 alphabet. */
 static const char base64_encode_table[64] = {
   'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
@@ -309,8 +321,10 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
   return (int) enclen;
 }
 
-/** As base64_encode, but do not add any internal spaces or external padding
- * to the output stream. */
+/** As base64_encode, but do not add any internal spaces, and remove external
+ * padding from the output stream.
+ * dest must be at least base64_encode_size(srclen, 0), including space for
+ * the removed external padding. */
 int
 base64_encode_nopad(char *dest, size_t destlen,
                     const uint8_t *src, size_t srclen)
diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h
index 7e3cc04f09..9cb03bab62 100644
--- a/src/lib/encoding/binascii.h
+++ b/src/lib/encoding/binascii.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen);
 
 #define BASE64_ENCODE_MULTILINE 1
 size_t base64_encode_size(size_t srclen, int flags);
+size_t base64_decode_maxsize(size_t srclen);
 int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
                   int flags);
 int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
@@ -57,4 +58,4 @@ size_t base32_encoded_size(size_t srclen);
 void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 
-#endif /* !defined(TOR_UTIL_FORMAT_H) */
+#endif /* !defined(TOR_BINASCII_H) */
diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c
index 8110f3dd9c..613e4a00c6 100644
--- a/src/lib/encoding/confline.c
+++ b/src/lib/encoding/confline.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -82,6 +82,19 @@ config_line_find(const config_line_t *lines,
   return NULL;
 }
 
+/** As config_line_find(), but perform a case-insensitive comparison. */
+const config_line_t *
+config_line_find_case(const config_line_t *lines,
+                      const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcasecmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
 /** Auxiliary function that does all the work of config_get_lines.
  * <b>recursion_level</b> is the count of how many nested %includes we have.
  * <b>opened_lst</b> will have a list of opened files if provided.
@@ -138,6 +151,8 @@ config_get_lines_aux(const char *string, config_line_t **result, int extended,
       if (allow_include && !strcmp(k, "%include") && handle_include) {
         tor_free(k);
         include_used = 1;
+        log_notice(LD_CONFIG, "Processing configuration path \"%s\" at "
+                   "recursion level %d.", v, recursion_level);
 
         config_line_t *include_list;
         if (handle_include(v, recursion_level, extended, &include_list,
@@ -148,9 +163,6 @@ config_get_lines_aux(const char *string, config_line_t **result, int extended,
           tor_free(v);
           return -1;
         }
-        log_notice(LD_CONFIG, "Included configuration file or "
-                   "directory at recursion level %d: \"%s\".",
-                   recursion_level, v);
         *next = include_list;
         if (list_last)
           next = &list_last->next;
@@ -240,10 +252,39 @@ config_lines_dup_and_filter(const config_line_t *inp,
   return result;
 }
 
+/**
+ * Given a linelist <b>inp</b> beginning with the key <b>header</b>, find the
+ * next line with that key, and remove that instance and all following lines
+ * from the list.  Return the lines that were removed.  Operate
+ * case-insensitively.
+ *
+ * For example, if the header is "H", and <b>inp</b> contains "H, A, B, H, C,
+ * H, D", this function will alter <b>inp</b> to contain only "H, A, B", and
+ * return the elements "H, C, H, D" as a separate list.
+ **/
+config_line_t *
+config_lines_partition(config_line_t *inp, const char *header)
+{
+  if (BUG(inp == NULL))
+    return NULL;
+  if (BUG(strcasecmp(inp->key, header)))
+    return NULL;
+
+  /* Advance ptr until it points to the link to the next segment of this
+     list. */
+  config_line_t **ptr = &inp->next;
+  while (*ptr && strcasecmp((*ptr)->key, header)) {
+    ptr = &(*ptr)->next;
+  }
+  config_line_t *remainder = *ptr;
+  *ptr = NULL;
+  return remainder;
+}
+
 /** Return true iff a and b contain identical keys and values in identical
  * order. */
 int
-config_lines_eq(config_line_t *a, config_line_t *b)
+config_lines_eq(const config_line_t *a, const config_line_t *b)
 {
   while (a && b) {
     if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h
index 3d9ae8a662..ce0d6c6e17 100644
--- a/src/lib/encoding/confline.h
+++ b/src/lib/encoding/confline.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,7 +48,10 @@ config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
                                            const char *key);
 const config_line_t *config_line_find(const config_line_t *lines,
                                       const char *key);
-int config_lines_eq(config_line_t *a, config_line_t *b);
+const config_line_t *config_line_find_case(const config_line_t *lines,
+                                           const char *key);
+config_line_t *config_lines_partition(config_line_t *inp, const char *header);
+int config_lines_eq(const config_line_t *a, const config_line_t *b);
 int config_count_key(const config_line_t *a, const char *key);
 void config_free_lines_(config_line_t *front);
 #define config_free_lines(front) \
diff --git a/src/lib/encoding/cstring.c b/src/lib/encoding/cstring.c
index 29d3714126..54c330fca3 100644
--- a/src/lib/encoding/cstring.c
+++ b/src/lib/encoding/cstring.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/cstring.h b/src/lib/encoding/cstring.h
index 904a2c9c1c..2a3f6d0fc4 100644
--- a/src/lib/encoding/cstring.h
+++ b/src/lib/encoding/cstring.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am
index 2d2aa3988a..48d0120bfc 100644
--- a/src/lib/encoding/include.am
+++ b/src/lib/encoding/include.am
@@ -4,12 +4,15 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-encoding-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_encoding_a_SOURCES =			\
 	src/lib/encoding/binascii.c			\
 	src/lib/encoding/confline.c			\
 	src/lib/encoding/cstring.c			\
 	src/lib/encoding/keyval.c			\
+	src/lib/encoding/kvline.c			\
 	src/lib/encoding/pem.c				\
+	src/lib/encoding/qstring.c			\
 	src/lib/encoding/time_fmt.c
 
 src_lib_libtor_encoding_testing_a_SOURCES = \
@@ -17,10 +20,13 @@ src_lib_libtor_encoding_testing_a_SOURCES = \
 src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/encoding/binascii.h			\
 	src/lib/encoding/confline.h			\
 	src/lib/encoding/cstring.h			\
 	src/lib/encoding/keyval.h			\
+	src/lib/encoding/kvline.h			\
 	src/lib/encoding/pem.h				\
+	src/lib/encoding/qstring.h			\
 	src/lib/encoding/time_fmt.h
diff --git a/src/lib/encoding/keyval.c b/src/lib/encoding/keyval.c
index c5da5a0bfc..0eb1219d43 100644
--- a/src/lib/encoding/keyval.c
+++ b/src/lib/encoding/keyval.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h
index cd327b7a82..b4966b01de 100644
--- a/src/lib/encoding/keyval.h
+++ b/src/lib/encoding/keyval.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,4 +14,4 @@
 
 int string_is_key_value(int severity, const char *string);
 
-#endif
+#endif /* !defined(TOR_KEYVAL_H) */
diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c
new file mode 100644
index 0000000000..5b220605d6
--- /dev/null
+++ b/src/lib/encoding/kvline.c
@@ -0,0 +1,297 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.c
+ *
+ * \brief Manipulating lines of key-value pairs.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
+
+/** Return true iff we need to quote and escape the string <b>s</b> to encode
+ * it.
+ *
+ * kvline_can_encode_lines() also uses this (with
+ * <b>as_keyless_val</b> true) to check whether a key would require
+ * quoting.
+ */
+static bool
+needs_escape(const char *s, bool as_keyless_val)
+{
+  if (as_keyless_val && *s == 0)
+    return true;
+  /* Keyless values containing '=' need to be escaped. */
+  if (as_keyless_val && strchr(s, '='))
+    return true;
+
+  for (; *s; ++s) {
+    if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) ||
+        *s == '\'' || *s == '\"') {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff the key in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_key(const config_line_t *line)
+{
+  return line->key == NULL || strlen(line->key) == 0;
+}
+
+/**
+ * Return true iff the value in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_val(const config_line_t *line)
+{
+  return line->value == NULL || strlen(line->value) == 0;
+}
+
+/**
+ * Return true iff the all the lines in <b>line</b> can be encoded
+ * using <b>flags</b>.
+ **/
+static bool
+kvline_can_encode_lines(const config_line_t *line, unsigned flags)
+{
+  for ( ; line; line = line->next) {
+    const bool keyless = line_has_no_key(line);
+    if (keyless && ! (flags & KV_OMIT_KEYS)) {
+      /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */
+      return false;
+    }
+
+    if (needs_escape(line->value, keyless) && ! (flags & (KV_QUOTED|KV_RAW))) {
+      /* If both KV_QUOTED and KV_RAW are false, we can't encode a
+         value that needs quotes. */
+      return false;
+    }
+    if (!keyless && needs_escape(line->key, true)) {
+      /* We can't handle keys that need quoting. */
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value'
+ * pairs, using the provided <b>flags</b> to encode it.  Return a newly
+ * allocated string on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then all values that contain
+ * spaces or unusual characters are escaped and quoted.  Otherwise, such
+ * values are not allowed.  Mutually exclusive with KV_RAW.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are
+ * allowed, and are encoded as 'Value'.  Otherwise, such pairs are not
+ * allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then an empty value is
+ * encoded as 'Key', not as 'Key=' or 'Key=""'.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * If KV_RAW is set in <b>flags</b>, then don't apply any quoting to
+ * the value, and assume that the caller has adequately quoted it.
+ * (The control protocol has some quirks that make this necessary.)
+ * Mutually exclusive with KV_QUOTED.
+ *
+ * KV_QUOTED_QSTRING is not supported.
+ */
+char *
+kvline_encode(const config_line_t *line,
+              unsigned flags)
+{
+  tor_assert(! (flags & KV_QUOTED_QSTRING));
+
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+  tor_assert((flags & (KV_QUOTED|KV_RAW)) != (KV_QUOTED|KV_RAW));
+
+  if (!kvline_can_encode_lines(line, flags))
+    return NULL;
+
+  smartlist_t *elements = smartlist_new();
+
+  for (; line; line = line->next) {
+
+    const char *k = "";
+    const char *eq = "=";
+    const char *v = "";
+    const bool keyless = line_has_no_key(line);
+    bool esc = needs_escape(line->value, keyless);
+    char *tmp = NULL;
+
+    if (! keyless) {
+      k = line->key;
+    } else {
+      eq = "";
+    }
+
+    if ((flags & KV_OMIT_VALS) && line_has_no_val(line)) {
+      eq = "";
+      v = "";
+    } else if (!(flags & KV_RAW) && esc) {
+      tmp = esc_for_log(line->value);
+      v = tmp;
+    } else {
+      v = line->value;
+    }
+
+    smartlist_add_asprintf(elements, "%s%s%s", k, eq, v);
+    tor_free(tmp);
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/**
+ * Decode a <b>line</b> containing a series of space-separated 'Key=Value'
+ * pairs, using the provided <b>flags</b> to decode it.  Return a newly
+ * allocated list of pairs on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are
+ * allowed and handled as C strings. Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are
+ * allowed.  Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then keys without values are
+ * allowed.  Otherwise, such keys are not allowed.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * If KV_QUOTED_QSTRING is set in <b>flags</b>, then double-quoted values
+ * are allowed and handled as QuotedStrings per qstring.c.  Do not add
+ * new users of this flag.
+ *
+ * KV_RAW is not supported.
+ */
+config_line_t *
+kvline_parse(const char *line, unsigned flags)
+{
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+  tor_assert(!(flags & KV_RAW));
+
+  const char *cp = line, *cplast = NULL;
+  const bool omit_keys = (flags & KV_OMIT_KEYS) != 0;
+  const bool omit_vals = (flags & KV_OMIT_VALS) != 0;
+  const bool quoted = (flags & (KV_QUOTED|KV_QUOTED_QSTRING)) != 0;
+  const bool c_quoted = (flags & (KV_QUOTED)) != 0;
+
+  config_line_t *result = NULL;
+  config_line_t **next_line = &result;
+
+  char *key = NULL;
+  char *val = NULL;
+
+  while (*cp) {
+    key = val = NULL;
+    /* skip all spaces */
+    {
+      size_t idx = strspn(cp, " \t\r\v\n");
+      cp += idx;
+    }
+    if (BUG(cp == cplast)) {
+      /* If we didn't parse anything since the last loop, this code is
+       * broken. */
+      goto err; // LCOV_EXCL_LINE
+    }
+    cplast = cp;
+    if (! *cp)
+      break; /* End of string; we're done. */
+
+    /* Possible formats are K=V, K="V", K, V, and "V", depending on flags. */
+
+    /* Find where the key ends */
+    if (*cp != '\"') {
+      size_t idx = strcspn(cp, " \t\r\v\n=");
+
+      if (cp[idx] == '=') {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx + 1;
+      } else if (omit_vals) {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx;
+        goto commit;
+      } else {
+        if (!omit_keys)
+          goto err;
+      }
+    }
+
+    if (*cp == '\"') {
+      /* The type is "V". */
+      if (!quoted)
+        goto err;
+      size_t len=0;
+      if (c_quoted) {
+        cp = unescape_string(cp, &val, &len);
+      } else {
+        cp = decode_qstring(cp, strlen(cp), &val, &len);
+      }
+      if (cp == NULL || len != strlen(val)) {
+        // The string contains a NUL or is badly coded.
+        goto err;
+      }
+    } else {
+      size_t idx = strcspn(cp, " \t\r\v\n");
+      val = tor_memdup_nulterm(cp, idx);
+      cp += idx;
+    }
+
+  commit:
+    if (key && strlen(key) == 0) {
+      /* We don't allow empty keys. */
+      goto err;
+    }
+
+    *next_line = tor_malloc_zero(sizeof(config_line_t));
+    (*next_line)->key = key ? key : tor_strdup("");
+    (*next_line)->value = val ? val : tor_strdup("");
+    next_line = &(*next_line)->next;
+    key = val = NULL;
+  }
+
+  if (! (flags & KV_QUOTED_QSTRING)) {
+    if (!kvline_can_encode_lines(result, flags)) {
+      goto err;
+    }
+  }
+  return result;
+
+ err:
+  tor_free(key);
+  tor_free(val);
+  config_free_lines(result);
+  return NULL;
+}
diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h
new file mode 100644
index 0000000000..34c52908e3
--- /dev/null
+++ b/src/lib/encoding/kvline.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.h
+ *
+ * \brief Header for kvline.c
+ **/
+
+#ifndef TOR_KVLINE_H
+#define TOR_KVLINE_H
+
+struct config_line_t;
+
+#define KV_QUOTED    (1u<<0)
+#define KV_OMIT_KEYS (1u<<1)
+#define KV_OMIT_VALS (1u<<2)
+#define KV_QUOTED_QSTRING (1u<<3)
+#define KV_RAW       (1u<<4)
+
+struct config_line_t *kvline_parse(const char *line, unsigned flags);
+char *kvline_encode(const struct config_line_t *line, unsigned flags);
+
+#endif /* !defined(TOR_KVLINE_H) */
diff --git a/src/lib/encoding/lib_encoding.md b/src/lib/encoding/lib_encoding.md
new file mode 100644
index 0000000000..66dd9d8caf
--- /dev/null
+++ b/src/lib/encoding/lib_encoding.md
@@ -0,0 +1,6 @@
+@dir /lib/encoding
+@brief lib/encoding: Encoding data in various forms, types, and transformations
+
+Here we have time formats (timefmt.c), quoted strings (qstring.c), C strings
+(string.c) base-16/32/64 (binascii.c), and more.
+
diff --git a/src/lib/encoding/pem.c b/src/lib/encoding/pem.c
index 51f37d0840..6c9f10e085 100644
--- a/src/lib/encoding/pem.c
+++ b/src/lib/encoding/pem.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,7 +42,7 @@ pem_encoded_size(size_t src_len, const char *objtype)
 
 /**
  * PEM-encode the <b>srclen</b>-byte object at <b>src</b> into the
- * <b>destlen<\b>-byte buffer at <b>dest</b>, tagging it with <b>objtype</b>.
+ * <b>destlen</b>-byte buffer at <b>dest</b>, tagging it with <b>objtype</b>.
  * Return 0 on success and -1 on failure.
  */
 int
diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h
index 0bbb06a794..027c31c315 100644
--- a/src/lib/encoding/pem.h
+++ b/src/lib/encoding/pem.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,4 +23,4 @@ int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen,
 int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen,
                const char *objtype);
 
-#endif
+#endif /* !defined(TOR_PEM_H) */
diff --git a/src/lib/encoding/qstring.c b/src/lib/encoding/qstring.c
new file mode 100644
index 0000000000..5a34924eab
--- /dev/null
+++ b/src/lib/encoding/qstring.c
@@ -0,0 +1,90 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.c
+ * \brief Implement QuotedString parsing.
+ *
+ * Note that this is only used for controller authentication; do not
+ * create new users for this.  Instead, prefer the cstring.c functions.
+ **/
+
+#include "orconfig.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
+ * QuotedString, return the length of that
+ * string (as encoded, including quotes).  Otherwise return -1. */
+static inline int
+get_qstring_length(const char *start, size_t in_len_max,
+                          int *chars_out)
+{
+  const char *cp, *end;
+  int chars = 0;
+
+  if (*start != '\"')
+    return -1;
+
+  cp = start+1;
+  end = start+in_len_max;
+
+  /* Calculate length. */
+  while (1) {
+    if (cp >= end) {
+      return -1; /* Too long. */
+    } else if (*cp == '\\') {
+      if (++cp == end)
+        return -1; /* Can't escape EOS. */
+      ++cp;
+      ++chars;
+    } else if (*cp == '\"') {
+      break;
+    } else {
+      ++cp;
+      ++chars;
+    }
+  }
+  if (chars_out)
+    *chars_out = chars;
+  return (int)(cp - start+1);
+}
+
+/** Given a pointer to a string starting at <b>start</b> containing
+ * <b>in_len_max</b> characters, decode a string beginning with one double
+ * quote, containing any number of non-quote characters or characters escaped
+ * with a backslash, and ending with a final double quote.  Place the resulting
+ * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
+ * store its length in <b>out_len</b>.  On success, return a pointer to the
+ * character immediately following the escaped string.  On failure, return
+ * NULL. */
+const char *
+decode_qstring(const char *start, size_t in_len_max,
+               char **out, size_t *out_len)
+{
+  const char *cp, *end;
+  char *outp;
+  int len, n_chars = 0;
+
+  len = get_qstring_length(start, in_len_max, &n_chars);
+  if (len<0)
+    return NULL;
+
+  end = start+len-1; /* Index of last quote. */
+  tor_assert(*end == '\"');
+  outp = *out = tor_malloc(len+1);
+  *out_len = n_chars;
+
+  cp = start+1;
+  while (cp < end) {
+    if (*cp == '\\')
+      ++cp;
+    *outp++ = *cp++;
+  }
+  *outp = '\0';
+  tor_assert((outp - *out) == (int)*out_len);
+
+  return end+1;
+}
diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h
new file mode 100644
index 0000000000..f19a7dad87
--- /dev/null
+++ b/src/lib/encoding/qstring.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.h
+ * \brief Header for qstring.c
+ */
+
+#ifndef TOR_ENCODING_QSTRING_H
+#define TOR_ENCODING_QSTRING_H
+
+#include <stddef.h>
+
+const char *decode_qstring(const char *start, size_t in_len_max,
+                           char **out, size_t *out_len);
+
+#endif /* !defined(TOR_ENCODING_QSTRING_H) */
diff --git a/src/lib/encoding/time_fmt.c b/src/lib/encoding/time_fmt.c
index 40543d41e0..573dfaad82 100644
--- a/src/lib/encoding/time_fmt.c
+++ b/src/lib/encoding/time_fmt.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h
index 0ddeca57fc..80e47c5332 100644
--- a/src/lib/encoding/time_fmt.h
+++ b/src/lib/encoding/time_fmt.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -41,4 +41,4 @@ int parse_iso_time_nospace(const char *cp, time_t *t);
 int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
-#endif
+#endif /* !defined(TOR_TIME_FMT_H) */
diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include
index 48cc0ef088..314424545e 100644
--- a/src/lib/err/.may_include
+++ b/src/lib/err/.may_include
@@ -1,3 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/defs/*.h
 lib/err/*.h
+lib/subsys/*.h
+lib/version/*.h
diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c
index 8606f42177..afb6b9503f 100644
--- a/src/lib/err/backtrace.c
+++ b/src/lib/err/backtrace.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -52,12 +52,14 @@
 #include <pthread.h>
 #endif
 
-#define EXPOSE_CLEAN_BACKTRACE
+#include "lib/cc/ctassert.h"
+
+#define BACKTRACE_PRIVATE
 #include "lib/err/backtrace.h"
-#include "lib/err/torerr.h"
 
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
-  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
+  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \
+  defined(HAVE_PTHREAD_H)
 #define USE_BACKTRACE
 #endif
 
@@ -72,15 +74,40 @@
 static char bt_version[128] = "";
 
 #ifdef USE_BACKTRACE
+
 /** Largest stack depth to try to dump. */
 #define MAX_DEPTH 256
-/** Static allocation of stack to dump. This is static so we avoid stack
- * pressure. */
-static void *cb_buf[MAX_DEPTH];
+/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */
+#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *))
 /** Protects cb_buf from concurrent access. Pthreads, since this code
  * is Unix-only, and since this code needs to be lowest-level. */
 static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+/** Lock and return a static stack pointer buffer that can hold up to
+ *  MAX_DEPTH function pointers. */
+static void **
+lock_cb_buf(void)
+{
+  /* Lock the mutex first, before even declaring the buffer. */
+  pthread_mutex_lock(&cb_buf_mutex);
+
+  /** Static allocation of stack to dump. This is static so we avoid stack
+   * pressure. */
+  static void *cb_buf[MAX_DEPTH];
+  CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf));
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+
+  return cb_buf;
+}
+
+/** Unlock the static stack pointer buffer. */
+static void
+unlock_cb_buf(void **cb_buf)
+{
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+  pthread_mutex_unlock(&cb_buf_mutex);
+}
+
 /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will
  * log the correct function from which a signal was received with context
  * <b>ctx</b>.  (When we get a signal, the current function will not have
@@ -104,7 +131,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
     return;
 
   stack[n] = (void*) ctx->PC_FROM_UCONTEXT;
-#else /* !(defined(PC_FROM_UCONTEXT)) */
+#else /* !defined(PC_FROM_UCONTEXT) */
   (void) depth;
   (void) ctx;
   (void) stack;
@@ -115,14 +142,14 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
  * that with a backtrace log.  Send messages via the tor_log function at
  * logger". */
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   size_t depth;
   char **symbols;
   size_t i;
 
-  pthread_mutex_lock(&cb_buf_mutex);
+  void **cb_buf = lock_cb_buf();
 
   depth = backtrace(cb_buf, MAX_DEPTH);
   symbols = backtrace_symbols(cb_buf, (int)depth);
@@ -140,7 +167,7 @@ log_backtrace_impl(int severity, int domain, const char *msg,
   raw_free(symbols);
 
  done:
-  pthread_mutex_unlock(&cb_buf_mutex);
+  unlock_cb_buf(cb_buf);
 }
 
 static void crash_handler(int sig, siginfo_t *si, void *ctx_)
@@ -156,6 +183,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   int n_fds, i;
   const int *fds = NULL;
 
+  void **cb_buf = lock_cb_buf();
+
   (void) si;
 
   depth = backtrace(cb_buf, MAX_DEPTH);
@@ -172,7 +201,9 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
 
-  abort();
+  unlock_cb_buf(cb_buf);
+
+  tor_raw_abort_();
 }
 
 /** Write a backtrace to all of the emergency-error fds. */
@@ -183,20 +214,26 @@ dump_stack_symbols_to_error_fds(void)
   const int *fds = NULL;
   size_t depth;
 
+  void **cb_buf = lock_cb_buf();
+
   depth = backtrace(cb_buf, MAX_DEPTH);
 
   n_fds = tor_log_get_sigsafe_err_fds(&fds);
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
+
+  unlock_cb_buf(cb_buf);
 }
 
+/* The signals that we want our backtrace handler to trap */
+static int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
+  SIGIO, -1 };
+
 /** Install signal handlers as needed so that when we crash, we produce a
  * useful stack trace. Return 0 on success, -errno on failure. */
 static int
 install_bt_handler(void)
 {
-  int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
-                         SIGIO, -1 };
   int i, rv=0;
 
   struct sigaction sa;
@@ -219,10 +256,12 @@ install_bt_handler(void)
      * libc has pre-loaded the symbols we need to dump things, so that later
      * reads won't be denied by the sandbox code */
     char **symbols;
+    void **cb_buf = lock_cb_buf();
     size_t depth = backtrace(cb_buf, MAX_DEPTH);
     symbols = backtrace_symbols(cb_buf, (int) depth);
     if (symbols)
       raw_free(symbols);
+    unlock_cb_buf(cb_buf);
   }
 
   return rv;
@@ -232,12 +271,29 @@ install_bt_handler(void)
 static void
 remove_bt_handler(void)
 {
+  int i;
+
+  struct sigaction sa;
+
+  memset(&sa, 0, sizeof(sa));
+  sa.sa_handler = SIG_DFL;
+  sigfillset(&sa.sa_mask);
+
+  for (i = 0; trap_signals[i] >= 0; ++i) {
+    /* remove_bt_handler() is called on shutdown, from low-level code.
+     * It's not a fatal error, so we just ignore it. */
+    (void)sigaction(trap_signals[i], &sa, NULL);
+  }
+
+  /* cb_buf_mutex is statically initialised, so we can not destroy it.
+   * If we destroy it, and then re-initialise tor, all our backtraces will
+   * fail. */
 }
 #endif /* defined(USE_BACKTRACE) */
 
 #ifdef NO_BACKTRACE_IMPL
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   logger(severity, domain, "%s: %s. (Stack trace not available)",
diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h
index 48b41fca02..d02e6960b5 100644
--- a/src/lib/err/backtrace.h
+++ b/src/lib/err/backtrace.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BACKTRACE_H
@@ -12,11 +12,14 @@
 
 #include "orconfig.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/defs/logging_types.h"
 
-typedef void (*tor_log_fn)(int, unsigned, const char *fmt, ...)
+typedef void (*tor_log_fn)(int, log_domain_mask_t, const char *fmt, ...)
   CHECK_PRINTF(3,4);
 
-void log_backtrace_impl(int severity, int domain, const char *msg,
+void log_backtrace_impl(int severity, log_domain_mask_t domain,
+                        const char *msg,
                         tor_log_fn logger);
 int configure_backtrace_handler(const char *tor_version);
 void clean_up_backtrace_handler(void);
@@ -26,11 +29,11 @@ const char *get_tor_backtrace_version(void);
 #define log_backtrace(sev, dom, msg) \
   log_backtrace_impl((sev), (dom), (msg), tor_log)
 
-#ifdef EXPOSE_CLEAN_BACKTRACE
+#ifdef BACKTRACE_PRIVATE
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
 void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx);
 #endif
-#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */
+#endif /* defined(BACKTRACE_PRIVATE) */
 
 #endif /* !defined(TOR_BACKTRACE_H) */
diff --git a/src/lib/err/include.am b/src/lib/err/include.am
index f2a409c51e..883ac91511 100644
--- a/src/lib/err/include.am
+++ b/src/lib/err/include.am
@@ -5,15 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-err-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_err_a_SOURCES =			\
-	src/lib/err/backtrace.c				\
-	src/lib/err/torerr.c
+	src/lib/err/backtrace.c			\
+	src/lib/err/torerr.c			\
+	src/lib/err/torerr_sys.c
 
 src_lib_libtor_err_testing_a_SOURCES = \
 	$(src_lib_libtor_err_a_SOURCES)
 src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/err/backtrace.h				\
-	src/lib/err/torerr.h
+	src/lib/err/torerr.h				\
+	src/lib/err/torerr_sys.h
diff --git a/src/lib/err/lib_err.md b/src/lib/err/lib_err.md
new file mode 100644
index 0000000000..cb4eba2e0d
--- /dev/null
+++ b/src/lib/err/lib_err.md
@@ -0,0 +1,13 @@
+@dir /lib/err
+@brief lib/err: Lowest-level error handling code.
+
+This module is responsible for generating stack traces, handling raw
+assertion failures, and otherwise reporting problems that might not be
+safe to report via the regular logging module.
+
+There are three kinds of users for the functions in this module:
+  * Code that needs a way to assert(), but which cannot use the regular
+    `tor_assert()` macros in logging module.
+  * Code that needs signal-safe error reporting.
+  * Higher-level error handling code.
+
diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c
index 6b5224273a..2de75c0be4 100644
--- a/src/lib/err/torerr.c
+++ b/src/lib/err/torerr.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -110,6 +110,14 @@ tor_log_get_sigsafe_err_fds(const int **out)
  * Update the list of fds that get errors from inside a signal handler or
  * other emergency condition. Ignore any beyond the first
  * TOR_SIGSAFE_LOG_MAX_FDS.
+ *
+ * These fds must remain open even after the log module has shut down. (And
+ * they should remain open even while logs are being reconfigured.) Therefore,
+ * any fds closed by the log module should be dup()ed, and the duplicate fd
+ * should be given to the err module in fds. In particular, the log module
+ * closes the file log fds, but does not close the stdio log fds.
+ *
+ * If fds is NULL or n is 0, clears the list of error fds.
  */
 void
 tor_log_set_sigsafe_err_fds(const int *fds, int n)
@@ -118,8 +126,52 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n)
     n = TOR_SIGSAFE_LOG_MAX_FDS;
   }
 
-  memcpy(sigsafe_log_fds, fds, n * sizeof(int));
-  n_sigsafe_log_fds = n;
+  /* Clear the entire array. This code mitigates against some race conditions,
+   * but there are still some races here:
+   * - err logs are disabled while the array is cleared, and
+   * - a thread can read the old value of n_sigsafe_log_fds, then read a
+   *   partially written array.
+   * We could fix these races using atomics, but atomics use the err module. */
+  n_sigsafe_log_fds = 0;
+  memset(sigsafe_log_fds, 0, sizeof(sigsafe_log_fds));
+  if (fds && n > 0) {
+    memcpy(sigsafe_log_fds, fds, n * sizeof(int));
+    n_sigsafe_log_fds = n;
+  }
+}
+
+/**
+ * Reset the list of emergency error fds to its default.
+ */
+void
+tor_log_reset_sigsafe_err_fds(void)
+{
+  int fds[] = { STDERR_FILENO };
+  tor_log_set_sigsafe_err_fds(fds, 1);
+}
+
+/**
+ * Flush the list of fds that get errors from inside a signal handler or
+ * other emergency condition. These fds are shared with the logging code:
+ * flushing them also flushes the log buffers.
+ *
+ * This function is safe to call during signal handlers.
+ */
+void
+tor_log_flush_sigsafe_err_fds(void)
+{
+  /* If we don't have fsync() in unistd.h, we can't flush the logs. */
+#ifdef HAVE_FSYNC
+  int n_fds, i;
+  const int *fds = NULL;
+
+  n_fds = tor_log_get_sigsafe_err_fds(&fds);
+  for (i = 0; i < n_fds; ++i) {
+    /* This function is called on error and on shutdown, so we don't log, or
+     * take any other action, if fsync() fails. */
+    (void)fsync(fds[i]);
+  }
+#endif /* defined(HAVE_FSYNC) */
 }
 
 /**
@@ -161,6 +213,18 @@ tor_raw_assertion_failed_msg_(const char *file, int line, const char *expr,
   tor_log_err_sigsafe_write("\n");
 }
 
+/**
+ * Call the abort() function to kill the current process with a fatal
+ * error. But first, flush the raw error file descriptors, so error messages
+ * are written before process termination.
+ **/
+void
+tor_raw_abort_(void)
+{
+  tor_log_flush_sigsafe_err_fds();
+  abort();
+}
+
 /* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument
  * in range 2..16 inclusive. */
 static int
@@ -195,7 +259,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
     unsigned digit = (unsigned) (x % radix);
     if (cp <= buf) {
       /* Not tor_assert(); see above. */
-      abort();
+      tor_raw_abort_();
     }
     --cp;
     *cp = "0123456789ABCDEF"[digit];
@@ -204,7 +268,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
 
   /* NOT tor_assert; see above. */
   if (cp != buf) {
-    abort(); // LCOV_EXCL_LINE
+    tor_raw_abort_(); // LCOV_EXCL_LINE
   }
 
   return len;
@@ -224,8 +288,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
  * does not guarantee that an int is wider than a char (an int must be at
  * least 16 bits but it is permitted for a char to be that wide as well), we
  * can't assume a signed int is sufficient to accommodate an unsigned char.
- * Thus, format_helper_exit_status() will still need to emit any require '-'
- * on its own.
+ * Thus, callers will still need to add any required '-' to the final string.
  *
  * For most purposes, you'd want to use tor_snprintf("%x") instead of this
  * function; it's designed to be used in code paths where you can't call
diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h
index 6ae91fbe85..ce1b049c47 100644
--- a/src/lib/err/torerr.h
+++ b/src/lib/err/torerr.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,13 +20,13 @@
 #define raw_assert(expr) STMT_BEGIN                                     \
     if (!(expr)) {                                                      \
       tor_raw_assertion_failed_msg_(__FILE__, __LINE__, #expr, NULL);   \
-      abort();                                                          \
+      tor_raw_abort_();                                                 \
     }                                                                   \
   STMT_END
 #define raw_assert_unreached(expr) raw_assert(0)
 #define raw_assert_unreached_msg(msg) STMT_BEGIN                    \
     tor_raw_assertion_failed_msg_(__FILE__, __LINE__, "0", (msg));  \
-    abort();                                                        \
+    tor_raw_abort_();                                               \
   STMT_END
 
 void tor_raw_assertion_failed_msg_(const char *file, int line,
@@ -39,9 +39,13 @@ void tor_raw_assertion_failed_msg_(const char *file, int line,
 void tor_log_err_sigsafe(const char *m, ...);
 int tor_log_get_sigsafe_err_fds(const int **out);
 void tor_log_set_sigsafe_err_fds(const int *fds, int n);
+void tor_log_reset_sigsafe_err_fds(void);
+void tor_log_flush_sigsafe_err_fds(void);
 void tor_log_sigsafe_err_set_granularity(int ms);
 
+void tor_raw_abort_(void) ATTR_NORETURN;
+
 int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len);
 int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_TORERR_H) */
diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c
new file mode 100644
index 0000000000..8ee1521f3b
--- /dev/null
+++ b/src/lib/err/torerr_sys.c
@@ -0,0 +1,45 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.c
+ * \brief Subsystem object for the error handling subsystem.
+ **/
+
+#include "orconfig.h"
+#include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/subsys/subsys.h"
+#include "lib/version/torversion.h"
+
+#include <stddef.h>
+
+static int
+subsys_torerr_initialize(void)
+{
+  if (configure_backtrace_handler(get_version()) < 0)
+    return -1;
+  tor_log_reset_sigsafe_err_fds();
+
+  return 0;
+}
+static void
+subsys_torerr_shutdown(void)
+{
+  /* Stop handling signals with backtraces, then flush the logs. */
+  clean_up_backtrace_handler();
+  tor_log_flush_sigsafe_err_fds();
+}
+
+const subsys_fns_t sys_torerr = {
+  .name = "err",
+  SUBSYS_DECLARE_LOCATION(),
+  /* Low-level error handling is a diagnostic feature, we want it to init
+   * right after windows process security, and shutdown last.
+   * (Security never shuts down.) */
+  .level = -99,
+  .supported = true,
+  .initialize = subsys_torerr_initialize,
+  .shutdown = subsys_torerr_shutdown
+};
diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h
new file mode 100644
index 0000000000..b86ccd2790
--- /dev/null
+++ b/src/lib/err/torerr_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.h
+ * \brief Declare subsystem object for torerr.c
+ **/
+
+#ifndef TOR_TORERR_SYS_H
+#define TOR_TORERR_SYS_H
+
+extern const struct subsys_fns_t sys_torerr;
+
+#endif /* !defined(TOR_TORERR_SYS_H) */
diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include
index 30af508914..54aa75fbff 100644
--- a/src/lib/evloop/.may_include
+++ b/src/lib/evloop/.may_include
@@ -8,9 +8,10 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/time/*.h
 
-src/ext/timeouts/timeout.c
-tor_queue.h
\ No newline at end of file
+ext/timeouts/timeout.c
+ext/tor_queue.h
\ No newline at end of file
diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c
index 939d77f857..0fd247d331 100644
--- a/src/lib/evloop/compat_libevent.c
+++ b/src/lib/evloop/compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2019, The Tor Project, Inc. */
+/* Copyright (c) 2009-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -130,7 +130,7 @@ rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg)
 
 /** Initialize the Libevent library and set up the event base. */
 void
-tor_libevent_initialize(tor_libevent_cfg *torcfg)
+tor_libevent_initialize(tor_libevent_cfg_t *torcfg)
 {
   tor_assert(the_event_base == NULL);
   /* some paths below don't use torcfg, so avoid unused variable warnings */
@@ -432,7 +432,7 @@ mainloop_event_activate(mainloop_event_t *event)
  *
  * If the event is scheduled for a different time, cancel it and run
  * after this delay instead.  If the event is currently pending to run
- * <em>now</b>, has no effect.
+ * <b>now</b>, has no effect.
  *
  * Do not call this function with <b>tv</b> == NULL -- use
  * mainloop_event_activate() instead.
diff --git a/src/lib/evloop/compat_libevent.h b/src/lib/evloop/compat_libevent.h
index 92724c369c..277ba3add6 100644
--- a/src/lib/evloop/compat_libevent.h
+++ b/src/lib/evloop/compat_libevent.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2019, The Tor Project, Inc. */
+/* Copyright (c) 2009-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -61,15 +61,15 @@ void mainloop_event_free_(mainloop_event_t *event);
 
 /** Defines a configuration for using libevent with Tor: passed as an argument
  * to tor_libevent_initialize() to describe how we want to set up. */
-typedef struct tor_libevent_cfg {
+typedef struct tor_libevent_cfg_t {
   /** How many CPUs should we use (not currently useful). */
   int num_cpus;
   /** How many milliseconds should we allow between updating bandwidth limits?
    * (Not currently useful). */
   int msec_per_tick;
-} tor_libevent_cfg;
+} tor_libevent_cfg_t;
 
-void tor_libevent_initialize(tor_libevent_cfg *cfg);
+void tor_libevent_initialize(tor_libevent_cfg_t *cfg);
 bool tor_libevent_is_initialized(void);
 MOCK_DECL(struct event_base *, tor_libevent_get_base, (void));
 const char *tor_libevent_get_method(void);
diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c
new file mode 100644
index 0000000000..b639810c23
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.c
+ * @brief Subsystem definition for the event loop module
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/evloop_sys.h"
+#include "lib/log/log.h"
+
+static int
+subsys_evloop_initialize(void)
+{
+  if (tor_init_libevent_rng() < 0) {
+    log_warn(LD_NET, "Problem initializing libevent RNG.");
+    return -1;
+  }
+  return 0;
+}
+
+static void
+subsys_evloop_postfork(void)
+{
+#ifdef TOR_UNIT_TESTS
+  tor_libevent_postfork();
+#endif
+}
+
+static void
+subsys_evloop_shutdown(void)
+{
+  tor_libevent_free_all();
+}
+
+const struct subsys_fns_t sys_evloop = {
+  .name = "evloop",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -20,
+  .initialize = subsys_evloop_initialize,
+  .shutdown = subsys_evloop_shutdown,
+  .postfork = subsys_evloop_postfork,
+};
diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h
new file mode 100644
index 0000000000..a37440e7a6
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.h
+ * @brief Declare subsystem object for the event loop module.
+ **/
+
+#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H
+#define TOR_LIB_EVLOOP_EVLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_evloop;
+
+#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */
diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am
index 6b0076272a..41cd2f45c5 100644
--- a/src/lib/evloop/include.am
+++ b/src/lib/evloop/include.am
@@ -5,21 +5,24 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-evloop-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_evloop_a_SOURCES =			\
 	src/lib/evloop/compat_libevent.c		\
+	src/lib/evloop/evloop_sys.c			\
 	src/lib/evloop/procmon.c			\
 	src/lib/evloop/timers.c				\
 	src/lib/evloop/token_bucket.c			\
 	src/lib/evloop/workqueue.c
 
-
 src_lib_libtor_evloop_testing_a_SOURCES = \
 	$(src_lib_libtor_evloop_a_SOURCES)
 src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/evloop/compat_libevent.h		\
+	src/lib/evloop/evloop_sys.h			\
 	src/lib/evloop/procmon.h			\
 	src/lib/evloop/timers.h				\
 	src/lib/evloop/token_bucket.h			\
diff --git a/src/lib/evloop/lib_evloop.md b/src/lib/evloop/lib_evloop.md
new file mode 100644
index 0000000000..830be88148
--- /dev/null
+++ b/src/lib/evloop/lib_evloop.md
@@ -0,0 +1,7 @@
+@dir /lib/evloop
+@brief lib/evloop: Low-level event loop.
+
+This modules has tools to manage the [libevent](https://libevent.org/) event
+loop and related functionality, in order to implement asynchronous
+networking, timers, periodic events, and other scheduling tasks.
+
diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c
index 52469fa5fc..718c7d4777 100644
--- a/src/lib/evloop/procmon.c
+++ b/src/lib/evloop/procmon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -165,8 +165,8 @@ tor_validate_process_specifier(const char *process_spec,
   return parse_process_specifier(process_spec, &ppspec, msg);
 }
 
-/* DOCDOC poll_interval_tv */
-static const struct timeval poll_interval_tv = {15, 0};
+/* We check this often for presence of owning controller process. */
+static const struct timeval poll_interval_tv = {15, 0}; // 15 seconds.
 
 /** Create a process-termination monitor for the process specifier
  * given in <b>process_spec</b>.  Return a newly allocated
@@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_)
       tor_free(errmsg);
     }
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   /* Unix makes this part easy, if a bit racy. */
   its_dead_jim = kill(procmon->pid, 0);
   its_dead_jim = its_dead_jim && (errno == ESRCH);
diff --git a/src/lib/evloop/procmon.h b/src/lib/evloop/procmon.h
index 6caae5be86..28f443da18 100644
--- a/src/lib/evloop/procmon.h
+++ b/src/lib/evloop/procmon.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/evloop/time_periodic.md b/src/lib/evloop/time_periodic.md
new file mode 100644
index 0000000000..8b3589d9db
--- /dev/null
+++ b/src/lib/evloop/time_periodic.md
@@ -0,0 +1,76 @@
+
+@page time_periodic Time and periodic events in Tor
+
+### What time is it? ###
+
+We have several notions of the current time in Tor.
+
+The *wallclock time* is available from time(NULL) with
+second-granularity and tor_gettimeofday() with microsecond
+granularity.  It corresponds most closely to "the current time and date".
+
+The *monotonic time* is available with the set of monotime_\*
+functions declared in compat_time.h.  Unlike the wallclock time, it
+can only move forward. It does not necessarily correspond to a real
+world time, and it is not portable between systems.
+
+The *coarse monotonic time* is available from the set of
+monotime_coarse_\* functions in compat_time.h. It is the same as
+monotime_\* on some platforms. On others, it gives a monotonic timer
+with less precision, but which it's more efficient to access.
+
+### Cached views of time. ###
+
+On some systems (like Linux), many time functions use a VDSO to avoid
+the overhead of a system call.  But on other systems, gettimeofday()
+and time() can be costly enough that you wouldn't want to call them
+tens of thousands of times.  To get a recent, but not especially
+accurate, view of the current time, see approx_time() and
+tor_gettimeofday_cached().
+
+
+### Parsing and encoding time values ###
+
+Tor has functions to parse and format time in these formats:
+
+ - RFC1123 format. ("Fri, 29 Sep 2006 15:54:20 GMT").  For this,
+   use format_rfc1123_time() and parse_rfc1123_time.
+
+ - ISO8601 format. ("2006-10-29 10:57:20") For this, use
+   format_local_iso_time() and format_iso_time().  We also support the
+   variant format "2006-10-29T10:57:20" with format_iso_time_nospace(), and
+   "2006-10-29T10:57:20.123456" with format_iso_time_nospace_usec().
+
+ - HTTP format collections (preferably "Mon, 25 Jul 2016 04:01:11
+   GMT" or possibly "Wed Jun 30 21:49:08 1993" or even "25-Jul-16
+   04:01:11 GMT"). For this, use parse_http_time().  Don't generate anything
+   but the first format.
+
+Some of these functions use struct tm. You can use the standard
+tor_localtime_r() and tor_gmtime_r() to wrap these in a safe way. We
+also have a tor_timegm() function.
+
+### Scheduling events ###
+
+The main way to schedule a not-too-frequent periodic event with
+respect to the Tor mainloop is via the mechanism in periodic.c.
+There's a big table of periodic_events in mainloop.c, each of which gets
+invoked on its own schedule.  You should not expect more than about
+one second of accuracy with these timers.
+
+You can create an independent timer using libevent directly, or using
+the periodic_timer_new() function.  But you should avoid doing this
+for per-connection or per-circuit timers: Libevent's internal timer
+implementation uses a min-heap, and those tend to start scaling poorly
+once you have a few thousand entries.
+
+If you need to create a large number of fine-grained timers for some
+purpose, you should consider the mechanism in src/common/timers.c,
+which is optimized for the case where you have a large number of
+timers with not-too-long duration, many of which will be deleted
+before they actually expire. These timers should be reasonably
+accurate within a handful of milliseconds -- possibly better on some
+platforms.  (The timers.c module uses William Ahern's timeout.c
+implementation as its backend, which is based on a hierarchical timing
+wheel algorithm. It's cool stuff; check it out.)
+
diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c
index e46d2635a8..11418e93fd 100644
--- a/src/lib/evloop/timers.c
+++ b/src/lib/evloop/timers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
  * The main advantage of tor_timer_t over using libevent's timers is that
  * they're way more efficient if we need to have thousands or millions of
  * them.  For more information, see
- *   http://www.25thandclement.com/~william/projects/timeout.c.html
+ *   https://www.25thandclement.com/~william/projects/timeout.c.html
  *
  * Periodic timers are available in the backend, but I've turned them off.
  * We can turn them back on if needed.
@@ -48,7 +48,7 @@
 #include <winsock2.h>
 #endif
 
-struct timeout_cb {
+struct timeout_cb_t {
   timer_cb_fn_t cb;
   void *arg;
 };
@@ -56,19 +56,21 @@ struct timeout_cb {
 /*
  * These definitions are for timeouts.c  and timeouts.h.
  */
-#ifdef __GNUC__
+#ifdef COCCI
+#define TIMEOUT_PUBLIC
+#elif defined(__GNUC__)
 /* We're not exposing any of the functions outside this file. */
 #define TIMEOUT_PUBLIC __attribute__((__unused__)) static
 #else
 /* We're not exposing any of the functions outside this file. */
 #define TIMEOUT_PUBLIC static
-#endif /* defined(__GNUC__) */
+#endif /* defined(COCCI) || ... */
 /* We're not using periodic events. */
 #define TIMEOUT_DISABLE_INTERVALS
 /* We always know the global_timeouts object, so we don't need each timeout
  * to keep a pointer to it. */
 #define TIMEOUT_DISABLE_RELATIVE_ACCESS
-/* We're providing our own struct timeout_cb. */
+/* We're providing our own struct timeout_cb_t. */
 #define TIMEOUT_CB_OVERRIDE
 /* We're going to support timers that are pretty far out in advance. Making
  * this big can be inefficient, but having a significant number of timers
@@ -80,7 +82,8 @@ struct timeout_cb {
  * use 32-bit math. */
 #define WHEEL_BIT 5
 #endif
-#include "src/ext/timeouts/timeout.c"
+
+#include "ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
 static struct mainloop_event_t *global_timer_event = NULL;
diff --git a/src/lib/evloop/timers.h b/src/lib/evloop/timers.h
index 7595554204..dd55446121 100644
--- a/src/lib/evloop/timers.h
+++ b/src/lib/evloop/timers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #include "lib/testsupport/testsupport.h"
 
 struct monotime_t;
+struct timeval;
 typedef struct timeout tor_timer_t;
 typedef void (*timer_cb_fn_t)(tor_timer_t *, void *,
                               const struct monotime_t *);
diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c
index ee6d631e3b..a2b330fddb 100644
--- a/src/lib/evloop/token_bucket.c
+++ b/src/lib/evloop/token_bucket.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket,
     flags |= TB_WRITE;
   return flags;
 }
+
+/** Initialize a token bucket in <b>bucket</b>, set up to allow <b>rate</b>
+ * per second, with a maximum burst of <b>burst</b>. The bucket is created
+ * such that <b>now_ts</b> is the current timestamp. The bucket starts out
+ * full. */
+void
+token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                      uint32_t burst, uint32_t now_ts)
+{
+  memset(bucket, 0, sizeof(token_bucket_ctr_t));
+  token_bucket_ctr_adjust(bucket, rate, burst);
+  token_bucket_ctr_reset(bucket, now_ts);
+}
+
+/** Change the configured rate and burst of the given token bucket object in
+ * <b>bucket</b>. */
+void
+token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                        uint32_t burst)
+{
+  token_bucket_cfg_init(&bucket->cfg, rate, burst);
+  token_bucket_raw_adjust(&bucket->counter, &bucket->cfg);
+}
+
+/** Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. */
+void
+token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  token_bucket_raw_reset(&bucket->counter, &bucket->cfg);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
+
+/** Refill <b>bucket</b> as appropriate, given that the current timestamp is
+ * <b>now_ts</b>. */
+void
+token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  const uint32_t elapsed_ticks =
+    (now_ts - bucket->last_refilled_at_timestamp);
+  if (elapsed_ticks > UINT32_MAX-(300*1000)) {
+    /* Either about 48 days have passed since the last refill, or the
+     * monotonic clock has somehow moved backwards. (We're looking at you,
+     * Windows.).  We accept up to a 5 minute jump backwards as
+     * "unremarkable".
+     */
+    return;
+  }
+
+  token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg,
+                                elapsed_ticks);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h
index 9398d2baa3..460dad23e4 100644
--- a/src/lib/evloop/token_bucket.h
+++ b/src/lib/evloop/token_bucket.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
   return token_bucket_raw_get(&bucket->write_bucket);
 }
 
+/**
+ * A specialized bucket containing a single counter.
+ */
+
+typedef struct token_bucket_ctr_t {
+  token_bucket_cfg_t cfg;
+  token_bucket_raw_t counter;
+  uint32_t last_refilled_at_timestamp;
+} token_bucket_ctr_t;
+
+void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                           uint32_t burst, uint32_t now_ts);
+void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                             uint32_t burst);
+void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts);
+void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts);
+
+static inline bool
+token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->counter, n);
+}
+
+static inline size_t
+token_bucket_ctr_get(const token_bucket_ctr_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->counter);
+}
+
 #ifdef TOKEN_BUCKET_PRIVATE
 
 /* To avoid making the rates too small, we consider units of "steps",
@@ -112,6 +141,6 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
 
 STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate);
 
-#endif
+#endif /* defined(TOKEN_BUCKET_PRIVATE) */
 
-#endif /* TOR_TOKEN_BUCKET_H */
+#endif /* !defined(TOR_TOKEN_BUCKET_H) */
diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c
index 931f65e710..603dddd5a3 100644
--- a/src/lib/evloop/workqueue.c
+++ b/src/lib/evloop/workqueue.c
@@ -15,7 +15,7 @@
  *
  * The main thread informs the worker threads of pending work by using a
  * condition variable.  The workers inform the main process of completed work
- * by using an alert_sockets_t object, as implemented in compat_threads.c.
+ * by using an alert_sockets_t object, as implemented in net/alertsock.c.
  *
  * The main thread can also queue an "update" that will be handled by all the
  * workers.  This is useful for updating state that all the workers share.
@@ -36,7 +36,7 @@
 #include "lib/net/socket.h"
 #include "lib/thread/threads.h"
 
-#include "tor_queue.h"
+#include "ext/tor_queue.h"
 #include <event2/event.h>
 #include <string.h>
 
@@ -44,13 +44,13 @@
 #define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW
 #define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1)
 
-TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s);
+TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_t);
 typedef struct work_tailq_t work_tailq_t;
 
-struct threadpool_s {
+struct threadpool_t {
   /** An array of pointers to workerthread_t: one for each running worker
    * thread. */
-  struct workerthread_s **threads;
+  struct workerthread_t **threads;
 
   /** Condition variable that we wait on when we have no work, and which
    * gets signaled when our queue becomes nonempty. */
@@ -59,9 +59,6 @@ struct threadpool_s {
    * <b>p</b> is work[p]. */
   work_tailq_t work[WORKQUEUE_N_PRIORITIES];
 
-  /** Weak RNG, used to decide when to ignore priority. */
-  tor_weak_rng_t weak_rng;
-
   /** The current 'update generation' of the threadpool.  Any thread that is
    * at an earlier generation needs to run the update function. */
   unsigned generation;
@@ -95,14 +92,14 @@ struct threadpool_s {
 /** Number of bits needed to hold all legal values of workqueue_priority_t */
 #define WORKQUEUE_PRIORITY_BITS 2
 
-struct workqueue_entry_s {
+struct workqueue_entry_t {
   /** The next workqueue_entry_t that's pending on the same thread or
    * reply queue. */
-  TOR_TAILQ_ENTRY(workqueue_entry_s) next_work;
+  TOR_TAILQ_ENTRY(workqueue_entry_t) next_work;
   /** The threadpool to which this workqueue_entry_t was assigned. This field
    * is set when the workqueue_entry_t is created, and won't be cleared until
    * after it's handled in the main thread. */
-  struct threadpool_s *on_pool;
+  struct threadpool_t *on_pool;
   /** True iff this entry is waiting for a worker to start processing it. */
   uint8_t pending;
   /** Priority of this entry. */
@@ -115,22 +112,22 @@ struct workqueue_entry_s {
   void *arg;
 };
 
-struct replyqueue_s {
+struct replyqueue_t {
   /** Mutex to protect the answers field */
   tor_mutex_t lock;
   /** Doubly-linked list of answers that the reply queue needs to handle. */
-  TOR_TAILQ_HEAD(, workqueue_entry_s) answers;
+  TOR_TAILQ_HEAD(, workqueue_entry_t) answers;
 
   /** Mechanism to wake up the main thread when it is receiving answers. */
   alert_sockets_t alert;
 };
 
 /** A worker thread represents a single thread in a thread pool. */
-typedef struct workerthread_s {
+typedef struct workerthread_t {
   /** Which thread it this?  In range 0..in_pool->n_threads-1 */
   int index;
   /** The pool this thread is a part of. */
-  struct threadpool_s *in_pool;
+  struct threadpool_t *in_pool;
   /** User-supplied state field that we pass to the worker functions of each
    * work item. */
   void *state;
@@ -238,7 +235,7 @@ worker_thread_extract_next_work(workerthread_t *thread)
     this_queue = &pool->work[i];
     if (!TOR_TAILQ_EMPTY(this_queue)) {
       queue = this_queue;
-      if (! tor_weak_random_one_in_n(&pool->weak_rng,
+      if (! crypto_fast_rng_one_in_n(get_thread_fast_rng(),
                                      thread->lower_priority_chance)) {
         /* Usually we'll just break now, so that we can get out of the loop
          * and use the queue where we found work. But with a small
@@ -555,11 +552,6 @@ threadpool_new(int n_threads,
   for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
     TOR_TAILQ_INIT(&pool->work[i]);
   }
-  {
-    unsigned seed;
-    crypto_rand((void*)&seed, sizeof(seed));
-    tor_init_weak_random(&pool->weak_rng, seed);
-  }
 
   pool->new_thread_state_fn = new_thread_state_fn;
   pool->new_thread_state_arg = arg;
@@ -622,8 +614,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg)
     tp->reply_cb(tp);
 }
 
-/** Register the threadpool <b>tp</b>'s reply queue with the libevent
- * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after
+/** Register the threadpool <b>tp</b>'s reply queue with Tor's global
+ * libevent mainloop. If <b>cb</b> is provided, it is run after
  * each time there is work to process from the reply queue. Return 0 on
  * success, -1 on failure.
  */
diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h
index 333a3f6dde..43cfebf788 100644
--- a/src/lib/evloop/workqueue.h
+++ b/src/lib/evloop/workqueue.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,12 +13,12 @@
 
 /** A replyqueue is used to tell the main thread about the outcome of
  * work that we queued for the workers. */
-typedef struct replyqueue_s replyqueue_t;
+typedef struct replyqueue_t replyqueue_t;
 /** A thread-pool manages starting threads and passing work to them. */
-typedef struct threadpool_s threadpool_t;
+typedef struct threadpool_t threadpool_t;
 /** A workqueue entry represents a request that has been passed to a thread
  * pool. */
-typedef struct workqueue_entry_s workqueue_entry_t;
+typedef struct workqueue_entry_t workqueue_entry_t;
 
 /** Possible return value from a work function: */
 typedef enum workqueue_reply_t {
@@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp);
 replyqueue_t *replyqueue_new(uint32_t alertsocks_flags);
 void replyqueue_process(replyqueue_t *queue);
 
-struct event_base;
 int threadpool_register_reply_event(threadpool_t *tp,
                                     void (*cb)(threadpool_t *tp));
 
diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c
index d723d04d2a..56e3818f5c 100644
--- a/src/lib/fdio/fdio.c
+++ b/src/lib/fdio/fdio.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,16 +17,21 @@
 #ifdef _WIN32
 #include <windows.h>
 #endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 
 #include "lib/fdio/fdio.h"
 #include "lib/cc/torint.h"
 #include "lib/err/torerr.h"
 
 #include <stdlib.h>
+#include <stdio.h>
 
-/** @{ */
-/** Some old versions of Unix didn't define constants for these values,
+/* Some old versions of Unix didn't define constants for these values,
  * and instead expect you to say 0, 1, or 2. */
+
+/** @cond */
 #ifndef SEEK_SET
 #define SEEK_SET 0
 #endif
@@ -36,7 +41,7 @@
 #ifndef SEEK_END
 #define SEEK_END 2
 #endif
-/** @} */
+/** @endcond */
 
 /** Return the position of <b>fd</b> with respect to the start of the file. */
 off_t
diff --git a/src/lib/fdio/fdio.h b/src/lib/fdio/fdio.h
index 8395af353b..99bc33c64b 100644
--- a/src/lib/fdio/fdio.h
+++ b/src/lib/fdio/fdio.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,9 @@
 #define TOR_FDIO_H
 
 #include <stddef.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 
 off_t tor_fd_getpos(int fd);
 int tor_fd_setpos(int fd, off_t pos);
diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am
index 6c18f00a0d..545bbc929e 100644
--- a/src/lib/fdio/include.am
+++ b/src/lib/fdio/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fdio-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fdio_a_SOURCES =			\
 	src/lib/fdio/fdio.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_fdio_testing_a_SOURCES = \
 src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fdio/fdio.h
diff --git a/src/lib/fdio/lib_fdio.md b/src/lib/fdio/lib_fdio.md
new file mode 100644
index 0000000000..9fe4b4d2be
--- /dev/null
+++ b/src/lib/fdio/lib_fdio.md
@@ -0,0 +1,5 @@
+@dir /lib/fdio
+@brief lib/fdio: Code to read/write on file descriptors.
+
+(This module also handles sockets, on platforms where a socket is not a kind
+of fd.)
diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include
index b1e49fc891..c192e6181c 100644
--- a/src/lib/fs/.may_include
+++ b/src/lib/fs/.may_include
@@ -13,4 +13,4 @@ lib/malloc/*.h
 lib/memarea/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/fs/conffile.c b/src/lib/fs/conffile.c
index 0d5d56b335..9583093c12 100644
--- a/src/lib/fs/conffile.c
+++ b/src/lib/fs/conffile.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -152,6 +152,7 @@ config_process_include(const char *path, int recursion_level, int extended,
 
   int rv = -1;
   SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
+    log_notice(LD_CONFIG, "Including configuration file \"%s\".", config_file);
     config_line_t *included_config = NULL;
     config_line_t *included_config_last = NULL;
     if (config_get_included_config(config_file, recursion_level, extended,
diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h
index 7af9119dbb..cbfe4ec7c9 100644
--- a/src/lib/fs/conffile.h
+++ b/src/lib/fs/conffile.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CONFFILE_H
@@ -20,4 +20,4 @@ int config_get_lines_include(const char *string, struct config_line_t **result,
                              int extended, int *has_include,
                              struct smartlist_t *opened_lst);
 
-#endif /* !defined(TOR_CONFLINE_H) */
+#endif /* !defined(TOR_CONFFILE_H) */
diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c
index 3c31e00d99..3432df0299 100644
--- a/src/lib/fs/dir.c
+++ b/src/lib/fs/dir.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check,
     }
   }
   close(fd);
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   /* Win32 case: we can't open() a directory. */
   (void)effective_user;
 
@@ -347,7 +347,7 @@ tor_listdir, (const char *dirname))
   }
   FindClose(handle);
   tor_free(pattern);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *prot_dname = sandbox_intern_string(dirname);
   DIR *d;
   struct dirent *de;
diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h
index 826bc2dfc5..c4ab430891 100644
--- a/src/lib/fs/dir.h
+++ b/src/lib/fs/dir.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_DIR_H
@@ -30,4 +30,4 @@ MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check,
 
 MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
 
-#endif
+#endif /* !defined(TOR_DIR_H) */
diff --git a/src/lib/fs/files.c b/src/lib/fs/files.c
index ec7dbca0e9..a0b5a40aac 100644
--- a/src/lib/fs/files.c
+++ b/src/lib/fs/files.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h
index 52c94c914f..a109cd6248 100644
--- a/src/lib/fs/files.h
+++ b/src/lib/fs/files.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,7 +27,7 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -108,7 +108,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
  * Tor is built for unit tests, or when Tor is built on an operating system
  * without its own getdelim(). */
 ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
-#endif
+#endif /* !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS) */
 
 #ifdef HAVE_GETDELIM
 /**
@@ -123,10 +123,10 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getdelim(lineptr, n, delim, stream) \
   getdelim((lineptr), (n), (delim), (stream))
-#else
+#else /* !defined(HAVE_GETDELIM) */
 #define tor_getdelim(lineptr, n, delim, stream) \
   compat_getdelim_((lineptr), (n), (delim), (stream))
-#endif
+#endif /* defined(HAVE_GETDELIM) */
 
 #ifdef HAVE_GETLINE
 /**
@@ -137,9 +137,9 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getline(lineptr, n, stream) \
   getline((lineptr), (n), (stream))
-#else
+#else /* !defined(HAVE_GETLINE) */
 #define tor_getline(lineptr, n, stream) \
   tor_getdelim((lineptr), (n), '\n', (stream))
-#endif
+#endif /* defined(HAVE_GETLINE) */
 
-#endif
+#endif /* !defined(TOR_FS_H) */
diff --git a/src/lib/fs/freespace.c b/src/lib/fs/freespace.c
index ee0f93073d..511f2a0b98 100644
--- a/src/lib/fs/freespace.c
+++ b/src/lib/fs/freespace.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am
index f33e4d6430..493db8f044 100644
--- a/src/lib/fs/include.am
+++ b/src/lib/fs/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fs-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fs_a_SOURCES =			\
 	src/lib/fs/conffile.c			\
 	src/lib/fs/dir.c			\
@@ -25,6 +26,7 @@ src_lib_libtor_fs_testing_a_SOURCES = \
 src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fs/conffile.h				\
 	src/lib/fs/dir.h				\
diff --git a/src/lib/fs/lib_fs.md b/src/lib/fs/lib_fs.md
new file mode 100644
index 0000000000..3b5b0ac7d5
--- /dev/null
+++ b/src/lib/fs/lib_fs.md
@@ -0,0 +1,9 @@
+@dir /lib/fs
+@brief lib/fs: Files, filenames, directories, etc.
+
+This module is mostly a set of compatibility wrappers around
+operating-system-specific filesystem access.
+
+It also contains a set of convenience functions for safely writing to files,
+creating directories, and so on.
+
diff --git a/src/lib/fs/lockfile.c b/src/lib/fs/lockfile.c
index 933ff1e02f..c081f57a5d 100644
--- a/src/lib/fs/lockfile.c
+++ b/src/lib/fs/lockfile.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h
index 8aeee4cc7f..91effd701d 100644
--- a/src/lib/fs/lockfile.h
+++ b/src/lib/fs/lockfile.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
                                   int *locked_out);
 void tor_lockfile_unlock(tor_lockfile_t *lockfile);
 
-#endif
+#endif /* !defined(TOR_LOCKFILE_H) */
diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c
index daaee1f9b1..cc1c40b7ab 100644
--- a/src/lib/fs/mmap.c
+++ b/src/lib/fs/mmap.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,8 +42,8 @@
  * failure, return NULL. Sets errno properly, using ERANGE to mean
  * "empty file". Must only be called on trusted Tor-owned files, as changing
  * the underlying file's size causes unspecified behavior. */
-tor_mmap_t *
-tor_mmap_file(const char *filename)
+MOCK_IMPL(tor_mmap_t *,
+tor_mmap_file,(const char *filename))
 {
   int fd; /* router file */
   char *string;
@@ -111,8 +111,8 @@ tor_mmap_file(const char *filename)
 }
 /** Release storage held for a memory mapping; returns 0 on success,
  * or -1 on failure (and logs a warning). */
-int
-tor_munmap_file(tor_mmap_t *handle)
+MOCK_IMPL(int,
+tor_munmap_file,(tor_mmap_t *handle))
 {
   int res;
 
@@ -132,8 +132,8 @@ tor_munmap_file(tor_mmap_t *handle)
   return res;
 }
 #elif defined(_WIN32)
-tor_mmap_t *
-tor_mmap_file(const char *filename)
+MOCK_IMPL(tor_mmap_t *,
+tor_mmap_file,(const char *filename))
 {
   TCHAR tfilename[MAX_PATH]= {0};
   tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t));
@@ -213,8 +213,8 @@ tor_mmap_file(const char *filename)
 }
 
 /* Unmap the file, and return 0 for success or -1 for failure */
-int
-tor_munmap_file(tor_mmap_t *handle)
+MOCK_IMPL(int,
+tor_munmap_file,(tor_mmap_t *handle))
 {
   if (handle == NULL)
     return 0;
@@ -237,4 +237,4 @@ tor_munmap_file(tor_mmap_t *handle)
 }
 #else
 #error "cannot implement tor_mmap_file"
-#endif /* defined(HAVE_MMAP) || ... || ... */
+#endif /* defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) || ... */
diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h
index 18fb18a13c..e142bd78c3 100644
--- a/src/lib/fs/mmap.h
+++ b/src/lib/fs/mmap.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_MMAP_H
 
 #include "lib/cc/compat_compiler.h"
+#include "lib/testsupport/testsupport.h"
 #include <stddef.h>
 
 #ifdef _WIN32
@@ -35,7 +36,7 @@ typedef struct tor_mmap_t {
 
 } tor_mmap_t;
 
-tor_mmap_t *tor_mmap_file(const char *filename);
-int tor_munmap_file(tor_mmap_t *handle);
+MOCK_DECL(tor_mmap_t *, tor_mmap_file, (const char *filename));
+MOCK_DECL(int, tor_munmap_file, (tor_mmap_t *handle));
 
-#endif
+#endif /* !defined(TOR_MMAP_H) */
diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c
index b3ef61979d..0d57be4b06 100644
--- a/src/lib/fs/path.c
+++ b/src/lib/fs/path.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -72,7 +72,7 @@ expand_filename(const char *filename)
    *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
    */
   return tor_strdup(filename);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   if (*filename == '~') {
     char *home, *result=NULL;
     const char *rest;
@@ -102,7 +102,7 @@ expand_filename(const char *filename)
       }
       tor_free(username);
       rest = slash ? (slash+1) : "";
-#else /* !(defined(HAVE_PWD_H)) */
+#else /* !defined(HAVE_PWD_H) */
       log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
       return tor_strdup(filename);
 #endif /* defined(HAVE_PWD_H) */
@@ -153,7 +153,7 @@ clean_fname_for_stat(char *name)
       return;
     name[len-1]='\0';
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)name;
 #endif /* defined(_WIN32) */
 }
@@ -233,7 +233,7 @@ alloc_getcwd(void)
     raw_free(cwd); // alias for free to avoid tripping check-spaces.
   }
   return result;
-#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */
   size_t size = 1024;
   char *buf = NULL;
   char *ptr = NULL;
@@ -255,9 +255,10 @@ alloc_getcwd(void)
 #endif /* !defined(_WIN32) */
 
 /** Expand possibly relative path <b>fname</b> to an absolute path.
- * Return a newly allocated string, possibly equal to <b>fname</b>. */
+ * Return a newly allocated string, which may be a duplicate of <b>fname</b>.
+ */
 char *
-make_path_absolute(char *fname)
+make_path_absolute(const char *fname)
 {
 #ifdef _WIN32
   char *absfname_malloced = _fullpath(NULL, fname, 1);
@@ -268,7 +269,7 @@ make_path_absolute(char *fname)
   if (absfname_malloced) raw_free(absfname_malloced);
 
   return absfname;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   char *absfname = NULL, *path = NULL;
 
   tor_assert(fname);
diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h
index 4675ac84e8..f0e253c556 100644
--- a/src/lib/fs/path.h
+++ b/src/lib/fs/path.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,6 +25,6 @@ char *expand_filename(const char *filename);
 int path_is_relative(const char *filename);
 void clean_fname_for_stat(char *name);
 int get_parent_directory(char *fname);
-char *make_path_absolute(char *fname);
+char *make_path_absolute(const char *fname);
 
-#endif
+#endif /* !defined(TOR_PATH_H) */
diff --git a/src/lib/fs/storagedir.c b/src/lib/fs/storagedir.c
index 2caddf1ad9..5e4f9ee257 100644
--- a/src/lib/fs/storagedir.c
+++ b/src/lib/fs/storagedir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/storagedir.h b/src/lib/fs/storagedir.h
index 7e6633a0bb..9997550b8f 100644
--- a/src/lib/fs/storagedir.h
+++ b/src/lib/fs/storagedir.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 
 typedef struct storage_dir_t storage_dir_t;
 struct config_line_t;
-struct sandbox_cfg_elem;
+struct sandbox_cfg_elem_t;
 struct tor_mmap_t;
 struct smartlist_t;
 
@@ -25,7 +25,7 @@ void storage_dir_free_(storage_dir_t *d);
   FREE_AND_NULL(storage_dir_t, storage_dir_free_, (d))
 
 int storage_dir_register_with_sandbox(storage_dir_t *d,
-                                      struct sandbox_cfg_elem **cfg);
+                                      struct sandbox_cfg_elem_t **cfg);
 const struct smartlist_t *storage_dir_list(storage_dir_t *d);
 uint64_t storage_dir_get_usage(storage_dir_t *d);
 struct tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname);
diff --git a/src/lib/fs/userdb.c b/src/lib/fs/userdb.c
index 95205c670e..40fc4dae97 100644
--- a/src/lib/fs/userdb.c
+++ b/src/lib/fs/userdb.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h
index 5c39794873..4341237c5f 100644
--- a/src/lib/fs/userdb.h
+++ b/src/lib/fs/userdb.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,6 +21,6 @@ struct passwd;
 const struct passwd *tor_getpwnam(const char *username);
 const struct passwd *tor_getpwuid(uid_t uid);
 char *get_user_homedir(const char *username);
-#endif
+#endif /* !defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_USERDB_H) */
diff --git a/src/lib/fs/winlib.c b/src/lib/fs/winlib.c
index b7302bd4ca..65ccdae40b 100644
--- a/src/lib/fs/winlib.c
+++ b/src/lib/fs/winlib.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h
index 64a22439e5..27837ac46e 100644
--- a/src/lib/fs/winlib.h
+++ b/src/lib/fs/winlib.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,6 @@
 #include <tchar.h>
 
 HANDLE load_windows_system_library(const TCHAR *library_name);
-#endif
+#endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_WINLIB_H) */
diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h
index 9a8911d494..feab554a16 100644
--- a/src/lib/geoip/country.h
+++ b/src/lib/geoip/country.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file country.h
+ * @brief Country type for geoip.
+ **/
+
 #ifndef TOR_COUNTRY_H
 #define TOR_COUNTRY_H
 
@@ -11,6 +16,7 @@
 /** A signed integer representing a country code. */
 typedef int16_t country_t;
 
+/** Maximum value for country_t. */
 #define COUNTRY_MAX INT16_MAX
 
-#endif
+#endif /* !defined(TOR_COUNTRY_H) */
diff --git a/src/lib/geoip/geoip.c b/src/lib/geoip/geoip.c
index 70b1c2dc8c..ee03d5baa1 100644
--- a/src/lib/geoip/geoip.c
+++ b/src/lib/geoip/geoip.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -70,12 +70,18 @@ static smartlist_t *geoip_countries = NULL;
  * The index is encoded in the pointer, and 1 is added so that NULL can mean
  * not found. */
 static strmap_t *country_idxplus1_by_lc_code = NULL;
-/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
- * by their respective ip_low. */
-static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
-
-/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
+/** List of all known geoip_ipv4_entry_t sorted
+ * by their respective ip_low values. */
+static smartlist_t *geoip_ipv4_entries = NULL;
+/** List of all known geoip_ipv6_entry_t, sorted by their respective
+ * ip_low values. */
+static smartlist_t *geoip_ipv6_entries = NULL;
+
+/** SHA1 digest of the IPv4 GeoIP file to include in extra-info
+ * descriptors. */
 static char geoip_digest[DIGEST_LEN];
+/** SHA1 digest of the IPv6 GeoIP file to include in extra-info
+ * descriptors. */
 static char geoip6_digest[DIGEST_LEN];
 
 /** Return a list of geoip_country_t for all known countries. */
diff --git a/src/lib/geoip/geoip.h b/src/lib/geoip/geoip.h
index f872ebd25f..2fc7fae754 100644
--- a/src/lib/geoip/geoip.h
+++ b/src/lib/geoip/geoip.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,6 +31,7 @@ int geoip_get_country_by_ipv6(const struct in6_addr *addr);
 
 /** A per-country GeoIP record. */
 typedef struct geoip_country_t {
+  /** A nul-terminated two-letter country-code. */
   char countrycode[3];
 } geoip_country_t;
 
diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am
index 9710d75ac7..ea426d14bc 100644
--- a/src/lib/geoip/include.am
+++ b/src/lib/geoip/include.am
@@ -4,6 +4,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-geoip-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_geoip_a_SOURCES =			\
 	src/lib/geoip/geoip.c
 
@@ -12,6 +13,7 @@ src_lib_libtor_geoip_testing_a_SOURCES = \
 src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/geoip/geoip.h   			\
 	src/lib/geoip/country.h
diff --git a/src/lib/geoip/lib_geoip.md b/src/lib/geoip/lib_geoip.md
new file mode 100644
index 0000000000..a3ee39d574
--- /dev/null
+++ b/src/lib/geoip/lib_geoip.md
@@ -0,0 +1,3 @@
+@dir /lib/geoip
+@brief lib/geoip: IP-to-country mapping
+
diff --git a/src/lib/intmath/addsub.c b/src/lib/intmath/addsub.c
index 12146f4e72..44613417fa 100644
--- a/src/lib/intmath/addsub.c
+++ b/src/lib/intmath/addsub.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h
index 83efa82919..bca911103b 100644
--- a/src/lib/intmath/addsub.h
+++ b/src/lib/intmath/addsub.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,4 +16,4 @@
 
 uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
 
-#endif /* !defined(TOR_INTMATH_MULDIV_H) */
+#endif /* !defined(TOR_INTMATH_ADDSUB_H) */
diff --git a/src/lib/intmath/bits.c b/src/lib/intmath/bits.c
index 2158790e3f..dace9ffe18 100644
--- a/src/lib/intmath/bits.c
+++ b/src/lib/intmath/bits.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/bits.h b/src/lib/intmath/bits.h
index c1483a18b8..687651ba35 100644
--- a/src/lib/intmath/bits.h
+++ b/src/lib/intmath/bits.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h
index d0b0e8b954..4e6c2b649a 100644
--- a/src/lib/intmath/cmp.h
+++ b/src/lib/intmath/cmp.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,4 +36,7 @@
     ((v) > (max)) ? (max) :                     \
     (v) )
 
+/** Give the absolute value of <b>x</b>, independent of its type. */
+#define ABS(x) ( ((x)<0) ? -(x) : (x) )
+
 #endif /* !defined(TOR_INTMATH_CMP_H) */
diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am
index 45ee3bd53b..155ffa145a 100644
--- a/src/lib/intmath/include.am
+++ b/src/lib/intmath/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-intmath-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_intmath_a_SOURCES =			\
 	src/lib/intmath/addsub.c			\
 	src/lib/intmath/bits.c				\
@@ -16,6 +17,7 @@ src_lib_libtor_intmath_testing_a_SOURCES = \
 src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/intmath/addsub.h			\
 	src/lib/intmath/cmp.h				\
diff --git a/src/lib/intmath/lib_intmath.md b/src/lib/intmath/lib_intmath.md
new file mode 100644
index 0000000000..4446b715cb
--- /dev/null
+++ b/src/lib/intmath/lib_intmath.md
@@ -0,0 +1,2 @@
+@dir /lib/intmath
+@brief lib/intmath: Integer mathematics.
diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h
index a4cecd69cc..b5fb79f66e 100644
--- a/src/lib/intmath/logic.h
+++ b/src/lib/intmath/logic.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@
 /** Macro: true if two values have different boolean values. */
 #define bool_neq(a,b) (!(a)!=!(b))
 
-#endif
+#endif /* !defined(HAVE_TOR_LOGIC_H) */
diff --git a/src/lib/intmath/muldiv.c b/src/lib/intmath/muldiv.c
index 6a292db7ba..875cf1bbf2 100644
--- a/src/lib/intmath/muldiv.c
+++ b/src/lib/intmath/muldiv.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -69,6 +69,20 @@ gcd64(uint64_t a, uint64_t b)
   return a;
 }
 
+/** Return the unsigned integer product of <b>a</b> and <b>b</b>. If overflow
+ * is detected, return UINT64_MAX instead. */
+uint64_t
+tor_mul_u64_nowrap(uint64_t a, uint64_t b)
+{
+  if (a == 0 || b == 0) {
+    return 0;
+  } else if (PREDICT_UNLIKELY(UINT64_MAX / a < b)) {
+    return UINT64_MAX;
+  } else {
+    return a*b;
+  }
+}
+
 /* Given a fraction *<b>numer</b> / *<b>denom</b>, simplify it.
  * Requires that the denominator is greater than 0. */
 void
diff --git a/src/lib/intmath/muldiv.h b/src/lib/intmath/muldiv.h
index 64500b6dce..43700cf1dc 100644
--- a/src/lib/intmath/muldiv.h
+++ b/src/lib/intmath/muldiv.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,8 @@ unsigned round_to_next_multiple_of(unsigned number, unsigned divisor);
 uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor);
 uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor);
 
+uint64_t tor_mul_u64_nowrap(uint64_t a, uint64_t b);
+
 void simplify_fraction64(uint64_t *numer, uint64_t *denom);
 
 /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
diff --git a/src/lib/intmath/weakrng.c b/src/lib/intmath/weakrng.c
index 99c9252c2b..a29a6a086b 100644
--- a/src/lib/intmath/weakrng.c
+++ b/src/lib/intmath/weakrng.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h
index e26bf58cbb..0394e05f79 100644
--- a/src/lib/intmath/weakrng.h
+++ b/src/lib/intmath/weakrng.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,8 +19,11 @@ typedef struct tor_weak_rng_t {
   uint32_t state;
 } tor_weak_rng_t;
 
+#ifndef COCCI
 #define TOR_WEAK_RNG_INIT {383745623}
+#endif
 #define TOR_WEAK_RANDOM_MAX (INT_MAX)
+
 void tor_init_weak_random(tor_weak_rng_t *weak_rng, unsigned seed);
 int32_t tor_weak_random(tor_weak_rng_t *weak_rng);
 int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
@@ -28,4 +31,4 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
  * <b>n</b> */
 #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n)))
 
-#endif
+#endif /* !defined(TOR_WEAKRNG_H) */
diff --git a/src/lib/lib.md b/src/lib/lib.md
new file mode 100644
index 0000000000..4f77a4c1d0
--- /dev/null
+++ b/src/lib/lib.md
@@ -0,0 +1,131 @@
+@dir /lib
+@brief lib: low-level functionality.
+
+The "lib" directory contains low-level functionality.  In general, this
+code is not necessarily Tor-specific, but is instead possibly useful for
+other applications.
+
+The modules in `lib` are currently well-factored: each one depends
+only on lower-level modules.  You can see an up-to-date list of the
+modules, sorted from lowest to highest level, by running
+`./scripts/maint/practracker/includes.py --toposort`.
+
+As of this writing, the library modules are (from lowest to highest
+level):
+
+   - \refdir{lib/cc} -- Macros for managing the C compiler and
+     language.
+
+   - \refdir{lib/version} -- Holds the current version of Tor.
+
+   - \refdir{lib/testsupport} -- Helpers for making
+     test-only code, and test mocking support.
+
+   - \refdir{lib/defs} -- Lowest-level constants.
+
+   - \refdir{lib/subsys} -- Types used for declaring a
+     "subsystem". (_A subsystem is a module with support for initialization,
+     shutdown, configuration, and so on._)
+
+   - \refdir{lib/conf} -- For declaring configuration options.
+
+   - \refdir{lib/arch} -- For handling differences in CPU
+     architecture.
+
+   - \refdir{lib/err} -- Lowest-level error handling code.
+
+   - \refdir{lib/malloc} -- Memory management.
+     management.
+
+   - \refdir{lib/intmath} -- Integer mathematics.
+
+   - \refdir{lib/fdio} -- For
+      reading and writing n file descriptors.
+
+   - \refdir{lib/lock} -- Simple locking support.
+      (_Lower-level than the rest of the threading code._)
+
+   - \refdir{lib/ctime} -- Constant-time code to avoid
+     side-channels.
+
+   - \refdir{lib/string} -- Low-level string manipulation.
+
+   - \refdir{lib/wallclock} --
+     For inspecting and manipulating the current (UTC) time.
+
+   - \refdir{lib/osinfo} -- For inspecting the OS version
+     and capabilities.
+
+   - \refdir{lib/smartlist_core} -- The bare-bones
+     pieces of our dynamic array ("smartlist") implementation.
+
+   - \refdir{lib/log} -- Log messages to files, syslogs, etc.
+
+   - \refdir{lib/container} -- General purpose containers,
+     including dynamic arrays ("smartlists"), hashtables, bit arrays,
+     etc.
+
+   - \refdir{lib/trace} -- A general-purpose API
+     function-tracing functionality Tor.  (_Currently not much used._)
+
+   - \refdir{lib/thread} -- Mid-level Threading.
+
+   - \refdir{lib/term} -- Terminal manipulation
+     (like reading a password from the user).
+
+   - \refdir{lib/memarea} -- A fast
+     "arena" style allocator, where the data is freed all at once.
+
+   - \refdir{lib/encoding} -- Encoding
+     data in various formats, datatypes, and transformations.
+
+   - \refdir{lib/dispatch} -- A general-purpose in-process
+     message delivery system.
+
+   - \refdir{lib/sandbox} -- Our Linux seccomp2 sandbox
+     implementation.
+
+   - \refdir{lib/pubsub} -- A publish/subscribe message passing system.
+
+   - \refdir{lib/fs} -- Files, filenames, directories, etc.
+
+   - \refdir{lib/confmgt} -- Parse, encode, and manipulate onfiguration files.
+
+   - \refdir{lib/crypt_ops} -- Cryptographic operations.
+
+   - \refdir{lib/meminfo} -- Functions for inspecting our
+     memory usage, if the malloc implementation exposes that to us.
+
+   - \refdir{lib/time} -- Higher level time functions, including
+     fine-gained and monotonic timers.
+
+   - \refdir{lib/math} -- Floating-point mathematical utilities.
+
+   - \refdir{lib/buf} -- An efficient byte queue.
+
+   - \refdir{lib/net} -- Networking code, including address
+     manipulation, compatibility wrappers, etc.
+
+   - \refdir{lib/compress} -- Wraps several compression libraries.
+
+   - \refdir{lib/geoip} -- IP-to-country mapping.
+
+   - \refdir{lib/tls} -- TLS library wrappers.
+
+   - \refdir{lib/evloop} -- Low-level event-loop.
+
+   - \refdir{lib/process} -- Launch and manage subprocesses.
+
+### What belongs in lib?
+
+In general, if you can imagine some program wanting the functionality
+you're writing, even if that program had nothing to do with Tor, your
+functionality belongs in lib.
+
+If it falls into one of the existing "lib" categories, your
+functionality belongs in lib.
+
+If you are using platform-specific `ifdef`s to manage compatibility
+issues among platforms, you should probably consider whether you can
+put your code into lib.
+
diff --git a/src/lib/llharden/.may_include b/src/lib/llharden/.may_include
new file mode 100644
index 0000000000..038237dadf
--- /dev/null
+++ b/src/lib/llharden/.may_include
@@ -0,0 +1,3 @@
+lib/llharden/*.h
+lib/subsys/*.h
+orconfig.h
diff --git a/src/lib/llharden/include.am b/src/lib/llharden/include.am
new file mode 100644
index 0000000000..0a4788c7dc
--- /dev/null
+++ b/src/lib/llharden/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-llharden.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-llharden-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_llharden_a_SOURCES =		\
+	src/lib/llharden/winprocess_sys.c
+
+src_lib_libtor_llharden_testing_a_SOURCES = \
+	$(src_lib_libtor_llharden_a_SOURCES)
+src_lib_libtor_llharden_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_llharden_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/llharden/winprocess_sys.h
diff --git a/src/lib/llharden/lib_llharden.md b/src/lib/llharden/lib_llharden.md
new file mode 100644
index 0000000000..69e9af5327
--- /dev/null
+++ b/src/lib/llharden/lib_llharden.md
@@ -0,0 +1,6 @@
+@dir /lib/llharden
+@brief lib/llharden: low-level unconditional process hardening
+
+This module contains process hardening code that we want to run before any
+other code, including configuration.  It needs to be self-contained, since
+nothing else will be initialized at this point.
diff --git a/src/lib/llharden/winprocess_sys.c b/src/lib/llharden/winprocess_sys.c
new file mode 100644
index 0000000000..f2c88d8c75
--- /dev/null
+++ b/src/lib/llharden/winprocess_sys.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.c
+ * \brief Subsystem object for windows process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/llharden/winprocess_sys.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef _WIN32
+#include <windows.h>
+
+#define WINPROCESS_SYS_ENABLED true
+
+static int
+subsys_winprocess_initialize(void)
+{
+#ifndef HeapEnableTerminationOnCorruption
+#define HeapEnableTerminationOnCorruption 1
+#endif
+
+  /* On heap corruption, just give up; don't try to play along. */
+  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
+
+  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
+   * (On 64-bit Windows it always fails, and some compilers don't like the
+   * PSETDEP cast.)
+   * 32-bit Windows defines _WIN32.
+   * 64-bit Windows defines _WIN32 and _WIN64. */
+#ifndef _WIN64
+  /* Call SetProcessDEPPolicy to permanently enable DEP.
+     The function will not resolve on earlier versions of Windows,
+     and failure is not dangerous. */
+  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
+  if (hMod) {
+    typedef BOOL (WINAPI *PSETDEP)(DWORD);
+    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
+                           "SetProcessDEPPolicy");
+    if (setdeppolicy) {
+      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
+      setdeppolicy(3);
+    }
+  }
+#endif /* !defined(_WIN64) */
+
+  return 0;
+}
+#else /* !defined(_WIN32) */
+#define WINPROCESS_SYS_ENABLED false
+#define subsys_winprocess_initialize NULL
+#endif /* defined(_WIN32) */
+
+const subsys_fns_t sys_winprocess = {
+  .name = "winprocess",
+  SUBSYS_DECLARE_LOCATION(),
+  /* HeapEnableTerminationOnCorruption and setdeppolicy() are security
+   * features, we want them to run first. */
+  .level = -100,
+  .supported = WINPROCESS_SYS_ENABLED,
+  .initialize = subsys_winprocess_initialize,
+};
diff --git a/src/lib/llharden/winprocess_sys.h b/src/lib/llharden/winprocess_sys.h
new file mode 100644
index 0000000000..bece1b3da9
--- /dev/null
+++ b/src/lib/llharden/winprocess_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.h
+ * \brief Declare subsystem object for winprocess.c
+ **/
+
+#ifndef TOR_WINPROCESS_SYS_H
+#define TOR_WINPROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_winprocess;
+
+#endif /* !defined(TOR_WINPROCESS_SYS_H) */
diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c
index 4ad5929715..b0084a3484 100644
--- a/src/lib/lock/compat_mutex.c
+++ b/src/lib/lock/compat_mutex.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,15 @@ tor_mutex_new_nonrecursive(void)
   tor_mutex_init_nonrecursive(m);
   return m;
 }
-/** Release all storage and system resources held by <b>m</b>. */
+/** Release all storage and system resources held by <b>m</b>.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_free_(tor_mutex_t *m)
 {
diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h
index b63ce24024..5631993cc4 100644
--- a/src/lib/lock/compat_mutex.h
+++ b/src/lib/lock/compat_mutex.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,7 +48,7 @@ typedef struct tor_mutex_t {
 #else
   /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
   int _unused;
-#endif /* defined(USE_WIN32_MUTEX) || ... */
+#endif /* defined(USE_WIN32_THREADS) || ... */
 } tor_mutex_t;
 
 tor_mutex_t *tor_mutex_new(void);
@@ -58,6 +58,11 @@ void tor_mutex_init_nonrecursive(tor_mutex_t *m);
 void tor_mutex_acquire(tor_mutex_t *m);
 void tor_mutex_release(tor_mutex_t *m);
 void tor_mutex_free_(tor_mutex_t *m);
+/**
+ * @copydoc tor_mutex_free_
+ *
+ * Additionally, set the pointer <b>m</b> to NULL.
+ **/
 #define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m))
 void tor_mutex_uninit(tor_mutex_t *m);
 
diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c
index ee5f520cd0..ac83c42a47 100644
--- a/src/lib/lock/compat_mutex_pthreads.c
+++ b/src/lib/lock/compat_mutex_pthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,8 +17,14 @@
  * "recursive" mutexes (i.e., once we can re-lock if we're already holding
  * them.) */
 static pthread_mutexattr_t attr_recursive;
+/**
+ * True iff <b>attr_recursive</b> has been initialized.
+ **/
 static int attr_initialized = 0;
 
+/**
+ * Initialize the locking module, if it is not already initialized.
+ **/
 void
 tor_locking_init(void)
 {
@@ -88,12 +94,26 @@ tor_mutex_release(tor_mutex_t *m)
 }
 /** Clean up the mutex <b>m</b> so that it no longer uses any system
  * resources.  Does not free <b>m</b>.  This function must only be called on
- * mutexes from tor_mutex_init(). */
+ * mutexes from tor_mutex_init().
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_uninit(tor_mutex_t *m)
 {
   int err;
   raw_assert(m);
+  /* If the mutex is already locked, wait until after it is unlocked to destroy
+   * it. Locking and releasing the mutex makes undefined behaviour less likely,
+   * but does not prevent it. Another thread can lock the mutex between release
+   * and destroy. */
+  tor_mutex_acquire(m);
+  tor_mutex_release(m);
   err = pthread_mutex_destroy(&m->mutex);
   if (PREDICT_UNLIKELY(err)) {
     // LCOV_EXCL_START
diff --git a/src/lib/lock/compat_mutex_winthreads.c b/src/lib/lock/compat_mutex_winthreads.c
index b0f5999e42..5fe6870a93 100644
--- a/src/lib/lock/compat_mutex_winthreads.c
+++ b/src/lib/lock/compat_mutex_winthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am
index 4e6f444347..1475b9911b 100644
--- a/src/lib/lock/include.am
+++ b/src/lib/lock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-lock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_lock_a_SOURCES =			\
 	src/lib/lock/compat_mutex.c
 
@@ -20,5 +21,6 @@ src_lib_libtor_lock_testing_a_SOURCES = \
 src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/lock/compat_mutex.h
diff --git a/src/lib/lock/lib_lock.md b/src/lib/lock/lib_lock.md
new file mode 100644
index 0000000000..6f6727bfc2
--- /dev/null
+++ b/src/lib/lock/lib_lock.md
@@ -0,0 +1,6 @@
+@dir /lib/lock
+@brief lib/lock: Simple locking support.
+
+This module is more low-level than the rest of the threading code, since it
+is needed by more intermediate-level modules.
+
diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include
index 852173aab3..54d96324db 100644
--- a/src/lib/log/.may_include
+++ b/src/lib/log/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/cc/*.h
+lib/defs/*.h
 lib/smartlist_core/*.h
 lib/err/*.h
 lib/fdio/*.h
@@ -9,7 +10,7 @@ lib/lock/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
+lib/version/*.h
 lib/wallclock/*.h
-
-micro-revision.i
\ No newline at end of file
diff --git a/src/lib/log/escape.c b/src/lib/log/escape.c
index 6ca01c6963..88ca52a78c 100644
--- a/src/lib/log/escape.c
+++ b/src/lib/log/escape.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h
index 2f726186c5..0d1eaef4c1 100644
--- a/src/lib/log/escape.h
+++ b/src/lib/log/escape.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ char *esc_for_log(const char *string) ATTR_MALLOC;
 char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC;
 const char *escaped(const char *string);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_ESCAPE_H) */
diff --git a/src/lib/log/include.am b/src/lib/log/include.am
index 4a6c9b3686..5b9f7113ba 100644
--- a/src/lib/log/include.am
+++ b/src/lib/log/include.am
@@ -5,11 +5,12 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-log-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_log_a_SOURCES =			\
 	src/lib/log/escape.c			\
-        src/lib/log/git_revision.c              \
 	src/lib/log/ratelim.c			\
 	src/lib/log/log.c			\
+	src/lib/log/log_sys.c			\
 	src/lib/log/util_bug.c
 
 if WIN32
@@ -21,16 +22,11 @@ src_lib_libtor_log_testing_a_SOURCES = \
 src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
-# Declare that these object files depend on micro-revision.i.  Without this
-# rule, we could try to build them before micro-revision.i was created.
-src/lib/log/git_revision.$(OBJEXT) \
-  src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \
-	micro-revision.i
-
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/log/escape.h				\
-        src/lib/log/git_revision.h                      \
 	src/lib/log/ratelim.h				\
 	src/lib/log/log.h				\
+	src/lib/log/log_sys.h				\
 	src/lib/log/util_bug.h				\
 	src/lib/log/win32err.h
diff --git a/src/lib/log/lib_log.md b/src/lib/log/lib_log.md
new file mode 100644
index 0000000000..8740d6a02f
--- /dev/null
+++ b/src/lib/log/lib_log.md
@@ -0,0 +1,10 @@
+@dir /lib/log
+@brief lib/log: Log messages to files, syslogs, etc.
+
+You can think of this as the logical "midpoint" of the
+\refdir{lib} code": much of the higher-level code is higher-level
+_because_ it uses the logging module, and much of the lower-level code is
+specifically written to avoid having to log, because the logging module
+depends on it.
+
+
diff --git a/src/lib/log/log.c b/src/lib/log/log.c
index a9ad38fb25..9ee87c0668 100644
--- a/src/lib/log/log.c
+++ b/src/lib/log/log.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,7 +32,8 @@
 
 #define LOG_PRIVATE
 #include "lib/log/log.h"
-#include "lib/log/git_revision.h"
+#include "lib/log/log_sys.h"
+#include "lib/version/git_revision.h"
 #include "lib/log/ratelim.h"
 #include "lib/lock/compat_mutex.h"
 #include "lib/smartlist_core/smartlist_core.h"
@@ -48,15 +49,12 @@
 #include "lib/wallclock/approx_time.h"
 #include "lib/wallclock/time_to_tm.h"
 #include "lib/fdio/fdio.h"
+#include "lib/cc/ctassert.h"
 
 #ifdef HAVE_ANDROID_LOG_H
 #include <android/log.h>
 #endif // HAVE_ANDROID_LOG_H.
 
-/** Given a severity, yields an index into log_severity_list_t.masks to use
- * for that severity. */
-#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
-
 /** @{ */
 /** The string we stick at the end of a log message when it is too long,
  * and its length. */
@@ -153,7 +151,7 @@ severity_to_android_log_priority(int severity)
       // LCOV_EXCL_STOP
   }
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** A mutex to guard changes to logfiles and logging. */
 static tor_mutex_t log_mutex;
@@ -223,6 +221,7 @@ int log_global_min_severity_ = LOG_NOTICE;
 
 static void delete_log(logfile_t *victim);
 static void close_log(logfile_t *victim);
+static void close_log_sigsafe(logfile_t *victim);
 
 static char *domain_to_string(log_domain_mask_t domain,
                              char *buf, size_t buflen);
@@ -277,8 +276,8 @@ static int log_time_granularity = 1;
 
 /** Define log time granularity for all logs to be <b>granularity_msec</b>
  * milliseconds. */
-void
-set_log_time_granularity(int granularity_msec)
+MOCK_IMPL(void,
+set_log_time_granularity,(int granularity_msec))
 {
   log_time_granularity = granularity_msec;
   tor_log_sigsafe_err_set_granularity(granularity_msec);
@@ -524,7 +523,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
      * pass them, and some very old ones do not detect overflow so well.
      * Regrettably, they call their maximum line length MAXLINE. */
 #if MAXLINE < 64
-#warn "MAXLINE is a very low number; it might not be from syslog.h after all"
+#warning "MAXLINE is very low; it might not be from syslog.h."
 #endif
     char *m = msg_after_prefix;
     if (msg_len >= MAXLINE)
@@ -533,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
     if (m != msg_after_prefix) {
       tor_free(m);
     }
-#else /* !(defined(MAXLINE)) */
+#else /* !defined(MAXLINE) */
     /* We have syslog but not MAXLINE.  That's promising! */
     syslog(severity, "%s", msg_after_prefix);
 #endif /* defined(MAXLINE) */
@@ -585,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname,
   /* check that severity is sane.  Overrunning the masks array leads to
    * interesting and hard to diagnose effects */
   raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG);
-  /* check that we've initialised the log mutex before we try to lock it */
-  raw_assert(log_mutex_initialized);
+
   LOCK_LOGS();
 
   if ((! (domain & LD_NOCB)) && pending_cb_messages
@@ -630,6 +628,10 @@ void
 tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
 {
   va_list ap;
+
+  /* check that domain is composed of known domains and flags */
+  raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain);
+
   if (severity > log_global_min_severity_)
     return;
   va_start(ap,format);
@@ -663,18 +665,21 @@ tor_log_update_sigsafe_err_fds(void)
   const logfile_t *lf;
   int found_real_stderr = 0;
 
+  /* The fds are the file descriptors of tor's stdout, stderr, and file
+   * logs. The log and err modules flush these fds during their shutdowns. */
   int fds[TOR_SIGSAFE_LOG_MAX_FDS];
   int n_fds;
 
   LOCK_LOGS();
   /* Reserve the first one for stderr. This is safe because when we daemonize,
-   * we dup2 /dev/null to stderr, */
+   * we dup2 /dev/null to stderr. */
   fds[0] = STDERR_FILENO;
   n_fds = 1;
 
   for (lf = logfiles; lf; lf = lf->next) {
-     /* Don't try callback to the control port, or syslogs: We can't
-      * do them from a signal handler. Don't try stdout: we always do stderr.
+     /* Don't try callback to the control port, syslogs, android logs, or any
+      * other non-file descriptor log: We can't call arbitrary functions from a
+      * signal handler.
       */
     if (lf->is_temporary || logfile_is_external(lf)
         || lf->seems_dead || lf->fd < 0)
@@ -683,10 +688,12 @@ tor_log_update_sigsafe_err_fds(void)
         (LD_BUG|LD_GENERAL)) {
       if (lf->fd == STDERR_FILENO)
         found_real_stderr = 1;
-      /* Avoid duplicates */
+      /* Avoid duplicates by checking the log module fd against fds */
       if (int_array_contains(fds, n_fds, lf->fd))
         continue;
-      fds[n_fds++] = lf->fd;
+      /* Update fds using the log module's fd */
+      fds[n_fds] = lf->fd;
+      n_fds++;
       if (n_fds == TOR_SIGSAFE_LOG_MAX_FDS)
         break;
     }
@@ -694,9 +701,13 @@ tor_log_update_sigsafe_err_fds(void)
 
   if (!found_real_stderr &&
       int_array_contains(fds, n_fds, STDOUT_FILENO)) {
-    /* Don't use a virtual stderr when we're also logging to stdout. */
+    /* Don't use a virtual stderr when we're also logging to stdout.
+     * If we reached max_fds logs, we'll now have (max_fds - 1) logs.
+     * That's ok, max_fds is large enough that most tor instances don't exceed
+     * it. */
     raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */
-    fds[0] = fds[--n_fds];
+    --n_fds;
+    fds[0] = fds[n_fds];
   }
 
   UNLOCK_LOGS();
@@ -727,7 +738,7 @@ tor_log_get_logfile_names(smartlist_t *out)
 
 /** Implementation of the log_fn backend, used when we have
  * variadic macros. All arguments are as for log_fn, except for
- * <b>fn</b>, which is the name of the calling functions. */
+ * <b>fn</b>, which is the name of the calling function. */
 void
 log_fn_(int severity, log_domain_mask_t domain, const char *fn,
         const char *format, ...)
@@ -804,16 +815,42 @@ logs_free_all(void)
   }
 
   /* We _could_ destroy the log mutex here, but that would screw up any logs
-   * that happened between here and the end of execution. */
+   * that happened between here and the end of execution.
+   * If tor is re-initialized, log_mutex_initialized will still be 1. So we
+   * won't trigger any undefined behaviour by trying to re-initialize the
+   * log mutex. */
+}
+
+/** Flush the signal-safe log files.
+ *
+ * This function is safe to call from a signal handler. It is currenly called
+ * by the BUG() macros, when terminating the process on an abnormal condition.
+ */
+void
+logs_flush_sigsafe(void)
+{
+  /* If we don't have fsync() in unistd.h, we can't flush the logs. */
+#ifdef HAVE_FSYNC
+  logfile_t *victim, *next;
+  /* We can't LOCK_LOGS() in a signal handler, because it may call
+   * signal-unsafe functions. And we can't deallocate memory, either. */
+  next = logfiles;
+  logfiles = NULL;
+  while (next) {
+    victim = next;
+    next = next->next;
+    if (victim->needs_close) {
+      /* We can't do anything useful if the flush fails. */
+      (void)fsync(victim->fd);
+    }
+  }
+#endif /* defined(HAVE_FSYNC) */
 }
 
 /** Remove and free the log entry <b>victim</b> from the linked-list
  * logfiles (it is probably present, but it might not be due to thread
  * racing issues). After this function is called, the caller shouldn't
  * refer to <b>victim</b> anymore.
- *
- * Long-term, we need to do something about races in the log subsystem
- * in general. See bug 222 for more details.
  */
 static void
 delete_log(logfile_t *victim)
@@ -833,13 +870,26 @@ delete_log(logfile_t *victim)
 }
 
 /** Helper: release system resources (but not memory) held by a single
- * logfile_t. */
+ * signal-safe logfile_t. If the log's resources can not be released in
+ * a signal handler, does nothing. */
 static void
-close_log(logfile_t *victim)
+close_log_sigsafe(logfile_t *victim)
 {
   if (victim->needs_close && victim->fd >= 0) {
+    /* We can't do anything useful here if close() fails: we're shutting
+     * down logging, and the err module only does fatal errors. */
     close(victim->fd);
     victim->fd = -1;
+  }
+}
+
+/** Helper: release system resources (but not memory) held by a single
+ * logfile_t. */
+static void
+close_log(logfile_t *victim)
+{
+  if (victim->needs_close) {
+    close_log_sigsafe(victim);
   } else if (victim->is_syslog) {
 #ifdef HAVE_SYSLOG_H
     if (--syslog_count == 0) {
@@ -863,15 +913,15 @@ set_log_severity_config(int loglevelMin, int loglevelMax,
   raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
   memset(severity_out, 0, sizeof(log_severity_list_t));
   for (i = loglevelMin; i >= loglevelMax; --i) {
-    severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+    severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
 }
 
 /** Add a log handler named <b>name</b> to send all messages in <b>severity</b>
  * to <b>fd</b>. Copies <b>severity</b>. Helper: does no locking. */
-static void
-add_stream_log_impl(const log_severity_list_t *severity,
-                    const char *name, int fd)
+MOCK_IMPL(STATIC void,
+add_stream_log_impl,(const log_severity_list_t *severity,
+                     const char *name, int fd))
 {
   logfile_t *lf;
   lf = tor_malloc_zero(sizeof(logfile_t));
@@ -927,18 +977,16 @@ logs_set_domain_logging(int enabled)
   UNLOCK_LOGS();
 }
 
-/** Add a log handler to receive messages during startup (before the real
- * logs are initialized).
+/** Add a log handler to accept messages when no other log is configured.
  */
 void
-add_temp_log(int min_severity)
+add_default_log(int min_severity)
 {
   log_severity_list_t *s = tor_malloc_zero(sizeof(log_severity_list_t));
   set_log_severity_config(min_severity, LOG_ERR, s);
   LOCK_LOGS();
-  add_stream_log_impl(s, "<temp>", fileno(stdout));
+  add_stream_log_impl(s, "<default>", fileno(stdout));
   tor_free(s);
-  logfiles->is_temporary = 1;
   UNLOCK_LOGS();
 }
 
@@ -1020,7 +1068,7 @@ flush_pending_log_callbacks(void)
   do {
     SMARTLIST_FOREACH_BEGIN(messages, pending_log_message_t *, msg) {
       const int severity = msg->severity;
-      const int domain = msg->domain;
+      const log_domain_mask_t domain = msg->domain;
       for (lf = logfiles; lf; lf = lf->next) {
         if (! lf->callback || lf->seems_dead ||
             ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) {
@@ -1081,8 +1129,7 @@ flush_log_messages_from_startup(void)
   UNLOCK_LOGS();
 }
 
-/** Close any log handlers added by add_temp_log() or marked by
- * mark_logs_temp(). */
+/** Close any log handlers marked by mark_logs_temp(). */
 void
 close_temp_logs(void)
 {
@@ -1134,10 +1181,10 @@ mark_logs_temp(void)
  * opening the logfile failed, -1 is returned and errno is set appropriately
  * (by open(2)).  Takes ownership of fd.
  */
-int
-add_file_log(const log_severity_list_t *severity,
-             const char *filename,
-             int fd)
+MOCK_IMPL(int,
+add_file_log,(const log_severity_list_t *severity,
+              const char *filename,
+              int fd))
 {
   logfile_t *lf;
 
@@ -1231,7 +1278,7 @@ add_android_log(const log_severity_list_t *severity,
   UNLOCK_LOGS();
   return 0;
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** If <b>level</b> is a valid log severity, return the corresponding
  * numeric value.  Otherwise, return -1. */
@@ -1267,9 +1314,16 @@ static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
   "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL",
-  "SCHED", "GUARD", "CONSDIFF", "DOS", NULL
+  "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", "MESG",
+  NULL
 };
 
+CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1);
+
+CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS);
+CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_);
+CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS);
+
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,
  * or 0 if there is no such name. */
 static log_domain_mask_t
@@ -1278,7 +1332,7 @@ parse_log_domain(const char *domain)
   int i;
   for (i=0; domain_list[i]; ++i) {
     if (!strcasecmp(domain, domain_list[i]))
-      return (1u<<i);
+      return (UINT64_C(1)<<i);
   }
   return 0;
 }
@@ -1350,7 +1404,7 @@ parse_log_severity_config(const char **cfg_ptr,
     const char *dash, *space;
     char *sev_lo, *sev_hi;
     int low, high, i;
-    log_domain_mask_t domains = ~0u;
+    log_domain_mask_t domains = LD_ALL_DOMAINS;
 
     if (*cfg == '[') {
       int err = 0;
@@ -1368,9 +1422,9 @@ parse_log_severity_config(const char **cfg_ptr,
       tor_free(domains_str);
       SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) {
             if (!strcmp(domain, "*")) {
-              domains = ~0u;
+              domains = LD_ALL_DOMAINS;
             } else {
-              int d;
+              log_domain_mask_t d;
               int negate=0;
               if (*domain == '~') {
                 negate = 1;
@@ -1464,7 +1518,7 @@ switch_logs_debug(void)
   LOCK_LOGS();
   for (lf = logfiles; lf; lf=lf->next) {
     for (i = LOG_DEBUG; i >= LOG_ERR; --i)
-      lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+      lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
   log_global_min_severity_ = get_min_log_level();
   UNLOCK_LOGS();
diff --git a/src/lib/log/log.h b/src/lib/log/log.h
index d7a5070610..aafbf9be2f 100644
--- a/src/lib/log/log.h
+++ b/src/lib/log/log.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,20 +11,24 @@
  **/
 
 #ifndef TOR_TORLOG_H
+#define TOR_TORLOG_H
 
 #include <stdarg.h>
 #include "lib/cc/torint.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/logging_types.h"
 #include "lib/testsupport/testsupport.h"
 
 #ifdef HAVE_SYSLOG_H
 #include <syslog.h>
 #define LOG_WARN LOG_WARNING
 #if LOG_DEBUG < LOG_ERR
+#ifndef COCCI
 #error "Your syslog.h thinks high numbers are more important.  " \
        "We aren't prepared to deal with that."
 #endif
-#else /* !(defined(HAVE_SYSLOG_H)) */
+#endif /* LOG_DEBUG < LOG_ERR */
+#else /* !defined(HAVE_SYSLOG_H) */
 /* Note: Syslog's logging code refers to priorities, with 0 being the most
  * important.  Thus, all our comparisons needed to be reversed when we added
  * syslog support.
@@ -55,75 +59,92 @@
 /* Logging domains */
 
 /** Catch-all for miscellaneous events and fatal errors. */
-#define LD_GENERAL  (1u<<0)
+#define LD_GENERAL  (UINT64_C(1)<<0)
 /** The cryptography subsystem. */
-#define LD_CRYPTO   (1u<<1)
+#define LD_CRYPTO   (UINT64_C(1)<<1)
 /** Networking. */
-#define LD_NET      (1u<<2)
+#define LD_NET      (UINT64_C(1)<<2)
 /** Parsing and acting on our configuration. */
-#define LD_CONFIG   (1u<<3)
+#define LD_CONFIG   (UINT64_C(1)<<3)
 /** Reading and writing from the filesystem. */
-#define LD_FS       (1u<<4)
+#define LD_FS       (UINT64_C(1)<<4)
 /** Other servers' (non)compliance with the Tor protocol. */
-#define LD_PROTOCOL (1u<<5)
+#define LD_PROTOCOL (UINT64_C(1)<<5)
 /** Memory management. */
-#define LD_MM       (1u<<6)
+#define LD_MM       (UINT64_C(1)<<6)
 /** HTTP implementation. */
-#define LD_HTTP     (1u<<7)
+#define LD_HTTP     (UINT64_C(1)<<7)
 /** Application (socks) requests. */
-#define LD_APP      (1u<<8)
+#define LD_APP      (UINT64_C(1)<<8)
 /** Communication via the controller protocol. */
-#define LD_CONTROL  (1u<<9)
+#define LD_CONTROL  (UINT64_C(1)<<9)
 /** Building, using, and managing circuits. */
-#define LD_CIRC     (1u<<10)
+#define LD_CIRC     (UINT64_C(1)<<10)
 /** Hidden services. */
-#define LD_REND     (1u<<11)
+#define LD_REND     (UINT64_C(1)<<11)
 /** Internal errors in this Tor process. */
-#define LD_BUG      (1u<<12)
+#define LD_BUG      (UINT64_C(1)<<12)
 /** Learning and using information about Tor servers. */
-#define LD_DIR      (1u<<13)
+#define LD_DIR      (UINT64_C(1)<<13)
 /** Learning and using information about Tor servers. */
-#define LD_DIRSERV  (1u<<14)
+#define LD_DIRSERV  (UINT64_C(1)<<14)
 /** Onion routing protocol. */
-#define LD_OR       (1u<<15)
+#define LD_OR       (UINT64_C(1)<<15)
 /** Generic edge-connection functionality. */
-#define LD_EDGE     (1u<<16)
+#define LD_EDGE     (UINT64_C(1)<<16)
 #define LD_EXIT     LD_EDGE
 /** Bandwidth accounting. */
-#define LD_ACCT     (1u<<17)
+#define LD_ACCT     (UINT64_C(1)<<17)
 /** Router history */
-#define LD_HIST     (1u<<18)
+#define LD_HIST     (UINT64_C(1)<<18)
 /** OR handshaking */
-#define LD_HANDSHAKE (1u<<19)
+#define LD_HANDSHAKE (UINT64_C(1)<<19)
 /** Heartbeat messages */
-#define LD_HEARTBEAT (1u<<20)
+#define LD_HEARTBEAT (UINT64_C(1)<<20)
 /** Abstract channel_t code */
-#define LD_CHANNEL   (1u<<21)
+#define LD_CHANNEL   (UINT64_C(1)<<21)
 /** Scheduler */
-#define LD_SCHED     (1u<<22)
+#define LD_SCHED     (UINT64_C(1)<<22)
 /** Guard nodes */
-#define LD_GUARD     (1u<<23)
+#define LD_GUARD     (UINT64_C(1)<<23)
 /** Generation and application of consensus diffs. */
-#define LD_CONSDIFF  (1u<<24)
+#define LD_CONSDIFF  (UINT64_C(1)<<24)
 /** Denial of Service mitigation. */
-#define LD_DOS       (1u<<25)
-/** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 26
+#define LD_DOS       (UINT64_C(1)<<25)
+/** Processes */
+#define LD_PROCESS   (UINT64_C(1)<<26)
+/** Pluggable Transports. */
+#define LD_PT        (UINT64_C(1)<<27)
+/** Bootstrap tracker. */
+#define LD_BTRACK    (UINT64_C(1)<<28)
+/** Message-passing backend. */
+#define LD_MESG      (UINT64_C(1)<<29)
 
-/** This log message is not safe to send to a callback-based logger
- * immediately.  Used as a flag, not a log domain. */
-#define LD_NOCB (1u<<31)
-/** This log message should not include a function name, even if it otherwise
- * would. Used as a flag, not a log domain. */
-#define LD_NOFUNCNAME (1u<<30)
+/** The number of log domains. */
+#define N_LOGGING_DOMAINS 30
+/** The highest log domain */
+#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1))
+/** All log domains. */
+#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS))
+
+/** The number of log flags. */
+#define N_LOGGING_FLAGS 3
+/** First bit that is reserved in log_domain_mask_t for non-domain flags. */
+#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS))
+/** All log flags. */
+#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS))
 
 #ifdef TOR_UNIT_TESTS
 /** This log message should not be intercepted by mock_saving_logv */
-#define LD_NO_MOCK (1u<<29)
+#define LD_NO_MOCK (UINT64_C(1)<<61)
 #endif
 
-/** Mask of zero or more log domains, OR'd together. */
-typedef uint32_t log_domain_mask_t;
+/** This log message is not safe to send to a callback-based logger
+ * immediately.  Used as a flag, not a log domain. */
+#define LD_NOCB (UINT64_C(1)<<62)
+/** This log message should not include a function name, even if it otherwise
+ * would. Used as a flag, not a log domain. */
+#define LD_NOFUNCNAME (UINT64_C(1)<<63)
 
 /** Configures which severities are logged for each logging domain for a given
  * log target. */
@@ -134,7 +155,8 @@ typedef struct log_severity_list_t {
 } log_severity_list_t;
 
 /** Callback type used for add_callback_log. */
-typedef void (*log_callback)(int severity, uint32_t domain, const char *msg);
+typedef void (*log_callback)(int severity, log_domain_mask_t domain,
+                             const char *msg);
 
 void init_logging(int disable_startup_queue);
 int parse_log_level(const char *level);
@@ -143,11 +165,11 @@ int parse_log_severity_config(const char **cfg,
                               log_severity_list_t *severity_out);
 void set_log_severity_config(int minSeverity, int maxSeverity,
                              log_severity_list_t *severity_out);
-void add_stream_log(const log_severity_list_t *severity, const char *name,
-                    int fd);
-int add_file_log(const log_severity_list_t *severity,
-                 const char *filename,
-                 int fd);
+void add_stream_log(const log_severity_list_t *severity,
+                    const char *name, int fd);
+MOCK_DECL(int, add_file_log,(const log_severity_list_t *severity,
+                             const char *filename,
+                             int fd));
 
 #ifdef HAVE_SYSLOG_H
 int add_syslog_log(const log_severity_list_t *severity,
@@ -164,7 +186,8 @@ void logs_set_domain_logging(int enabled);
 int get_min_log_level(void);
 void switch_logs_debug(void);
 void logs_free_all(void);
-void add_temp_log(int min_severity);
+void logs_flush_sigsafe(void);
+void add_default_log(int min_severity);
 void close_temp_logs(void);
 void rollback_log_changes(void);
 void mark_logs_temp(void);
@@ -173,7 +196,7 @@ void change_callback_log_severity(int loglevelMin, int loglevelMax,
 void flush_pending_log_callbacks(void);
 void flush_log_messages_from_startup(void);
 void log_set_application_name(const char *name);
-void set_log_time_granularity(int granularity_msec);
+MOCK_DECL(void, set_log_time_granularity,(int granularity_msec));
 void truncate_logs(void);
 
 void tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
@@ -186,6 +209,21 @@ void tor_log_get_logfile_names(struct smartlist_t *out);
 
 extern int log_global_min_severity_;
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we try to avoid our log_debug optimization, since it
+ * can have weird effects on internal macro coverage. */
+#define debug_logging_enabled() (1)
+#else
+static inline bool debug_logging_enabled(void);
+/**
+ * Return true iff debug logging is enabled for at least one domain.
+ */
+static inline bool debug_logging_enabled(void)
+{
+  return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG);
+}
+#endif /* defined(TOR_COVERAGE) */
+
 void log_fn_(int severity, log_domain_mask_t domain,
              const char *funcname, const char *format, ...)
   CHECK_PRINTF(4,5);
@@ -215,8 +253,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
   log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, args)
 #define log_debug(domain, args...)                                      \
   STMT_BEGIN                                                            \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))        \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);            \
+    if (debug_logging_enabled())                                        \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);                   \
   STMT_END
 #define log_info(domain, args...)                           \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args)
@@ -233,8 +271,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
 
 #define log_debug(domain, args, ...)                                        \
   STMT_BEGIN                                                                \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))            \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \
+    if (debug_logging_enabled())                                            \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__);        \
   STMT_END
 #define log_info(domain, args,...)                                      \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args, ##__VA_ARGS__)
@@ -270,7 +308,14 @@ extern const log_domain_mask_t LD_GENERAL_;
 MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
     const char *funcname, const char *suffix, const char *format,
     va_list ap) CHECK_PRINTF(5,0));
+MOCK_DECL(STATIC void, add_stream_log_impl,(
+         const log_severity_list_t *severity, const char *name, int fd));
+#endif /* defined(LOG_PRIVATE) */
+
+#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS)
+/** Given a severity, yields an index into log_severity_list_t.masks to use
+ * for that severity. */
+#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
 #endif
 
-# define TOR_TORLOG_H
 #endif /* !defined(TOR_TORLOG_H) */
diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c
new file mode 100644
index 0000000000..021c05d3e6
--- /dev/null
+++ b/src/lib/log/log_sys.c
@@ -0,0 +1,38 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.c
+ * \brief Setup and tear down the logging module.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/log_sys.h"
+
+static int
+subsys_logging_initialize(void)
+{
+  init_logging(0);
+  return 0;
+}
+
+static void
+subsys_logging_shutdown(void)
+{
+  logs_free_all();
+  escaped(NULL);
+}
+
+const subsys_fns_t sys_logging = {
+  .name = "log",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  /* Logging depends on threads, approx time, raw logging, and security.
+   * Most other lib modules depend on logging. */
+  .level = -90,
+  .initialize = subsys_logging_initialize,
+  .shutdown = subsys_logging_shutdown,
+};
diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h
new file mode 100644
index 0000000000..523c2e5008
--- /dev/null
+++ b/src/lib/log/log_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.h
+ * \brief Declare subsystem object for the logging module.
+ **/
+
+#ifndef TOR_LOG_SYS_H
+#define TOR_LOG_SYS_H
+
+extern const struct subsys_fns_t sys_logging;
+
+#endif /* !defined(TOR_LOG_SYS_H) */
diff --git a/src/lib/log/ratelim.c b/src/lib/log/ratelim.c
index 5eec742aa7..ac401fb398 100644
--- a/src/lib/log/ratelim.c
+++ b/src/lib/log/ratelim.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h
index 48edd7c849..e9b55d40dc 100644
--- a/src/lib/log/ratelim.h
+++ b/src/lib/log/ratelim.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -45,9 +45,11 @@ typedef struct ratelim_t {
   int n_calls_since_last_time;
 } ratelim_t;
 
+#ifndef COCCI
 #define RATELIM_INIT(r) { (r), 0, 0 }
+#endif
 #define RATELIM_TOOMANY (16*1000*1000)
 
 char *rate_limit_log(ratelim_t *lim, time_t now);
 
-#endif
+#endif /* !defined(TOR_RATELIM_H) */
diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c
index c65a91ae9e..d698ddd8a6 100644
--- a/src/lib/log/util_bug.c
+++ b/src/lib/log/util_bug.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/log/log.h"
 #include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
 #ifdef TOR_UNIT_TESTS
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -63,7 +64,7 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 {
   failed_assertion_cb = fn;
 }
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
 #define capturing_bugs() (0)
 #define add_captured_bug(s) do { } while (0)
 #endif /* defined(TOR_UNIT_TESTS) */
@@ -71,24 +72,42 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 /** Helper for tor_assert: report the assertion failure. */
 void
 tor_assertion_failed_(const char *fname, unsigned int line,
-                      const char *func, const char *expr)
+                      const char *func, const char *expr,
+                      const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
+  char *extra = NULL;
+  va_list ap;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+  if (fmt) {
+    va_start(ap,fmt);
+    tor_vasprintf(&extra, fmt, ap);
+    va_end(ap);
+  }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
   log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.",
           fname, line, func, expr);
-  tor_snprintf(buf, sizeof(buf),
-               "Assertion %s failed in %s at %s:%u",
-               expr, func, fname, line);
+  tor_asprintf(&buf, "Assertion %s failed in %s at %s:%u: %s",
+               expr, func, fname, line, extra ? extra : "");
+  tor_free(extra);
   log_backtrace(LOG_ERR, LD_BUG, buf);
+  tor_free(buf);
 }
 
 /** Helper for tor_assert_nonfatal: report the assertion failure. */
 void
 tor_bug_occurred_(const char *fname, unsigned int line,
                   const char *func, const char *expr,
-                  int once)
+                  int once, const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
   const char *once_str = once ?
     " (Future instances of this warning will be silenced.)": "";
   if (! expr) {
@@ -98,7 +117,7 @@ tor_bug_occurred_(const char *fname, unsigned int line,
     }
     log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s",
              fname, line, func, once_str);
-    tor_snprintf(buf, sizeof(buf),
+    tor_asprintf(&buf,
                  "Line unexpectedly reached at %s at %s:%u",
                  func, fname, line);
   } else {
@@ -106,13 +125,32 @@ tor_bug_occurred_(const char *fname, unsigned int line,
       add_captured_bug(expr);
       return;
     }
+
+    va_list ap;
+    char *extra = NULL;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+    if (fmt) {
+      va_start(ap,fmt);
+      tor_vasprintf(&extra, fmt, ap);
+      va_end(ap);
+    }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
     log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s",
              fname, line, func, expr, once_str);
-    tor_snprintf(buf, sizeof(buf),
-                 "Non-fatal assertion %s failed in %s at %s:%u",
-                 expr, func, fname, line);
+    tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s",
+                 expr, func, fname, line, fmt ? " : " : "",
+                 extra ? extra : "");
+    tor_free(extra);
   }
   log_backtrace(LOG_WARN, LD_BUG, buf);
+  tor_free(buf);
 
 #ifdef TOR_UNIT_TESTS
   if (failed_assertion_cb) {
@@ -122,16 +160,18 @@ tor_bug_occurred_(const char *fname, unsigned int line,
 }
 
 /**
- * Call the abort() function to kill the current process with a fatal
- * error.
+ * Call the tor_raw_abort_() function to close raw logs, then kill the current
+ * process with a fatal error. But first, close the file-based log file
+ * descriptors, so error messages are written before process termination.
  *
  * (This is a separate function so that we declare it in util_bug.h without
- * including stdlib in all the users of util_bug.h)
+ * including torerr.h in all the users of util_bug.h)
  **/
 void
 tor_abort_(void)
 {
-  abort();
+  logs_flush_sigsafe();
+  tor_raw_abort_();
 }
 
 #ifdef _WIN32
diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h
index 17e8d0c5a7..684dc7c6dd 100644
--- a/src/lib/log/util_bug.h
+++ b/src/lib/log/util_bug.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -80,10 +80,10 @@
     tor__assert_tmp_value__;                    \
   } )
 #define ASSERT_PREDICT_LIKELY_(e) ASSERT_PREDICT_UNLIKELY_(e)
-#else
+#else /* !(defined(TOR_UNIT_TESTS) && defined(__GNUC__)) */
 #define ASSERT_PREDICT_UNLIKELY_(e) PREDICT_UNLIKELY(e)
 #define ASSERT_PREDICT_LIKELY_(e) PREDICT_LIKELY(e)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) && defined(__GNUC__) */
 
 /* Sometimes we don't want to use assertions during branch coverage tests; it
  * leads to tons of unreached branches which in reality are only assertions we
@@ -92,21 +92,28 @@
 #define tor_assert(a) STMT_BEGIN                                        \
   (void)(a);                                                            \
   STMT_END
-#else
+#define tor_assertf(a, fmt, ...) STMT_BEGIN                             \
+  (void)(a);                                                            \
+  (void)(fmt);                                                          \
+  STMT_END
+#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */
 /** Like assert(3), but send assertion failures to the log as well as to
  * stderr. */
-#define tor_assert(expr) STMT_BEGIN                                     \
+#define tor_assert(expr) tor_assertf(expr, NULL)
+
+#define tor_assertf(expr, fmt, ...) STMT_BEGIN                          \
   if (ASSERT_PREDICT_LIKELY_(expr)) {                                   \
   } else {                                                              \
-    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr);     \
-    tor_abort_();                                                       \
+    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr,      \
+                          fmt, ##__VA_ARGS__);                          \
+    tor_abort_();                                                        \
   } STMT_END
 #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */
 
 #define tor_assert_unreached()                                  \
   STMT_BEGIN {                                                  \
     tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__,     \
-                          "line should be unreached");          \
+                          "line should be unreached", NULL);    \
     tor_abort_();                                               \
   } STMT_END
 
@@ -124,7 +131,9 @@
 #undef BUG
 // Coverity defines this in global headers; let's override it.  This is a
 // magic coverity-only preprocessor thing.
+#ifndef COCCI
 #nodef BUG(x) (x)
+#endif
 #endif /* defined(__COVERITY__) */
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
@@ -133,37 +142,58 @@
 #define ALL_BUGS_ARE_FATAL
 #endif
 
+/** Define ALL_BUGS_ARE_FATAL if you want Tor to crash when any problem comes
+ * up, so you can get a coredump and track things down. */
 #ifdef ALL_BUGS_ARE_FATAL
 #define tor_assert_nonfatal_unreached() tor_assert(0)
 #define tor_assert_nonfatal(cond) tor_assert((cond))
+#define tor_assertf_nonfatal(cond, fmt, ...)    \
+  tor_assertf(cond, fmt, ##__VA_ARGS__)
 #define tor_assert_nonfatal_unreached_once() tor_assert(0)
 #define tor_assert_nonfatal_once(cond) tor_assert((cond))
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \
+   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",NULL), \
     tor_abort_(), 1)                                                    \
    : 0)
+#ifndef COCCI
+#define IF_BUG_ONCE(cond) if (BUG(cond))
+#endif
 #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS)
 #define tor_assert_nonfatal_unreached() STMT_NIL
 #define tor_assert_nonfatal(cond) ((void)(cond))
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  (void)cond;                                                           \
+  (void)fmt;                                                            \
+  STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_NIL
 #define tor_assert_nonfatal_once(cond) ((void)(cond))
 #define BUG(cond) (ASSERT_PREDICT_UNLIKELY_(cond) ? 1 : 0)
+#ifndef COCCI
+#define IF_BUG_ONCE(cond) if (BUG(cond))
+#endif
 #else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */
 #define tor_assert_nonfatal_unreached() STMT_BEGIN                      \
-  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0);         \
+  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0, NULL);   \
   STMT_END
 #define tor_assert_nonfatal(cond) STMT_BEGIN                            \
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else {                                                              \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, NULL);\
+  }                                                                     \
+  STMT_END
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  if (ASSERT_PREDICT_UNLIKELY_(cond)) {                                 \
+  } else {                                                              \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0,        \
+                      fmt, ##__VA_ARGS__);                               \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_BEGIN                 \
   static int warning_logged__ = 0;                                      \
   if (!warning_logged__) {                                              \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1);       \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1, NULL); \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_once(cond) STMT_BEGIN                       \
@@ -171,15 +201,15 @@
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else if (!warning_logged__) {                                       \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1, NULL);\
   }                                                                     \
   STMT_END
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \
+  (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0,NULL),1) \
    : 0)
-#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
 
+#ifndef COCCI
 #ifdef __GNUC__
 #define IF_BUG_ONCE__(cond,var)                                         \
   if (( {                                                               \
@@ -188,26 +218,28 @@
       if (bool_result && !var) {                                        \
         var = 1;                                                        \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                          "!("#cond")", 1);                             \
+                          ("!("#cond")"), 1, NULL);                     \
       }                                                                 \
       bool_result; } ))
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 #define IF_BUG_ONCE__(cond,var)                                         \
   static int var = 0;                                                   \
   if ((cond) ?                                                          \
       (var ? 1 :                                                        \
        (var=1,                                                          \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                           "!("#cond")", 1),                            \
+                          ("!("#cond")"), 1, NULL),                     \
         1))                                                             \
       : 0)
 #endif /* defined(__GNUC__) */
+#endif /* !defined(COCCI) */
+
 #define IF_BUG_ONCE_VARNAME_(a)               \
   warning_logged_on_ ## a ## __
 #define IF_BUG_ONCE_VARNAME__(a)              \
   IF_BUG_ONCE_VARNAME_(a)
 
-/** This macro behaves as 'if (bug(x))', except that it only logs its
+/** This macro behaves as 'if (BUG(x))', except that it only logs its
  * warning once, no matter how many times it triggers.
  */
 
@@ -215,6 +247,8 @@
   IF_BUG_ONCE__(ASSERT_PREDICT_UNLIKELY_(cond),                 \
                 IF_BUG_ONCE_VARNAME__(__LINE__))
 
+#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
+
 /**
  * Use this macro after a nonfatal assertion, and before a case statement
  * where you would want to fall through.
@@ -226,16 +260,23 @@
 #define FALLTHROUGH_UNLESS_ALL_BUGS_ARE_FATAL FALLTHROUGH
 #endif
 
-/** Define this if you want Tor to crash when any problem comes up,
- * so you can get a coredump and track things down. */
-// #define tor_fragile_assert() tor_assert_unreached(0)
+/** In older code, we used tor_fragile_assert() to mark optional failure
+ * points. At these points, we could make some debug builds fail.
+ * (But release builds would continue.)
+ *
+ * To get the same behaviour in recent tor versions, define
+ * ALL_BUGS_ARE_FATAL, and use any non-fatal assertion or *BUG() macro.
+ */
 #define tor_fragile_assert() tor_assert_nonfatal_unreached_once()
 
 void tor_assertion_failed_(const char *fname, unsigned int line,
-                           const char *func, const char *expr);
+                           const char *func, const char *expr,
+                           const char *fmt, ...)
+    CHECK_PRINTF(5,6);
 void tor_bug_occurred_(const char *fname, unsigned int line,
                        const char *func, const char *expr,
-                       int once);
+                       int once, const char *fmt, ...)
+  CHECK_PRINTF(6,7);
 
 void tor_abort_(void) ATTR_NORETURN;
 
diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c
index dc45cb4c3d..8136813aab 100644
--- a/src/lib/log/win32err.c
+++ b/src/lib/log/win32err.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -47,7 +47,7 @@ format_win32_error(DWORD err)
     result = tor_malloc(len);
     wcstombs(result,str,len);
     result[len-1] = '\0';
-#else /* !(defined(UNICODE)) */
+#else /* !defined(UNICODE) */
     result = tor_strdup(str);
 #endif /* defined(UNICODE) */
   } else {
diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h
index 33413dfd15..5c1386a64d 100644
--- a/src/lib/log/win32err.h
+++ b/src/lib/log/win32err.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@
 char *format_win32_error(DWORD err);
 #endif
 
-#endif
+#endif /* !defined(TOR_WIN32ERR_H) */
diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include
index cc62bb1013..7686bf862a 100644
--- a/src/lib/malloc/.may_include
+++ b/src/lib/malloc/.may_include
@@ -3,4 +3,4 @@ orconfig.h
 lib/cc/*.h
 lib/err/*.h
 lib/malloc/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am
index 502cc1c6b7..b74292bc6e 100644
--- a/src/lib/malloc/include.am
+++ b/src/lib/malloc/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-malloc-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_malloc_a_SOURCES =			\
-	src/lib/malloc/malloc.c
+	src/lib/malloc/malloc.c				\
+	src/lib/malloc/map_anon.c
 
 if USE_OPENBSD_MALLOC
 src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c
@@ -17,5 +19,7 @@ src_lib_libtor_malloc_testing_a_SOURCES = \
 src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/malloc/malloc.h
+	src/lib/malloc/malloc.h				\
+	src/lib/malloc/map_anon.h
diff --git a/src/lib/malloc/lib_malloc.md b/src/lib/malloc/lib_malloc.md
new file mode 100644
index 0000000000..ff61722f02
--- /dev/null
+++ b/src/lib/malloc/lib_malloc.md
@@ -0,0 +1,76 @@
+@dir /lib/malloc
+@brief lib/malloc: Wrappers and utilities for memory management.
+
+
+Tor imposes a few light wrappers over C's native malloc and free
+functions, to improve convenience, and to allow wholescale replacement
+of malloc and free as needed.
+
+You should never use 'malloc', 'calloc', 'realloc, or 'free' on their
+own; always use the variants prefixed with 'tor_'.
+They are the same as the standard C functions, with the following
+exceptions:
+
+   * `tor_free(NULL)` is a no-op.
+   * `tor_free()` is a macro that takes an lvalue as an argument and sets it to
+     NULL after freeing it.  To avoid this behavior, you can use `tor_free_()`
+     instead.
+   * tor_malloc() and friends fail with an assertion if they are asked to
+     allocate a value so large that it is probably an underflow.
+   * It is always safe to `tor_malloc(0)`, regardless of whether your libc
+     allows it.
+   * `tor_malloc()`, `tor_realloc()`, and friends are never allowed to fail.
+     Instead, Tor will die with an assertion.  This means that you never
+     need to check their return values.  See the next subsection for
+     information on why we think this is a good idea.
+
+We define additional general-purpose memory allocation functions as well:
+
+   * `tor_malloc_zero(x)` behaves as `calloc(1, x)`, except the it makes clear
+     the intent to allocate a single zeroed-out value.
+   * `tor_reallocarray(x,y)` behaves as the OpenBSD reallocarray function.
+     Use it for cases when you need to realloc() in a multiplication-safe
+     way.
+
+And specific-purpose functions as well:
+
+   * `tor_strdup()` and `tor_strndup()` behaves as the underlying libc
+     functions, but use `tor_malloc()` instead of the underlying function.
+   * `tor_memdup()` copies a chunk of memory of a given size.
+   * `tor_memdup_nulterm()` copies a chunk of memory of a given size, then
+     NUL-terminates it just to be safe.
+
+#### Why assert on allocation failure?
+
+Why don't we allow `tor_malloc()` and its allies to return NULL?
+
+First, it's error-prone.  Many programmers forget to check for NULL return
+values, and testing for `malloc()` failures is a major pain.
+
+Second, it's not necessarily a great way to handle OOM conditions. It's
+probably better (we think) to have a memory target where we dynamically free
+things ahead of time in order to stay under the target.  Trying to respond to
+an OOM at the point of `tor_malloc()` failure, on the other hand, would involve
+a rare operation invoked from deep in the call stack.  (Again, that's
+error-prone and hard to debug.)
+
+Third, thanks to the rise of Linux and other operating systems that allow
+memory to be overcommitted, you can't actually ever rely on getting a NULL
+from `malloc()` when you're out of memory; instead you have to use an approach
+closer to tracking the total memory usage.
+
+#### Conventions for your own allocation functions.
+
+Whenever you create a new type, the convention is to give it a pair of
+`x_new()` and `x_free_()` functions, named after the type.
+
+Calling `x_free(NULL)` should always be a no-op.
+
+There should additionally be an `x_free()` macro, defined in terms of
+`x_free_()`.  This macro should set its lvalue to NULL.  You can define it
+using the FREE_AND_NULL macro, as follows:
+
+```
+#define x_free(ptr) FREE_AND_NULL(x_t, x_free_, (ptr))
+```
+
diff --git a/src/lib/malloc/malloc.c b/src/lib/malloc/malloc.c
index 8628acfc97..9c9d600260 100644
--- a/src/lib/malloc/malloc.c
+++ b/src/lib/malloc/malloc.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h
index ef6b509ca4..80e8091adc 100644
--- a/src/lib/malloc/malloc.h
+++ b/src/lib/malloc/malloc.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,12 +48,12 @@ void tor_free_(void *mem);
     raw_free(*tor_free__tmpvar);                               \
     *tor_free__tmpvar=NULL;                                    \
   STMT_END
-#else
+#else /* !defined(__GNUC__) */
 #define tor_free(p) STMT_BEGIN                                 \
   raw_free(p);                                                 \
   (p)=NULL;                                                    \
   STMT_END
-#endif
+#endif /* defined(__GNUC__) */
 
 #define tor_malloc(size)       tor_malloc_(size)
 #define tor_malloc_zero(size)  tor_malloc_zero_(size)
diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c
new file mode 100644
index 0000000000..628966012a
--- /dev/null
+++ b/src/lib/malloc/map_anon.c
@@ -0,0 +1,271 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.c
+ * \brief Manage anonymous mappings.
+ **/
+
+#include "orconfig.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/malloc/malloc.h"
+#include "lib/err/torerr.h"
+
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_MACH_VM_INHERIT_H
+#include <mach/vm_inherit.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <string.h>
+#include <errno.h>
+
+/**
+ * Macro to get the high bytes of a size_t, if there are high bytes.
+ * Windows needs this; other operating systems define a size_t that does
+ * what it should.
+ */
+#if SIZEOF_SIZE_T > 4
+#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32)
+#else
+#define HIGH_SIZE_T_BYTES(sz) (0)
+#endif
+
+/* Here we define a MINHERIT macro that is minherit() or madvise(), depending
+ * on what we actually want.
+ *
+ * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO
+ * to be that flag.  If there's a flag unmaps pages after fork, we define
+ * FLAG_NOINHERIT to be that flag.
+ */
+#if defined(HAVE_MINHERIT)
+#define MINHERIT minherit
+
+#ifdef INHERIT_ZERO
+#define FLAG_ZERO INHERIT_ZERO
+#elif defined(MAP_INHERIT_ZERO)
+#define FLAG_ZERO MAP_INHERIT_ZERO
+#endif
+#ifdef INHERIT_NONE
+#define FLAG_NOINHERIT INHERIT_NONE
+#elif defined(VM_INHERIT_NONE)
+#define FLAG_NOINHERIT VM_INHERIT_NONE
+#elif defined(MAP_INHERIT_NONE)
+#define FLAG_NOINHERIT MAP_INHERIT_NONE
+#endif /* defined(INHERIT_NONE) || ... */
+
+#elif defined(HAVE_MADVISE)
+
+#define MINHERIT madvise
+
+#ifdef MADV_WIPEONFORK
+#define FLAG_ZERO MADV_WIPEONFORK
+#endif
+#ifdef MADV_DONTFORK
+#define FLAG_NOINHERIT MADV_DONTFORK
+#endif
+
+#endif /* defined(HAVE_MINHERIT) || ... */
+
+#if defined(HAVE_MINHERIT) && !defined(FLAG_ZERO) && !defined(FLAG_NOINHERIT)
+#warning "minherit() is defined, but FLAG_ZERO/NOINHERIT are not."
+#warning "This is probably a bug in Tor's support for this platform."
+#endif
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped
+ * to disk.  Return 0 on success or if the facility is not available on this
+ * OS; return -1 on failure.
+ */
+static int
+lock_mem(void *mem, size_t sz)
+{
+#ifdef _WIN32
+  return VirtualLock(mem, sz) ? 0 : -1;
+#elif defined(HAVE_MLOCK)
+  return mlock(mem, sz);
+#else
+  (void) mem;
+  (void) sz;
+
+  return 0;
+#endif /* defined(_WIN32) || ... */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in
+ * a core dump.  Return 0 on success or if the facility is not available on
+ * this OS; return -1 on failure.
+ */
+static int
+nodump_mem(void *mem, size_t sz)
+{
+#if defined(MADV_DONTDUMP)
+  int rv = madvise(mem, sz, MADV_DONTDUMP);
+  if (rv == 0) {
+    return 0;
+  } else if (errno == ENOSYS || errno == EINVAL) {
+    return 0; // syscall not supported, or flag not supported.
+  } else {
+    tor_log_err_sigsafe("Unexpected error from madvise: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !defined(MADV_DONTDUMP) */
+  (void) mem;
+  (void) sz;
+  return 0;
+#endif /* defined(MADV_DONTDUMP) */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being
+ * accessible in child processes -- ideally by having them set to 0 after a
+ * fork, and if that doesn't work, by having them unmapped after a fork.
+ * Return 0 on success or if the facility is not available on this OS; return
+ * -1 on failure.
+ *
+ * If we successfully make the memory uninheritable, adjust the value of
+ * *<b>inherit_result_out</b>.
+ */
+static int
+noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out)
+{
+#ifdef FLAG_ZERO
+  int r = MINHERIT(mem, sz, FLAG_ZERO);
+  if (r == 0) {
+    *inherit_result_out = INHERIT_RES_ZERO;
+    return 0;
+  }
+#endif /* defined(FLAG_ZERO) */
+
+#ifdef FLAG_NOINHERIT
+  int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT);
+  if (r2 == 0) {
+    *inherit_result_out = INHERIT_RES_DROP;
+    return 0;
+  }
+#endif /* defined(FLAG_NOINHERIT) */
+
+#if defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)
+  /* At least one operation was tried, and neither succeeded. */
+
+  if (errno == ENOSYS || errno == EINVAL) {
+    /* Syscall not supported, or flag not supported. */
+    return 0;
+  } else {
+    tor_log_err_sigsafe("Unexpected error from minherit: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !(defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)) */
+  (void)inherit_result_out;
+  (void)mem;
+  (void)sz;
+  return 0;
+#endif /* defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) */
+}
+
+/**
+ * Return a new anonymous memory mapping that holds <b>sz</b> bytes.
+ *
+ * Memory mappings are unlike the results from malloc() in that they are
+ * handled separately by the operating system, and as such can have different
+ * kernel-level flags set on them.
+ *
+ * The "flags" argument may be zero or more of ANONMAP_PRIVATE and
+ * ANONMAP_NOINHERIT.
+ *
+ * Memory returned from this function must be released with
+ * tor_munmap_anonymous().
+ *
+ * If <b>inherit_result_out</b> is non-NULL, set it to one of
+ * INHERIT_RES_KEEP, INHERIT_RES_DROP, or INHERIT_RES_ZERO, depending on the
+ * properties of the returned memory.
+ *
+ * [Note: OS people use the word "anonymous" here to mean that the memory
+ * isn't associated with any file. This has *nothing* to do with the kind of
+ * anonymity that Tor is trying to provide.]
+ */
+void *
+tor_mmap_anonymous(size_t sz, unsigned flags,
+                   inherit_res_t *inherit_result_out)
+{
+  void *ptr;
+  inherit_res_t itmp=0;
+  if (inherit_result_out == NULL) {
+    inherit_result_out = &itmp;
+  }
+  *inherit_result_out = INHERIT_RES_KEEP;
+
+#if defined(_WIN32)
+  HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE,
+                                     NULL, /*attributes*/
+                                     PAGE_READWRITE,
+                                     HIGH_SIZE_T_BYTES(sz),
+                                     sz & 0xffffffff,
+                                     NULL /* name */);
+  raw_assert(mapping != NULL);
+  ptr = MapViewOfFile(mapping, FILE_MAP_WRITE,
+                      0, 0, /* Offset */
+                      0 /* Extend to end of mapping */);
+  raw_assert(ptr);
+  CloseHandle(mapping); /* mapped view holds a reference */
+#elif defined(HAVE_SYS_MMAN_H)
+  ptr = mmap(NULL, sz,
+             PROT_READ|PROT_WRITE,
+             MAP_ANON|MAP_PRIVATE,
+             -1, 0);
+  raw_assert(ptr != MAP_FAILED);
+  raw_assert(ptr != NULL);
+#else
+  ptr = tor_malloc_zero(sz);
+#endif /* defined(_WIN32) || ... */
+
+  if (flags & ANONMAP_PRIVATE) {
+    int lock_result = lock_mem(ptr, sz);
+    raw_assert(lock_result == 0);
+    int nodump_result = nodump_mem(ptr, sz);
+    raw_assert(nodump_result == 0);
+  }
+
+  if (flags & ANONMAP_NOINHERIT) {
+    int noinherit_result = noinherit_mem(ptr, sz, inherit_result_out);
+    raw_assert(noinherit_result == 0);
+  }
+
+  return ptr;
+}
+
+/**
+ * Release <b>sz</b> bytes of memory that were previously mapped at
+ * <b>mapping</b> by tor_mmap_anonymous().
+ **/
+void
+tor_munmap_anonymous(void *mapping, size_t sz)
+{
+  if (!mapping)
+    return;
+
+#if defined(_WIN32)
+  (void)sz;
+  UnmapViewOfFile(mapping);
+#elif defined(HAVE_SYS_MMAN_H)
+  munmap(mapping, sz);
+#else
+  (void)sz;
+  tor_free(mapping);
+#endif /* defined(_WIN32) || ... */
+}
diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h
new file mode 100644
index 0000000000..0354668d65
--- /dev/null
+++ b/src/lib/malloc/map_anon.h
@@ -0,0 +1,71 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.h
+ * \brief Headers for map_anon.c
+ **/
+
+#ifndef TOR_MAP_ANON_H
+#define TOR_MAP_ANON_H
+
+#include "lib/malloc/malloc.h"
+#include <stddef.h>
+
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * swapped or dumped.
+ *
+ * In some operating systems, this flag is not implemented.
+ */
+#define ANONMAP_PRIVATE   (1u<<0)
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * inherited after a fork().  In some operating systems, trying to access it
+ * afterwards will cause its contents to be zero.  In others, trying to access
+ * it afterwards will cause a crash.
+ *
+ * In some operating systems, this flag is not implemented at all.
+ */
+#define ANONMAP_NOINHERIT (1u<<1)
+
+typedef enum {
+  /** Possible value for inherit_result_out: the memory will be kept
+   * by any child process. */
+  INHERIT_RES_KEEP=0,
+  /** Possible value for inherit_result_out: the memory will be dropped in the
+   * child process. Attempting to access it will likely cause a segfault. */
+  INHERIT_RES_DROP,
+  /** Possible value for inherit_result_out: the memory will be cleared in
+   * the child process. */
+  INHERIT_RES_ZERO
+} inherit_res_t;
+
+/* Here we define the NOINHERIT_CAN_FAIL macro if and only if
+ * it's possible that ANONMAP_NOINHERIT might yield inheritable memory.
+ */
+#ifdef _WIN32
+/* Windows can't fork, so NOINHERIT is never needed. */
+#elif defined(HAVE_MINHERIT)
+/* minherit() will always have a working MAP_INHERIT_NONE or MAP_INHERIT_ZERO.
+ * NOINHERIT should always work.
+ */
+#elif defined(HAVE_MADVISE)
+/* madvise() sometimes has neither MADV_DONTFORK and MADV_WIPEONFORK.
+ * We need to be ready for the possibility it failed.
+ *
+ * (Linux added DONTFORK in 2.6.16 and WIPEONFORK in 4.14. If we someday
+ * require 2.6.16 or later, we can assume that DONTFORK will work.)
+ */
+#define NOINHERIT_CAN_FAIL
+#else
+#define NOINHERIT_CAN_FAIL
+#endif /* defined(_WIN32) || ... */
+
+void *tor_mmap_anonymous(size_t sz, unsigned flags,
+                         inherit_res_t *inherit_result_out);
+void tor_munmap_anonymous(void *mapping, size_t sz);
+
+#endif /* !defined(TOR_MAP_ANON_H) */
diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include
index 1fd26864dc..f8bc264a5f 100644
--- a/src/lib/math/.may_include
+++ b/src/lib/math/.may_include
@@ -3,3 +3,5 @@ orconfig.h
 lib/cc/*.h
 lib/log/*.h
 lib/math/*.h
+lib/testsupport/*.h
+lib/crypt_ops/*.h
diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c
index eafad358c3..c09555209e 100644
--- a/src/lib/math/fp.c
+++ b/src/lib/math/fp.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -74,8 +74,8 @@ clamp_double_to_int64(double number)
   branches that are not taken.
 */
 #define PROBLEMATIC_FLOAT_CONVERSION_WARNING
-DISABLE_GCC_WARNING(float-conversion)
-#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */
+DISABLE_GCC_WARNING("-Wfloat-conversion")
+#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */
 
 /*
   With clang 4.0 we apparently run into "double promotion" warnings here,
@@ -84,7 +84,7 @@ DISABLE_GCC_WARNING(float-conversion)
 #if defined(__clang__)
 #if __has_warning("-Wdouble-promotion")
 #define PROBLEMATIC_DOUBLE_PROMOTION_WARNING
-DISABLE_GCC_WARNING(double-promotion)
+DISABLE_GCC_WARNING("-Wdouble-promotion")
 #endif
 #endif /* defined(__clang__) */
 
@@ -115,9 +115,29 @@ DISABLE_GCC_WARNING(double-promotion)
   return signbit(number) ? INT64_MIN : INT64_MAX;
 
 #ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
-ENABLE_GCC_WARNING(double-promotion)
+ENABLE_GCC_WARNING("-Wdouble-promotion")
 #endif
 #ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
-ENABLE_GCC_WARNING(float-conversion)
+ENABLE_GCC_WARNING("-Wfloat-conversion")
+#endif
+}
+
+/* isinf() wrapper for tor */
+int
+tor_isinf(double x)
+{
+  /* Same as above, work around the "double promotion" warnings */
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+DISABLE_GCC_WARNING("-Wfloat-conversion")
+#endif
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+DISABLE_GCC_WARNING("-Wdouble-promotion")
+#endif
+  return isinf(x);
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+ENABLE_GCC_WARNING("-Wdouble-promotion")
+#endif
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+ENABLE_GCC_WARNING("-Wfloat-conversion")
 #endif
 }
diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h
index 6f07152e92..0a7a685485 100644
--- a/src/lib/math/fp.h
+++ b/src/lib/math/fp.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST;
 long tor_lround(double d) ATTR_CONST;
 int64_t tor_llround(double d) ATTR_CONST;
 int64_t clamp_double_to_int64(double number);
+int tor_isinf(double x);
 
-#endif
+#endif /* !defined(TOR_FP_H) */
diff --git a/src/lib/math/include.am b/src/lib/math/include.am
index b088b3f3cc..b2ca280f47 100644
--- a/src/lib/math/include.am
+++ b/src/lib/math/include.am
@@ -5,16 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-math-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_math_a_SOURCES =	\
 		src/lib/math/fp.c		\
-		src/lib/math/laplace.c
-
+		src/lib/math/laplace.c 	\
+		src/lib/math/prob_distr.c
 
 src_lib_libtor_math_testing_a_SOURCES = \
 	$(src_lib_libtor_math_a_SOURCES)
 src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/math/fp.h		\
-		src/lib/math/laplace.h
+		src/lib/math/laplace.h  \
+		src/lib/math/prob_distr.h
diff --git a/src/lib/math/laplace.c b/src/lib/math/laplace.c
index 302edb20b8..a0e67384e6 100644
--- a/src/lib/math/laplace.c
+++ b/src/lib/math/laplace.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,7 @@ sample_laplace_distribution(double mu, double b, double p)
   tor_assert(p >= 0.0 && p < 1.0);
 
   /* This is the "inverse cumulative distribution function" from:
-   * http://en.wikipedia.org/wiki/Laplace_distribution */
+   * https://en.wikipedia.org/wiki/Laplace_distribution */
   if (p <= 0.0) {
     /* Avoid taking log(0.0) == -INFINITY, as some processors or compiler
      * options can cause the program to trap. */
diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h
index e8651e5197..e0dd166bbd 100644
--- a/src/lib/math/laplace.h
+++ b/src/lib/math/laplace.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ int64_t sample_laplace_distribution(double mu, double b, double p);
 int64_t add_laplace_noise(int64_t signal, double random, double delta_f,
                           double epsilon);
 
-#endif
+#endif /* !defined(TOR_LAPLACE_H) */
diff --git a/src/lib/math/lib_math.md b/src/lib/math/lib_math.md
new file mode 100644
index 0000000000..9cc256d24b
--- /dev/null
+++ b/src/lib/math/lib_math.md
@@ -0,0 +1,6 @@
+@dir /lib/math
+@brief lib/math: Floating-point math utilities.
+
+This module includes a bunch of floating-point compatibility code, and
+implementations for several probability distributions.
+
diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c
new file mode 100644
index 0000000000..31d485120e
--- /dev/null
+++ b/src/lib/math/prob_distr.c
@@ -0,0 +1,1692 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file prob_distr.c
+ *
+ * \brief
+ *  Implements various probability distributions.
+ *  Almost all code is courtesy of Riastradh.
+ *
+ * \details
+ * Here are some details that might help you understand this file:
+ *
+ * - Throughout this file, `eps' means the largest relative error of a
+ *   correctly rounded floating-point operation, which in binary64
+ *   floating-point arithmetic is 2^-53.  Here the relative error of a
+ *   true value x from a computed value y is |x - y|/|x|.  This
+ *   definition of epsilon is conventional for numerical analysts when
+ *   writing error analyses.  (If your libm doesn't provide correctly
+ *   rounded exp and log, their relative error is usually below 2*2^-53
+ *   and probably closer to 1.1*2^-53 instead.)
+ *
+ *   The C constant DBL_EPSILON is actually twice this, and should
+ *   perhaps rather be named ulp(1) -- that is, it is the distance from
+ *   1 to the next greater floating-point number, which is usually of
+ *   more interest to programmers and hardware engineers.
+ *
+ *   Since this file is concerned mainly with error bounds rather than
+ *   with low-level bit-hacking of floating-point numbers, we adopt the
+ *   numerical analysts' definition in the comments, though we do use
+ *   DBL_EPSILON in a handful of places where it is convenient to use
+ *   some function of eps = DBL_EPSILON/2 in a case analysis.
+ *
+ * - In various functions (e.g. sample_log_logistic()) we jump through hoops so
+ *   that we can use reals closer to 0 than closer to 1, since we achieve much
+ *   greater accuracy for floating point numbers near 0. In particular, we can
+ *   represent differences as small as 10^-300 for numbers near 0, but of no
+ *   less than 10^-16 for numbers near 1.
+ **/
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "lib/math/prob_distr.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/cc/ctassert.h"
+#include "lib/log/util_bug.h"
+
+#include <float.h>
+#include <math.h>
+#include <stddef.h>
+
+#ifndef COCCI
+/** Declare a function that downcasts from a generic dist struct to the actual
+ *  subtype probablity distribution it represents. */
+#define DECLARE_PROB_DISTR_DOWNCAST_FN(name) \
+  static inline                           \
+  const struct name##_t *                             \
+  dist_to_const_##name(const struct dist_t *obj) {    \
+    tor_assert(obj->ops == &name##_ops);            \
+    return SUBTYPE_P(obj, struct name ## _t, base);   \
+  }
+DECLARE_PROB_DISTR_DOWNCAST_FN(uniform)
+DECLARE_PROB_DISTR_DOWNCAST_FN(geometric)
+DECLARE_PROB_DISTR_DOWNCAST_FN(logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(log_logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(genpareto)
+DECLARE_PROB_DISTR_DOWNCAST_FN(weibull)
+#endif /* !defined(COCCI) */
+
+/**
+ * Count number of one bits in 32-bit word.
+ */
+static unsigned
+bitcount32(uint32_t x)
+{
+
+  /* Count two-bit groups.  */
+  x -= (x >> 1) & UINT32_C(0x55555555);
+
+  /* Count four-bit groups.  */
+  x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333));
+
+  /* Count eight-bit groups.  */
+  x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f);
+
+  /* Sum all eight-bit groups, and extract the sum.  */
+  return (x * UINT32_C(0x01010101)) >> 24;
+}
+
+/**
+ * Count leading zeros in 32-bit word.
+ */
+static unsigned
+clz32(uint32_t x)
+{
+
+  /* Round up to a power of two.  */
+  x |= x >> 1;
+  x |= x >> 2;
+  x |= x >> 4;
+  x |= x >> 8;
+  x |= x >> 16;
+
+  /* Subtract count of one bits from 32.  */
+  return (32 - bitcount32(x));
+}
+
+/*
+ * Some lemmas that will be used throughout this file to prove various error
+ * bounds:
+ *
+ * Lemma 1.  If |d| <= 1/2, then 1/(1 + d) <= 2.
+ *
+ * Proof.  If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1.
+ * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2.  QED.
+ *
+ * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b,
+ * then b = a*(1 + e) for |e| <= 2|d' - d|.
+ *
+ * Proof.  |a - b|/|a|
+ *             = |a - a*(1 + d)/(1 + d')|/|a|
+ *             = |1 - (1 + d)/(1 + d')|
+ *             = |(1 + d' - 1 - d)/(1 + d')|
+ *             = |(d' - d)/(1 + d')|
+ *            <= 2|d' - d|, by Lemma 1,
+ *
+ * QED.
+ *
+ * Lemma 3.  For |d|, |d'| < 1/4,
+ *
+ *     |log((1 + d)/(1 + d'))| <= 4|d - d'|.
+ *
+ * Proof.  Write
+ *
+ *     log((1 + d)/(1 + d'))
+ *      = log(1 + (1 + d)/(1 + d') - 1)
+ *      = log(1 + (1 + d - 1 - d')/(1 + d')
+ *      = log(1 + (d - d')/(1 + d')).
+ *
+ * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor
+ * series of log(1 + x) converges absolutely for (d - d')/(1 + d'),
+ * and thus we have
+ *
+ *     |log(1 + (d - d')/(1 + d'))|
+ *      = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n|
+ *     <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n
+ *      = 1/(1 - |2(d' - d)|)
+ *     <= 4|d' - d|,
+ *
+ * QED.
+ *
+ * Lemma 4.  If 1/e <= 1 + x <= e, then
+ *
+ *     log(1 + (1 + d) x) = (1 + d') log(1 + x)
+ *
+ * for |d'| < 8|d|.
+ *
+ * Proof.  Write
+ *
+ *     log(1 + (1 + d) x)
+ *     = log(1 + x + x*d)
+ *     = log((1 + x) (1 + x + x*d)/(1 + x))
+ *     = log(1 + x) + log((1 + x + x*d)/(1 + x))
+ *     = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)).
+ *
+ * The relative error is bounded by
+ *
+ *     |log((1 + x + x*d)/(1 + x))/log(1 + x)|
+ *     <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3,
+ *      = 4|x*d|/|log(1 + x)|
+ *      < 8|d|,
+ *
+ * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2.  QED.
+ */
+
+/**
+ * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x).
+ * Maps a log-odds-space probability in [-infinity, +infinity] into a
+ * direct-space probability in [0,1].  Inverse of logit.
+ *
+ * Ill-conditioned for large x; the identity logistic(-x) = 1 -
+ * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may
+ * help to rearrange a computation.
+ *
+ * This implementation gives relative error bounded by 7 eps.
+ */
+STATIC double
+logistic(double x)
+{
+  if (x <= log(DBL_EPSILON/2)) {
+    /*
+     * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case
+     * we will approximate the logistic() function with e^x because the
+     * relative error is less than eps. Here is a calculation of the
+     * relative error between the logistic() function and e^x and a proof
+     * that it's less than eps:
+     *
+     *     |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)|
+     *     <= |1 - 1/(1 + e^x)|*|1 + e^x|
+     *      = |e^x/(1 + e^x)|*|1 + e^x|
+     *      = |e^x|
+     *     <= eps.
+     */
+    return exp(x); /* return e^x */
+  } else if (x <= -log(DBL_EPSILON/2)) {
+    /*
+     * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 +
+     * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we
+     * also have 0 < 1/(1 + e^{-x}) < 1.  Thus, if exp has
+     * relative error d0, + has relative error d1, and /
+     * has relative error d2, then we get
+     *
+     *     (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)]
+     *     = (1 + d0)/[1 + e^{-x} + d0 e^{-x}
+     *                     + d1 + d1 e^{-x} + d0 d1 e^{-x}]
+     *     = (1 + d0)/[(1 + e^{-x})
+     *                 * (1 + d0 e^{-x}/(1 + e^{-x})
+     *                      + d1/(1 + e^{-x})
+     *                      + d0 d1 e^{-x}/(1 + e^{-x}))].
+     *     = (1 + d0)/[(1 + e^{-x})(1 + d')]
+     *     = [1/(1 + e^{-x})] (1 + d0)/(1 + d')
+     *
+     * where
+     *
+     *     d' = d0 e^{-x}/(1 + e^{-x})
+     *          + d1/(1 + e^{-x})
+     *          + d0 d1 e^{-x}/(1 + e^{-x}).
+     *
+     * By Lemma 2 this relative error is bounded by
+     *
+     *     2|d0 - d'|
+     *      = 2|d0 - d0 e^{-x}/(1 + e^{-x})
+     *             - d1/(1 + e^{-x})
+     *             - d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})|
+     *             + 2|d1/(1 + e^{-x})|
+     *             + 2|d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1|
+     *     <= 4|d0| + 2|d1| + 2|d0 d1|
+     *     <= 6 eps + 2 eps^2.
+     */
+    return 1/(1 + exp(-x));
+  } else {
+    /*
+     * e^{-x} <= eps, so the relative error of 1 from 1/(1
+     * + e^{-x}) is
+     *
+     *     |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})|
+     *      = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})|
+     *      = |e^{-x}|
+     *     <= eps.
+     *
+     * This computation avoids an intermediate overflow
+     * exception, although the effect on the result is
+     * harmless.
+     *
+     * XXX Should maybe raise inexact here.
+     */
+    return 1;
+  }
+}
+
+/**
+ * Compute the logit function: log p/(1 - p).  Defined on [0,1].  Maps
+ * a direct-space probability in [0,1] to a log-odds-space probability
+ * in [-infinity, +infinity].  Inverse of logistic.
+ *
+ * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) =
+ * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help
+ * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)].
+ *
+ * This implementation gives relative error bounded by 10 eps.
+ */
+STATIC double
+logit(double p)
+{
+
+  /* logistic(-1) <= p <= logistic(+1) */
+  if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) {
+    /*
+     * For inputs near 1/2, we want to compute log1p(near
+     * 0) rather than log(near 1), so write this as:
+     *
+     * log(p/(1 - p)) = -log((1 - p)/p)
+     * = -log(1 + (1 - p)/p - 1)
+     * = -log(1 + (1 - p - p)/p)
+     * = -log(1 + (1 - 2p)/p).
+     *
+     * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point
+     * evaluation of 1 - 2p is exact; the only error arises
+     * from division and log1p.  First, note that if
+     * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies
+     * in the bounds of Lemma 4.
+     *
+     * If division has relative error d0 and log1p has
+     * relative error d1, the outcome is
+     *
+     *     -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p)
+     *     = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p)
+     *     = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p).
+     *
+     * where |d'| < 8|d0| by Lemma 4.  The relative error
+     * is then bounded by
+     *
+     *     |d1 + d' + d1 d'|
+     *     <= |d1| + 8|d0| + 8|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return -log1p((1 - 2*p)/p);
+  } else {
+    /*
+     * For inputs near 0, although 1 - p may be rounded to
+     * 1, it doesn't matter much because the magnitude of
+     * the result is so much larger.  For inputs near 1, we
+     * can compute 1 - p exactly, although the precision on
+     * the input is limited so we won't ever get more than
+     * about 700 for the output.
+     *
+     * If - has relative error d0, / has relative error d1,
+     * and log has relative error d2, then
+     *
+     *     (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)])
+     *     = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))]
+     *     = log(p/(1 - p)) + d2 log(p/(1 - p))
+     *       + (1 + d2) log((1 + d0)/(1 + d1))
+     *     = log(p/(1 - p))*[1 + d2 +
+     *         + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))]
+     *
+     * Since 0 <= p < logistic(-1) or logistic(+1) < p <=
+     * 1, we have |log(p/(1 - p))| > 1.  Hence this error
+     * is bounded by
+     *
+     *     |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))
+     *                      / log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))|
+     *     <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3,
+     *     <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0|
+     *     <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return log(p/(1 - p));
+  }
+}
+
+/**
+ * Compute the logit function, translated in input by 1/2: logithalf(p)
+ * = logit(1/2 + p).  Defined on [-1/2, 1/2].  Inverse of logistichalf.
+ *
+ * Ill-conditioned near +/-1/2.  If |p0| > 1/2 - 1/(1 + e), it may be
+ * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead.
+ * This implementation gives relative error bounded by 34 eps.
+ */
+STATIC double
+logithalf(double p0)
+{
+
+  if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) {
+    /*
+     * logit(1/2 + p0)
+     * = log((1/2 + p0)/(1 - (1/2 + p0)))
+     * = log((1/2 + p0)/(1/2 - p0))
+     * = log(1 + (1/2 + p0)/(1/2 - p0) - 1)
+     * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0))
+     * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0))
+     * = log(1 + 2 p0/(1/2 - p0))
+     *
+     * If the error of subtraction is d0, the error of
+     * division is d1, and the error of log1p is d2, then
+     * what we compute is
+     *
+     *  (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)])
+     *  = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0))
+     *  = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0))
+     *  = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)),
+     *
+     * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and
+     * |d''| < 8|d'| < 32 eps by Lemma 4 since
+     *
+     *  1/e <= 1 + 2*p0/(1/2 - p0) <= e
+     *
+     * when |p0| <= 1/2 - 1/(1 + e).  Hence the relative
+     * error is bounded by
+     *
+     *  |d2 + d'' + d2 d''|
+     *  <= |d2| + |d''| + |d2 d''|
+     *  <= |d1| + 32 |d0| + 32 |d1 d0|
+     *  <= 33 eps + 32 eps^2.
+     */
+    return log1p(2*p0/(0.5 - p0));
+  } else {
+    /*
+     * We have a choice of computing logit(1/2 + p0) or
+     * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0).  It
+     * doesn't matter which way we do this: either way,
+     * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference
+     * are computed exactly.  So let's do the one that
+     * skips the final negation.
+     *
+     * The result is
+     *
+     *  (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)])
+     *  = (1 + d1) (1 + log((1 + d0)/(1 + d2))
+     *                  / log((1/2 + p0)/(1/2 - p0)))
+     *    * log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') logit(1/2 + p0)
+     *
+     * where
+     *
+     *  d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0)
+     *       + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0).
+     *
+     * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1.
+     * Provided |d0|, |d2| < 1/4, by Lemma 3 we have
+     *
+     *  |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|.
+     *
+     * Hence the relative error is bounded by
+     *
+     *  |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2|
+     *       <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2|
+     *       <= 9 eps + 8 eps^2.
+     */
+    return log((0.5 + p0)/(0.5 - p0));
+  }
+}
+
+/*
+ * The following random_uniform_01 is tailored for IEEE 754 binary64
+ * floating-point or smaller.  It can be adapted to larger
+ * floating-point formats like i387 80-bit or IEEE 754 binary128, but
+ * it may require sampling more bits.
+ */
+CTASSERT(FLT_RADIX == 2);
+CTASSERT(-DBL_MIN_EXP <= 1021);
+CTASSERT(DBL_MANT_DIG <= 53);
+
+/**
+ * Draw a floating-point number in [0, 1] with uniform distribution.
+ *
+ * Note that the probability of returning 0 is less than 2^-1074, so
+ * callers need not check for it.  However, callers that cannot handle
+ * rounding to 1 must deal with that, because it occurs with
+ * probability 2^-54, which is small but nonnegligible.
+ */
+STATIC double
+random_uniform_01(void)
+{
+  uint32_t z, x, hi, lo;
+  double s;
+
+  /*
+   * Draw an exponent, geometrically distributed, but give up if
+   * we get a run of more than 1088 zeros, which really means the
+   * system is broken.
+   */
+  z = 0;
+  while ((x = crypto_fast_rng_get_u32(get_thread_fast_rng())) == 0) {
+    if (z >= 1088)
+      /* Your bit sampler is broken.  Go home.  */
+      return 0;
+    z += 32;
+  }
+  z += clz32(x);
+
+  /*
+   * Pick 32-bit halves of an odd normalized significand.
+   * Picking it odd breaks ties in the subsequent rounding, which
+   * occur only with measure zero in the uniform distribution on
+   * [0, 1].
+   */
+  hi = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x80000000);
+  lo = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x00000001);
+
+  /* Round to nearest scaled significand in [2^63, 2^64].  */
+  s = hi*(double)4294967296 + lo;
+
+  /* Rescale into [1/2, 1] and apply exponent in one swell foop.  */
+  return s * ldexp(1, -(64 + z));
+}
+
+/*******************************************************************/
+
+/* Functions for specific probability distributions start here: */
+
+/*
+ * Logistic(mu, sigma) distribution, supported on (-infinity,+infinity)
+ *
+ * This is the uniform distribution on [0,1] mapped into log-odds
+ * space, scaled by sigma and translated by mu.
+ *
+ * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2
+ * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma)
+ * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma)
+ * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p)
+ * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p)
+ */
+
+/**
+ * Compute the CDF of the Logistic(mu, sigma) distribution: the
+ * logistic function.  Well-conditioned for negative inputs and small
+ * positive inputs; ill-conditioned for large positive inputs.
+ */
+STATIC double
+cdf_logistic(double x, double mu, double sigma)
+{
+  return logistic((x - mu)/sigma);
+}
+
+/**
+ * Compute the SF of the Logistic(mu, sigma) distribution: the logistic
+ * function reflected over the y axis.  Well-conditioned for positive
+ * inputs and small negative inputs; ill-conditioned for large negative
+ * inputs.
+ */
+STATIC double
+sf_logistic(double x, double mu, double sigma)
+{
+  return logistic(-(x - mu)/sigma);
+}
+
+/**
+ * Compute the inverse of the CDF of the Logistic(mu, sigma)
+ * distribution: the logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+icdf_logistic(double p, double mu, double sigma)
+{
+  return mu + sigma*logit(p);
+}
+
+/**
+ * Compute the inverse of the SF of the Logistic(mu, sigma)
+ * distribution: the -logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+isf_logistic(double p, double mu, double sigma)
+{
+  return mu - sigma*logit(p);
+}
+
+/*
+ * LogLogistic(alpha, beta) distribution, supported on (0, +infinity).
+ *
+ * This is the uniform distribution on [0,1] mapped into odds space,
+ * scaled by positive alpha and shaped by positive beta.
+ *
+ * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample.
+ * (Name arises because the pdf has LogLogistic(x; alpha, beta) =
+ * Logistic(log x; log alpha, 1/beta) and mathematicians got their
+ * covariance contravariant.)
+ *
+ * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2
+ *        = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} /
+ *              (1 + (x/e^mu)^{1/sigma})^2
+ * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x}/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x - mu})^{-1/sigma})
+ *        = 1/(1 + e^{-(log x - mu)/sigma})
+ *        = logistic((log x - mu)/sigma)
+ *        = logistic((log x - log alpha)/(1/beta))
+ * sf(x) = 1 - 1/(1 + (x/alpha)^-beta)
+ *       = (x/alpha)^-beta/(1 + (x/alpha)^-beta)
+ *       = 1/((x/alpha)^beta + 1)
+ *       = 1/(1 + (x/alpha)^beta)
+ * icdf(p) = alpha (p/(1 - p))^{1/beta}
+ *         = alpha e^{logit(p)/beta}
+ *         = e^{mu + sigma logit(p)}
+ * isf(p) = alpha ((1 - p)/p)^{1/beta}
+ *        = alpha e^{-logit(p)/beta}
+ *        = e^{mu - sigma logit(p)}
+ */
+
+/**
+ * Compute the CDF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      -beta/[1 + (x/alpha)^{-beta}]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps),
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, around which point it levels off at 1.
+ */
+STATIC double
+cdf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and
+   * d3, of the final quotient.  The exponentiation gives
+   *
+   *    ((1 + d0) x/alpha)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + d0)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1)
+   *    = (x/alpha)^{-beta} (1 + d')
+   *
+   * where d' = (1 + d0)^{-beta} - 1.  If y = (x/alpha)^{-beta},
+   * the denominator is
+   *
+   *    (1 + d2) (1 + (1 + d1) (1 + d') y)
+   *    = (1 + d2) (1 + y + (d1 + d' + d1 d') y)
+   *    = 1 + y + (1 + d2) (d1 + d' + d1 d') y
+   *    = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y))
+   *    = (1 + y) (1 + d''),
+   *
+   * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y).  The
+   * final result is
+   *
+   *    (1 + d3) / [(1 + d2) (1 + d'') (1 + y)]
+   *    = (1 + d''') / (1 + y)
+   *
+   * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2
+   * (which may not be the case for very large beta).  This
+   * relative error is therefore bounded by
+   *
+   *    |d'''|
+   *    <= 2|d3 - d''|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')|
+   *     = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'|
+   *      <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'|
+   *         + 2|d2 d1 d'|
+   *      <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|.
+   *
+   * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps,
+   * until it levels off at 1.
+   */
+  return 1/(1 + pow(x/alpha, -beta));
+}
+
+/**
+ * Compute the SF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      beta/[1 + (x/alpha)^beta]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps)
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, beyond which point it grows unbounded.
+ */
+STATIC double
+sf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * The error analysis here is essentially the same as in
+   * cdf_log_logistic, except that rather than levelling off at
+   * 1, |(1 + d0)^beta - 1| grows unbounded.
+   */
+  return 1/(1 + pow(x/alpha, beta));
+}
+
+/**
+ * Compute the inverse of the CDF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and beta near 0 with
+ * condition number 1/[beta (1 - p)].
+ */
+STATIC double
+icdf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow(p/(1 - p), 1/beta);
+}
+
+/**
+ * Compute the inverse of the SF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and for large beta, with
+ * condition number -1/[beta (1 - p)].
+ */
+STATIC double
+isf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow((1 - p)/p, 1/beta);
+}
+
+/*
+ * Weibull(lambda, k) distribution, supported on (0, +infinity).
+ *
+ * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k}
+ * cdf(x) = 1 - e^{-(x/lambda)^k}
+ * icdf(p) = lambda * (-log (1 - p))^{1/k}
+ * sf(x) = e^{-(x/lambda)^k}
+ * isf(p) = lambda * (-log p)^{1/k}
+ */
+
+/**
+ * Compute the CDF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1]
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+cdf_weibull(double x, double lambda, double k)
+{
+  return -expm1(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the SF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+sf_weibull(double x, double lambda, double k)
+{
+  return exp(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the inverse of the CDF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 1, and for k near 0;
+ * condition number is
+ *
+ *      (p/(1 - p))/(k log(1 - p)).
+ */
+STATIC double
+icdf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log1p(-p), 1/k);
+}
+
+/**
+ * Compute the inverse of the SF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 0, and for k near 0;
+ * condition number is
+ *
+ *      1/(k log(p)).
+ */
+STATIC double
+isf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log(p), 1/k);
+}
+
+/*
+ * GeneralizedPareto(mu, sigma, xi), supported on (mu, +infinity) for
+ * nonnegative xi, or (mu, mu - sigma/xi) for negative xi.
+ *
+ * Samples:
+ * = mu - sigma log U, if xi = 0;
+ * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0,
+ * where U is uniform on (0,1].
+ * = mu + sigma (e^{xi X} - 1)/xi,
+ * where X has standard exponential distribution.
+ *
+ * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)}
+ * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi}
+ *        = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi}
+ *        --> 1 - e^{-(x - mu)/sigma}  as  xi --> 0
+ * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi}
+ *       --> e^{-(x - mu)/sigma}  as  xi --> 0
+ * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi
+ *         = mu + sigma*expm1(-xi log p)/xi
+ *         --> mu + sigma*log p  as  xi --> 0
+ * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi
+ *        = mu + sigma*expm1(-xi log1p(-p))/xi
+ *        --> mu + sigma*log1p(-p)  as  xi --> 0
+ */
+
+/**
+ * Compute the CDF of the GeneralizedPareto(mu, sigma, xi)
+ * distribution.  Well-conditioned everywhere.  For standard
+ * distribution (mu=0, sigma=1), condition number
+ *
+ *      (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1)
+ *
+ * is bounded by 1, attained only at x = 0.
+ */
+STATIC double
+cdf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  /*
+   * log(1 + xi x_0)/xi
+   * = (-1/xi) \sum_{n=1}^infinity (-xi x_0)^n/n
+   * = (-1/xi) (-xi x_0 + \sum_{n=2}^infinity (-xi x_0)^n/n)
+   * = x_0 - (1/xi) \sum_{n=2}^infinity (-xi x_0)^n/n
+   * = x_0 - x_0 \sum_{n=2}^infinity (-xi x_0)^{n-1}/n
+   * = x_0 (1 - d),
+   *
+   * where d = \sum_{n=2}^infinity (-xi x_0)^{n-1}/n.  If |xi| <
+   * eps/4|x_0|, then
+   *
+   * |d| <= \sum_{n=2}^infinity (eps/4)^{n-1}/n
+   *     <= \sum_{n=2}^infinity (eps/4)^{n-1}
+   *      = \sum_{n=1}^infinity (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^infinity (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2
+   *
+   * for any 0 < eps < 2.  Thus, the relative error of x_0 from
+   * log(1 + xi x_0)/xi is bounded by eps.
+   */
+  if (fabs(xi) < 1e-17/x_0)
+    return -expm1(-x_0);
+  else
+    return -expm1(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution.
+ * For standard distribution (mu=0, sigma=1), ill-conditioned for xi
+ * near 0; condition number
+ *
+ *      -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi}
+ *      = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi}
+ *      = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi}
+ *      = -x/(1 + x xi)
+ *
+ * is bounded by 1/xi.
+ */
+STATIC double
+sf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  if (fabs(xi) < 1e-17/x_0)
+    return exp(-x_0);
+  else
+    return exp(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; condition number is
+ *
+ *      xi (p/(1 - p))/(1 - (1 - p)^xi)
+ */
+STATIC double
+icdf_genpareto(double p, double mu, double sigma, double xi)
+{
+  /*
+   * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi
+   * for xi near zero (note f(xi) --> -log U as xi --> 0), write
+   * the absolutely convergent Taylor expansion
+   *
+   * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^infinity (-xi log U)^n/n!
+   *       = -log U + (1/xi)*\sum_{n=2}^infinity (-xi log U)^n/n!
+   *       = -log U + \sum_{n=2}^infinity xi^{n-1} (-log U)^n/n!
+   *       = -log U - log U \sum_{n=2}^infinity (-xi log U)^{n-1}/n!
+   *       = -log U (1 + \sum_{n=2}^infinity (-xi log U)^{n-1}/n!).
+   *
+   * Let d = \sum_{n=2}^infinity (-xi log U)^{n-1}/n!.  What do we
+   * lose if we discard it and use -log U as an approximation to
+   * f(xi)?  If |xi| < eps/-4log U, then
+   *
+   * |d| <= \sum_{n=2}^infinity |xi log U|^{n-1}/n!
+   *     <= \sum_{n=2}^infinity (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^infinity (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^infinity (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/-2log U,
+   * f(xi) = -log U (1 + d) for |d| <= eps/2.  |d| is the
+   * relative error of f(xi) from -log U; from this bound, the
+   * relative error of -log U from f(xi) is at most (eps/2)/(1 -
+   * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 <
+   * eps < 1.  Since -log U < 1000 for all U in (0, 1] in
+   * binary64 floating-point, we can safely cut xi off at 1e-20 <
+   * eps/4000 and attain <1ulp error from series truncation.
+   */
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log1p(-p);
+  else
+    return mu + sigma*expm1(-xi*log1p(-p))/xi;
+}
+
+/**
+ * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; conditon number is
+ *
+ *      -xi/(1 - p^{-xi})
+ */
+STATIC double
+isf_genpareto(double p, double mu, double sigma, double xi)
+{
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log(p);
+  else
+    return mu + sigma*expm1(-xi*log(p))/xi;
+}
+
+/*******************************************************************/
+
+/**
+ * Deterministic samplers, parametrized by uniform integer and (0,1]
+ * samples.  No guarantees are made about _which_ mapping from the
+ * integer and (0,1] samples these use; all that is guaranteed is the
+ * distribution of the outputs conditioned on a uniform distribution on
+ * the inputs.  The automatic tests in test_prob_distr.c double-check
+ * the particular mappings we use.
+ *
+ * Beware: Unlike random_uniform_01(), these are not guaranteed to be
+ * supported on all possible outputs.  See Ilya Mironov, `On the
+ * Significance of the Least Significant Bits for Differential
+ * Privacy', for an example of what can go wrong if you try to use
+ * these to conceal information from an adversary but you expose the
+ * specific full-precision floating-point values.
+ *
+ * Note: None of these samplers use rejection sampling; they are all
+ * essentially inverse-CDF transforms with tweaks.  If you were to add,
+ * say, a Gamma sampler with the Marsaglia-Tsang method, you would have
+ * to parametrize it by a potentially infinite stream of uniform (and
+ * perhaps normal) samples rather than a fixed number, which doesn't
+ * make for quite as nice automatic testing as for these.
+ */
+
+/**
+ * Deterministically sample from the interval [a, b], indexed by a
+ * uniform random floating-point number p0 in (0, 1].
+ *
+ * Note that even if p0 is nonzero, the result may be equal to a, if
+ * ulp(a)/2 is nonnegligible, e.g. if a = 1.  For maximum resolution,
+ * arrange |a| <= |b|.
+ */
+STATIC double
+sample_uniform_interval(double p0, double a, double b)
+{
+  /*
+   * XXX Prove that the distribution is, in fact, uniform on
+   * [a,b], particularly around p0 = 1, or at least has very
+   * small deviation from uniform, quantified appropriately
+   * (e.g., like in Monahan 1984, or by KL divergence).  It
+   * almost certainly does but it would be nice to quantify the
+   * error.
+   */
+  if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) {
+    /*
+     * When ab < 0, (1 - t) a + t b is monotonic, since for
+     * a <= b it is a sum of nondecreasing functions of t,
+     * and for b <= a, of nonincreasing functions of t.
+     * Further, clearly at 0 and 1 it attains a and b,
+     * respectively.  Hence it is bounded within [a, b].
+     */
+    return (1 - p0)*a + p0*b;
+  } else {
+    /*
+     * a + (b - a) t is monotonic -- it is obviously a
+     * nondecreasing function of t for a <= b.  Further, it
+     * attains a at 0, and while it may overshoot b at 1,
+     * we have a
+     *
+     * Theorem.  If 0 <= t < 1, then the floating-point
+     * evaluation of a + (b - a) t is bounded in [a, b].
+     *
+     * Lemma 1.  If 0 <= t < 1 is a floating-point number,
+     * then for any normal floating-point number x except
+     * the smallest in magnitude, |round(x*t)| < |x|.
+     *
+     * Proof.  WLOG, assume x >= 0.  Since the rounding
+     * function and t |---> x*t are nondecreasing, their
+     * composition t |---> round(x*t) is also
+     * nondecreasing, so it suffices to consider the
+     * largest floating-point number below 1, in particular
+     * t = 1 - ulp(1)/2.
+     *
+     * Case I: If x is a power of two, then the next
+     * floating-point number below x is x - ulp(x)/2 = x -
+     * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t
+     * is a floating-point number, multiplication is exact,
+     * and thus round(x*t) = x*t < x.
+     *
+     * Case II: If x is not a power of two, then the
+     * greatest lower bound of real numbers rounded to x is
+     * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2,
+     * where T(X) is the largest power of two below x.
+     * Anything below this bound is rounded to a
+     * floating-point number smaller than x, and x*t = x*(1
+     * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2
+     * since T(x) < x, so round(x*t) < x*t < x.  QED.
+     *
+     * Lemma 2.  If x and y are subnormal, then round(x +
+     * y) = x + y.
+     *
+     * Proof.  It is a matter of adding the significands,
+     * since if we treat subnormals as having an implicit
+     * zero bit before the `binary' point, their exponents
+     * are all the same.  There is at most one carry/borrow
+     * bit, which can always be acommodated either in a
+     * subnormal, or, at largest, in the implicit one bit
+     * of a normal.
+     *
+     * Lemma 3.  Let x and y be floating-point numbers.  If
+     * round(x - y) is subnormal or zero, then it is equal
+     * to x - y.
+     *
+     * Proof.  Case I (equal): round(x - y) = 0 iff x = y;
+     * hence if round(x - y) = 0, then round(x - y) = 0 = x
+     * - y.
+     *
+     * Case II (subnormal/subnormal): If x and y are both
+     * subnormal, this follows directly from Lemma 2.
+     *
+     * Case IIIa (normal/subnormal): If x is normal and y
+     * is subnormal, then x and y must share sign, or else
+     * x - y would be larger than x and thus rounded to
+     * normal.  If s is the smallest normal positive
+     * floating-point number, |x| < 2s since by
+     * construction 2s - |y| is normal for all subnormal y.
+     * This means that x and y must have the same exponent,
+     * so the difference is the difference of significands,
+     * which is exact.
+     *
+     * Case IIIb (subnormal/normal): Same as case IIIa for
+     * -(y - x).
+     *
+     * Case IV (normal/normal): If x and y are both normal,
+     * then they must share sign, or else x - y would be
+     * larger than x and thus rounded to normal.  Note that
+     * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <=
+     * -|x| but -|x| is normal like x.  Also, |x|/2 < |y|:
+     * if |x|/2 is subnormal, it must hold because y is
+     * normal; if |x|/2 is normal, then |x|/2 >= s, so
+     * since |x| - |y| < s,
+     *
+     *  |x|/2 = |x| - |x|/2 <= |x| - s <= |y|;
+     *
+     * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz
+     * lemma, round(x - y) = x - y.  QED.
+     *
+     * Proof of theorem.  WLOG, assume 0 <= a <= b.  Since
+     * round(a + round(round(b - a)*t) is nondecreasing in
+     * t and attains a at 0, the lower end of the bound is
+     * trivial; we must show the upper end of the bound
+     * strictly.  It suffices to show this for the largest
+     * floating-point number below 1, namely 1 - ulp(1)/2.
+     *
+     * Case I: round(b - a) is normal.  Then it is at most
+     * the smallest floating-point number above b - a.  By
+     * Lemma 1, round(round(b - a)*t) < round(b - a).
+     * Since the inequality is strict, and since
+     * round(round(b - a)*t) is a floating-point number
+     * below round(b - a), and since there are no
+     * floating-point numbers between b - a and round(b -
+     * a), we must have round(round(b - a)*t) < b - a.
+     * Then since y |---> round(a + y) is nondecreasing, we
+     * must have
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b) = b.
+     *
+     * Case II: round(b - a) is subnormal.  In this case,
+     * Lemma 1 falls apart -- we are not guaranteed the
+     * strict inequality.  However, by Lemma 3, the
+     * difference is exact: round(b - a) = b - a.  Thus,
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + round((b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b)
+     *   = b,
+     *
+     * QED.
+     */
+
+    /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal.  */
+    if (p0 >= 1)
+      return b;
+    return a + (b - a)*p0;
+  }
+}
+
+/**
+ * Deterministically sample from the standard logistic distribution,
+ * indexed by a uniform random 32-bit integer s and uniform random
+ * floating-point numbers t and p0 in (0, 1].
+ */
+STATIC double
+sample_logistic(uint32_t s, double t, double p0)
+{
+  double sign = (s & 1) ? -1 : +1;
+  double r;
+
+  /*
+   * We carve up the interval (0, 1) into subregions to compute
+   * the inverse CDF precisely:
+   *
+   * A = (0, 1/(1 + e)] ---> (-infinity, -1]
+   * B = [1/(1 + e), 1/2] ---> [-1, 0]
+   * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1]
+   * D = [1 - 1/(1 + e), 1) ---> [1, +infinity)
+   *
+   * Cases D and C are mirror images of cases A and B,
+   * respectively, so we choose between them by the sign chosen
+   * by a fair coin toss.  We choose between cases A and B by a
+   * coin toss weighted by
+   *
+   *    2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2):
+   *
+   * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)]
+   * sample p; if it comes up tails, scale p0 into a uniform (0,
+   * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p =
+   * 1/2 - p0.
+   */
+  if (t <= 2/(1 + exp(1))) {
+    /* p uniform in (0, 1/(1 + e)], represented by p.  */
+    p0 /= 1 + exp(1);
+    r = logit(p0);
+  } else {
+    /*
+     * p uniform in [1/(1 + e), 1/2), actually represented
+     * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so
+     * that p = 1/2 - p.
+     */
+    p0 *= 0.5 - 1/(1 + exp(1));
+    r = logithalf(p0);
+  }
+
+  /*
+   * We have chosen from the negative half of the standard
+   * logistic distribution, which is symmetric with the positive
+   * half.  Now use the sign to choose uniformly between them.
+   */
+  return sign*r;
+}
+
+/**
+ * Deterministically sample from the logistic distribution scaled by
+ * sigma and translated by mu.
+ */
+static double
+sample_logistic_locscale(uint32_t s, double t, double p0, double mu,
+    double sigma)
+{
+
+  return mu + sigma*sample_logistic(s, t, p0);
+}
+
+/**
+ * Deterministically sample from the standard log-logistic
+ * distribution, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_log_logistic(uint32_t s, double p0)
+{
+
+  /*
+   * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the
+   * condition numbers of the icdf and the isf coincide at 1/2.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0) {
+    /* p = p0 in (0, 1/2] */
+    return p0/(1 - p0);
+  } else {
+    /* p = 1 - p0 in [1/2, 1) */
+    return (1 - p0)/p0;
+  }
+}
+
+/**
+ * Deterministically sample from the log-logistic distribution with
+ * scale alpha and shape beta.
+ */
+static double
+sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha,
+    double beta)
+{
+  double x = sample_log_logistic(s, p0);
+
+  return alpha*pow(x, 1/beta);
+}
+
+/**
+ * Deterministically sample from the standard exponential distribution,
+ * indexed by a uniform random 32-bit integer s and a uniform random
+ * floating-point number p0 in (0, 1].
+ */
+static double
+sample_exponential(uint32_t s, double p0)
+{
+  /*
+   * We would like to evaluate log(p) for p near 0, and log1p(-p)
+   * for p near 1.  Simply carve the interval into (0, 1/2] and
+   * [1/2, 1) by a fair coin toss.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0)
+    /* p = p0 in (0, 1/2] */
+    return -log(p0);
+  else
+    /* p = 1 - p0 in [1/2, 1) */
+    return -log1p(-p0);
+}
+
+/**
+ * Deterministically sample from a Weibull distribution with scale
+ * lambda and shape k -- just an exponential with a shape parameter in
+ * addition to a scale parameter.  (Yes, lambda really is the scale,
+ * _not_ the rate.)
+ */
+STATIC double
+sample_weibull(uint32_t s, double p0, double lambda, double k)
+{
+
+  return lambda*pow(sample_exponential(s, p0), 1/k);
+}
+
+/**
+ * Deterministically sample from the generalized Pareto distribution
+ * with shape xi, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_genpareto(uint32_t s, double p0, double xi)
+{
+  double x = sample_exponential(s, p0);
+
+  /*
+   * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the
+   * absolutely convergent Taylor series
+   *
+   * f(x) = (1/xi) (xi x + \sum_{n=2}^infinity (xi x)^n/n!)
+   *      = x + (1/xi) \sum_{n=2}^infinity (xi x)^n/n!
+   *      = x + \sum_{n=2}^infinity xi^{n-1} x^n/n!
+   *      = x + x \sum_{n=2}^infinity (xi x)^{n-1}/n!
+   *      = x (1 + \sum_{n=2}^infinity (xi x)^{n-1}/n!).
+   *
+   * d = \sum_{n=2}^infinity (xi x)^{n-1}/n! is the relative error
+   * of f(x) from x.  If |xi| < eps/4x, then
+   *
+   * |d| <= \sum_{n=2}^infinity |xi x|^{n-1}/n!
+   *     <= \sum_{n=2}^infinity (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^infinity (eps/4)
+   *      = (eps/4) \sum_{n=0}^infinity (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/2x, f(xi)
+   * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'|
+   * <= eps.  What bound should we use for x?
+   *
+   * - If x is exponentially distributed, x > 200 with
+   *   probability below e^{-200} << 2^{-256}, i.e. never.
+   *
+   * - If x is computed by -log(U) for U in (0, 1], x is
+   *   guaranteed to be below 1000 in IEEE 754 binary64
+   *   floating-point.
+   *
+   * We can safely cut xi off at 1e-20 < eps/4000 and attain an
+   * error bounded by 0.5 ulp for this expression.
+   */
+  return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi);
+}
+
+/**
+ * Deterministically sample from a generalized Pareto distribution with
+ * shape xi, scaled by sigma and translated by mu.
+ */
+static double
+sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma,
+    double xi)
+{
+
+  return mu + sigma*sample_genpareto(s, p0, xi);
+}
+
+/**
+ * Deterministically sample from the geometric distribution with
+ * per-trial success probability p.
+ **/
+// clang-format off
+/*
+ * XXX Quantify the error (KL divergence?) of this
+ * ceiling-of-exponential sampler from a true geometric distribution,
+ * which we could get by rejection sampling.  Relevant papers:
+ *
+ *      John F. Monahan, `Accuracy in Random Number Generation',
+ *      Mathematics of Computation 45(172), October 1984, pp. 559--568.
+https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf
+ *      Karl Bringmann and Tobias Friedrich, `Exact and Efficient
+ *      Generation of Geometric Random Variates and Random Graphs', in
+ *      Proceedings of the 40th International Colloaquium on Automata,
+ *      Languages, and Programming -- ICALP 2013, Springer LNCS 7965,
+ *      pp.267--278.
+ *      https://doi.org/10.1007/978-3-642-39206-1_23
+ *      https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf
+ */
+// clang-format on
+static double
+sample_geometric(uint32_t s, double p0, double p)
+{
+  double x = sample_exponential(s, p0);
+
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1)
+    return 1;
+
+  return ceil(-x/log1p(-p));
+}
+
+/*******************************************************************/
+
+/** Public API for probability distributions:
+ *
+ *  These are wrapper functions on top of the various probability distribution
+ *  operations using the generic <b>dist</b> structure.
+
+ *  These are the functions that should be used by consumers of this API.
+ */
+
+/** Returns the name of the distribution in <b>dist</b>. */
+const char *
+dist_name(const struct dist_t *dist)
+{
+  return dist->ops->name;
+}
+
+/* Sample a value from <b>dist</b> and return it. */
+double
+dist_sample(const struct dist_t *dist)
+{
+  return dist->ops->sample(dist);
+}
+
+/** Compute the CDF of <b>dist</b> at <b>x</b>. */
+double
+dist_cdf(const struct dist_t *dist, double x)
+{
+  return dist->ops->cdf(dist, x);
+}
+
+/** Compute the SF (Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_sf(const struct dist_t *dist, double x)
+{
+  return dist->ops->sf(dist, x);
+}
+
+/** Compute the iCDF (Inverse CDF) of <b>dist</b> at <b>x</b>. */
+double
+dist_icdf(const struct dist_t *dist, double p)
+{
+  return dist->ops->icdf(dist, p);
+}
+
+/** Compute the iSF (Inverse Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_isf(const struct dist_t *dist, double p)
+{
+  return dist->ops->isf(dist, p);
+}
+
+/** Functions for uniform distribution */
+
+static double
+uniform_sample(const struct dist_t *dist)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  double p0 = random_uniform_01();
+
+  return sample_uniform_interval(p0, U->a, U->b);
+}
+
+static double
+uniform_cdf(const struct dist_t *dist, double x)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  if (x < U->a)
+    return 0;
+  else if (x < U->b)
+    return (x - U->a)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_sf(const struct dist_t *dist, double x)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+
+  if (x > U->b)
+    return 0;
+  else if (x > U->a)
+    return (U->b - x)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_icdf(const struct dist_t *dist, double p)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p)));
+}
+
+static double
+uniform_isf(const struct dist_t *dist, double p)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p)));
+}
+
+const struct dist_ops_t uniform_ops = {
+  .name = "uniform",
+  .sample = uniform_sample,
+  .cdf = uniform_cdf,
+  .sf = uniform_sf,
+  .icdf = uniform_icdf,
+  .isf = uniform_isf,
+};
+
+/*******************************************************************/
+
+/** Private functions for each probability distribution. */
+
+/** Functions for logistic distribution: */
+
+static double
+logistic_sample(const struct dist_t *dist)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double t = random_uniform_01();
+  double p0 = random_uniform_01();
+
+  return sample_logistic_locscale(s, t, p0, L->mu, L->sigma);
+}
+
+static double
+logistic_cdf(const struct dist_t *dist, double x)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return cdf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_sf(const struct dist_t *dist, double x)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return sf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_icdf(const struct dist_t *dist, double p)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return icdf_logistic(p, L->mu, L->sigma);
+}
+
+static double
+logistic_isf(const struct dist_t *dist, double p)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return isf_logistic(p, L->mu, L->sigma);
+}
+
+const struct dist_ops_t logistic_ops = {
+  .name = "logistic",
+  .sample = logistic_sample,
+  .cdf = logistic_cdf,
+  .sf = logistic_sf,
+  .icdf = logistic_icdf,
+  .isf = logistic_isf,
+};
+
+/** Functions for log-logistic distribution: */
+
+static double
+log_logistic_sample(const struct dist_t *dist)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_cdf(const struct dist_t *dist, double x)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return cdf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_sf(const struct dist_t *dist, double x)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return sf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_icdf(const struct dist_t *dist, double p)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return icdf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_isf(const struct dist_t *dist, double p)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return isf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+const struct dist_ops_t log_logistic_ops = {
+  .name = "log logistic",
+  .sample = log_logistic_sample,
+  .cdf = log_logistic_cdf,
+  .sf = log_logistic_sf,
+  .icdf = log_logistic_icdf,
+  .isf = log_logistic_isf,
+};
+
+/** Functions for Weibull distribution */
+
+static double
+weibull_sample(const struct dist_t *dist)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_weibull(s, p0, W->lambda, W->k);
+}
+
+static double
+weibull_cdf(const struct dist_t *dist, double x)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return cdf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_sf(const struct dist_t *dist, double x)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return sf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_icdf(const struct dist_t *dist, double p)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return icdf_weibull(p, W->lambda, W->k);
+}
+
+static double
+weibull_isf(const struct dist_t *dist, double p)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return isf_weibull(p, W->lambda, W->k);
+}
+
+const struct dist_ops_t weibull_ops = {
+  .name = "Weibull",
+  .sample = weibull_sample,
+  .cdf = weibull_cdf,
+  .sf = weibull_sf,
+  .icdf = weibull_icdf,
+  .isf = weibull_isf,
+};
+
+/** Functions for generalized Pareto distributions */
+
+static double
+genpareto_sample(const struct dist_t *dist)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_cdf(const struct dist_t *dist, double x)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_sf(const struct dist_t *dist, double x)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return sf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_icdf(const struct dist_t *dist, double p)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_isf(const struct dist_t *dist, double p)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return isf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+const struct dist_ops_t genpareto_ops = {
+  .name = "generalized Pareto",
+  .sample = genpareto_sample,
+  .cdf = genpareto_cdf,
+  .sf = genpareto_sf,
+  .icdf = genpareto_icdf,
+  .isf = genpareto_isf,
+};
+
+/** Functions for geometric distribution on number of trials before success */
+
+static double
+geometric_sample(const struct dist_t *dist)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_geometric(s, p0, G->p);
+}
+
+static double
+geometric_cdf(const struct dist_t *dist, double x)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */
+  return -expm1(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_sf(const struct dist_t *dist, double x)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */
+  return exp(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_icdf(const struct dist_t *dist, double p)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  return log1p(-p)/log1p(-G->p);
+}
+
+static double
+geometric_isf(const struct dist_t *dist, double p)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  return log(p)/log1p(-G->p);
+}
+
+const struct dist_ops_t geometric_ops = {
+  .name = "geometric (1-based)",
+  .sample = geometric_sample,
+  .cdf = geometric_cdf,
+  .sf = geometric_sf,
+  .icdf = geometric_icdf,
+  .isf = geometric_isf,
+};
diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h
new file mode 100644
index 0000000000..a036073b93
--- /dev/null
+++ b/src/lib/math/prob_distr.h
@@ -0,0 +1,253 @@
+
+/**
+ * \file prob_distr.h
+ *
+ * \brief Header for prob_distr.c
+ **/
+
+#ifndef TOR_PROB_DISTR_H
+#define TOR_PROB_DISTR_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * Container for distribution parameters for sampling, CDF, &c.
+ */
+struct dist_t {
+  const struct dist_ops_t *ops;
+};
+
+/**
+ * Untyped initializer element for struct dist_t using the specified
+ * struct dist_ops_t pointer.  Don't actually use this directly -- use
+ * the type-specific macro built out of DIST_BASE_TYPED below -- but if
+ * you did use this directly, it would be something like:
+ *
+ *   struct weibull mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     .lambda = ...,
+ *     .k = ...,
+ *   };
+ *
+ * Note there is NO COMPILER FEEDBACK if you accidentally do something
+ * like
+ *
+ *   struct geometric mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     ...
+ *   };
+ */
+#define DIST_BASE(OPS)  { .ops = (OPS) }
+
+/** A compile-time type-checking macro for use with DIST_BASE_TYPED.
+ *
+ *  This macro works by checking that &OBJ is a pointer type that is the same
+ *  type (except for qualifiers) as (const TYPE *)&OBJ. It's a C constraint
+ *  violation (which requires a diagnostic) if two pointers are different types
+ *  and are subtracted. The sizeof() forces compile-time evaluation, and the
+ *  multiplication by zero is to discard the result of the sizeof() from the
+ *  expression.
+ *
+ *  We define this conditionally to suppress false positives from
+ *  Coverity, which gets confused by the sizeof business.
+ */
+#ifdef __COVERITY__
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0
+#else
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \
+  (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ)))
+#endif /* defined(__COVERITY__) */
+
+/**
+* Typed initializer element for struct dist_t using the specified struct
+* dist_ops_t pointer.  Don't actually use this directly -- use a
+* type-specific macro built out of it -- but if you did use this
+* directly, it would be something like:
+*
+*     struct weibull mydist = {
+*       DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull_t),
+*       .lambda = ...,
+*       .k = ...,
+*     };
+*
+* If you want to define a distribution type, define a canonical set of
+* operations and define a type-specific initializer element like so:
+*
+*     struct foo_t {
+*       struct dist_t base;
+*       int omega;
+*       double tau;
+*       double phi;
+*     };
+*
+*     struct dist_ops_t foo_ops = ...;
+*
+*     #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo_t)
+*
+* Then users can do:
+*
+*     struct foo_t mydist = {
+*       FOO(mydist),
+*       .omega = ...,
+*       .tau = ...,
+*       .phi = ...,
+*     };
+*
+* If you accidentally write
+*
+*     struct bar_t mydist = {
+*       FOO(mydist),
+*       ...
+*     };
+*
+* then the compiler will report a type mismatch in the sizeof
+* expression, which otherwise evaporates at runtime.
+*/
+#define DIST_BASE_TYPED(OPS, OBJ, TYPE)                         \
+  DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE))
+
+/**
+ * Generic operations on distributions.  These simply defer to the
+ * corresponding dist_ops_t function.  In the parlance of C++, these call
+ * virtual member functions.
+ */
+const char *dist_name(const struct dist_t *);
+double dist_sample(const struct dist_t *);
+double dist_cdf(const struct dist_t *, double x);
+double dist_sf(const struct dist_t *, double x);
+double dist_icdf(const struct dist_t *, double p);
+double dist_isf(const struct dist_t *, double p);
+
+/**
+ * Set of operations on a potentially parametric family of
+ * distributions.  In the parlance of C++, this would be called a
+ * `vtable' and the members are virtual member functions.
+ */
+struct dist_ops_t {
+  const char *name;
+  double (*sample)(const struct dist_t *);
+  double (*cdf)(const struct dist_t *, double x);
+  double (*sf)(const struct dist_t *, double x);
+  double (*icdf)(const struct dist_t *, double p);
+  double (*isf)(const struct dist_t *, double p);
+};
+
+/* Geometric distribution on positive number of trials before first success */
+
+struct geometric_t {
+  struct dist_t base;
+  double p; /* success probability */
+};
+
+extern const struct dist_ops_t geometric_ops;
+
+#define GEOMETRIC(OBJ)                                      \
+  DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric_t)
+
+/* Pareto distribution */
+
+struct genpareto_t {
+  struct dist_t base;
+  double mu;
+  double sigma;
+  double xi;
+};
+
+extern const struct dist_ops_t genpareto_ops;
+
+#define GENPARETO(OBJ)                                      \
+  DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto_t)
+
+/* Weibull distribution */
+
+struct weibull_t {
+  struct dist_t base;
+  double lambda;
+  double k;
+};
+
+extern const struct dist_ops_t weibull_ops;
+
+#define WEIBULL(OBJ)                                    \
+  DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull_t)
+
+/* Log-logistic distribution */
+
+struct log_logistic_t {
+  struct dist_t base;
+  double alpha;
+  double beta;
+};
+
+extern const struct dist_ops_t log_logistic_ops;
+
+#define LOG_LOGISTIC(OBJ)                                       \
+  DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic_t)
+
+/* Logistic distribution */
+
+struct logistic_t {
+  struct dist_t base;
+  double mu;
+  double sigma;
+};
+
+extern const struct dist_ops_t logistic_ops;
+
+#define LOGISTIC(OBJ)                                   \
+  DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic_t)
+
+/* Uniform distribution */
+
+struct uniform_t {
+  struct dist_t base;
+  double a;
+  double b;
+};
+
+extern const struct dist_ops_t uniform_ops;
+
+#define UNIFORM(OBJ)                                    \
+  DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform_t)
+
+/** Only by unittests */
+
+#ifdef PROB_DISTR_PRIVATE
+
+STATIC double logithalf(double p0);
+STATIC double logit(double p);
+
+STATIC double random_uniform_01(void);
+
+STATIC double logistic(double x);
+STATIC double cdf_logistic(double x, double mu, double sigma);
+STATIC double sf_logistic(double x, double mu, double sigma);
+STATIC double icdf_logistic(double p, double mu, double sigma);
+STATIC double isf_logistic(double p, double mu, double sigma);
+STATIC double sample_logistic(uint32_t s, double t, double p0);
+
+STATIC double cdf_log_logistic(double x, double alpha, double beta);
+STATIC double sf_log_logistic(double x, double alpha, double beta);
+STATIC double icdf_log_logistic(double p, double alpha, double beta);
+STATIC double isf_log_logistic(double p, double alpha, double beta);
+STATIC double sample_log_logistic(uint32_t s, double p0);
+
+STATIC double cdf_weibull(double x, double lambda, double k);
+STATIC double sf_weibull(double x, double lambda, double k);
+STATIC double icdf_weibull(double p, double lambda, double k);
+STATIC double isf_weibull(double p, double lambda, double k);
+STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k);
+
+STATIC double sample_uniform_interval(double p0, double a, double b);
+
+STATIC double cdf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double sf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double icdf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double isf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double sample_genpareto(uint32_t s, double p0, double xi);
+
+#endif /* defined(PROB_DISTR_PRIVATE) */
+
+#endif /* !defined(TOR_PROB_DISTR_H) */
diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include
index 814652a93c..a1edaf2231 100644
--- a/src/lib/memarea/.may_include
+++ b/src/lib/memarea/.may_include
@@ -1,7 +1,7 @@
 orconfig.h
 lib/arch/*.h
 lib/cc/*.h
-lib/container/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/memarea/*.h
+lib/smartlist_core/*.h
\ No newline at end of file
diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am
index 94343dcead..83fb99ec73 100644
--- a/src/lib/memarea/include.am
+++ b/src/lib/memarea/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-memarea-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_memarea_a_SOURCES =			\
 	src/lib/memarea/memarea.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_memarea_testing_a_SOURCES = \
 src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/memarea/memarea.h
diff --git a/src/lib/memarea/lib_memarea.md b/src/lib/memarea/lib_memarea.md
new file mode 100644
index 0000000000..fe5cb8293f
--- /dev/null
+++ b/src/lib/memarea/lib_memarea.md
@@ -0,0 +1,28 @@
+@dir /lib/memarea
+@brief lib/memarea: A fast arena-style allocator.
+
+This module has a fast "arena" style allocator, where memory is freed all at
+once.  This kind of allocation is very fast and avoids fragmentation, at the
+expense of requiring all the data to be freed at the same time.  We use this
+for parsing and diff calculations.
+
+It's often handy to allocate a large number of tiny objects, all of which
+need to disappear at the same time.  You can do this in tor using the
+memarea.c abstraction, which uses a set of grow-only buffers for allocation,
+and only supports a single "free" operation at the end.
+
+Using memareas also helps you avoid memory fragmentation.  You see, some libc
+malloc implementations perform badly on the case where a large number of
+small temporary objects are allocated at the same time as a few long-lived
+objects of similar size.  But if you use tor_malloc() for the long-lived ones
+and a memarea for the temporary object, the malloc implementation is likelier
+to do better.
+
+To create a new memarea, use `memarea_new()`.  To drop all the storage from a
+memarea, and invalidate its pointers, use `memarea_drop_all()`.
+
+The allocation functions `memarea_alloc()`, `memarea_alloc_zero()`,
+`memarea_memdup()`, `memarea_strdup()`, and `memarea_strndup()` are analogous
+to the similarly-named malloc() functions.  There is intentionally no
+`memarea_free()` or `memarea_realloc()`.
+
diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c
index 486673116c..4d26c20eeb 100644
--- a/src/lib/memarea/memarea.c
+++ b/src/lib/memarea/memarea.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2019, The Tor Project, Inc. */
+/* Copyright (c) 2008-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,8 @@
 
 #include "lib/arch/bytes.h"
 #include "lib/cc/torint.h"
-#include "lib/container/smartlist.h"
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
@@ -38,7 +39,7 @@
 #elif MEMAREA_ALIGN == 8
 #define MEMAREA_ALIGN_MASK ((uintptr_t)7)
 #else
-#error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff."
+#error "void* is neither 4 nor 8 bytes long."
 #endif /* MEMAREA_ALIGN == 4 || ... */
 
 #if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER)
@@ -67,7 +68,7 @@
   uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]);     \
   tor_assert(sent_val == SENTINEL_VAL);                                 \
   STMT_END
-#else /* !(defined(USE_SENTINELS)) */
+#else /* !defined(USE_SENTINELS) */
 #define SENTINEL_LEN 0
 #define SET_SENTINEL(chunk) STMT_NIL
 #define CHECK_SENTINEL(chunk) STMT_NIL
@@ -314,7 +315,7 @@ memarea_assert_ok(memarea_t *area)
   }
 }
 
-#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */
+#else /* defined(DISABLE_MEMORY_SENTINELS) */
 
 struct memarea_t {
   smartlist_t *pieces;
diff --git a/src/lib/memarea/memarea.h b/src/lib/memarea/memarea.h
index 9c23cf62e9..8b5e63e6b3 100644
--- a/src/lib/memarea/memarea.h
+++ b/src/lib/memarea/memarea.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2019, The Tor Project, Inc. */
+/* Copyright (c) 2008-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,9 @@ typedef struct memarea_t memarea_t;
 
 memarea_t *memarea_new(void);
 void memarea_drop_all_(memarea_t *area);
+/** @copydoc memarea_drop_all_
+ *
+ * Additionally, set <b>area</b> to NULL. */
 #define memarea_drop_all(area) \
   do {                                          \
     memarea_drop_all_(area);                    \
diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am
index d1fdde6313..12c1bff72d 100644
--- a/src/lib/meminfo/include.am
+++ b/src/lib/meminfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_meminfo_a_SOURCES =			\
 	src/lib/meminfo/meminfo.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_meminfo_testing_a_SOURCES = \
 src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/meminfo/meminfo.h
diff --git a/src/lib/meminfo/lib_meminfo.md b/src/lib/meminfo/lib_meminfo.md
new file mode 100644
index 0000000000..87f509d648
--- /dev/null
+++ b/src/lib/meminfo/lib_meminfo.md
@@ -0,0 +1,5 @@
+@dir /lib/meminfo
+@brief lib/meminfo: Inspecting malloc() usage.
+
+Only available when malloc() provides mallinfo() or something similar.
+
diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c
index bc3eef8419..b7d991e410 100644
--- a/src/lib/meminfo/meminfo.c
+++ b/src/lib/meminfo/meminfo.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h
index 6f29fe0ea8..36a85c133d 100644
--- a/src/lib/meminfo/meminfo.h
+++ b/src/lib/meminfo/meminfo.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@
 
 MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
 
-#endif
+#endif /* !defined(TOR_MEMINFO_H) */
diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include
index 13b209bbed..6e9af9737a 100644
--- a/src/lib/net/.may_include
+++ b/src/lib/net/.may_include
@@ -1,8 +1,9 @@
 orconfig.h
-siphash.h
-ht.h
+ext/siphash.h
+ext/ht.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
@@ -11,5 +12,7 @@ lib/lock/*.h
 lib/log/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
-lib/malloc/*.h
\ No newline at end of file
+lib/malloc/*.h
+lib/smartlist_core/*.h
diff --git a/src/lib/net/address.c b/src/lib/net/address.c
index 69004ddb0e..6d46f9b955 100644
--- a/src/lib/net/address.c
+++ b/src/lib/net/address.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,6 +40,7 @@
 
 #include "lib/net/address.h"
 #include "lib/net/socket.h"
+#include "lib/cc/ctassert.h"
 #include "lib/container/smartlist.h"
 #include "lib/ctime/di_ops.h"
 #include "lib/log/log.h"
@@ -52,7 +53,7 @@
 #include "lib/string/printf.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
@@ -96,8 +97,9 @@
  * work correctly. Bail out here if we've found a platform where AF_UNSPEC
  * isn't 0. */
 #if AF_UNSPEC != 0
-#error We rely on AF_UNSPEC being 0. Let us know about your platform, please!
+#error "We rely on AF_UNSPEC being 0. Yours isn't. Please tell us more!"
 #endif
+CTASSERT(AF_UNSPEC == 0);
 
 /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a
  * sockaddr object in *<b>sa_out</b> of object size <b>len</b>.  If not enough
@@ -337,7 +339,7 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
       break;
     case AF_INET6:
       /* Shortest addr [ :: ] + \0 */
-      if (len < (3 + (decorate ? 2u : 0u)))
+      if (len < (3u + (decorate ? 2 : 0)))
         return NULL;
 
       if (decorate)
@@ -371,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
  *
  * If <b>accept_regular</b> is set and the address is in neither recognized
  * reverse lookup hostname format, try parsing the address as a regular
- * IPv4 or IPv6 address too.
+ * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or
+ * without square brackets.
  */
 int
 tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
@@ -605,7 +608,8 @@ tor_addr_parse_mask_ports(const char *s,
         family = AF_INET;
         tor_addr_from_ipv4h(addr_out, 0);
       } else if (flags & TAPMP_STAR_IPV6_ONLY) {
-        static char nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
+        static uint8_t nil_bytes[16] =
+          { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
         family = AF_INET6;
         tor_addr_from_ipv6_bytes(addr_out, nil_bytes);
       } else {
@@ -626,7 +630,7 @@ tor_addr_parse_mask_ports(const char *s,
     tor_addr_from_ipv4h(addr_out, 0);
     any_flag = 1;
   } else if (!strcmp(address, "*6") && (flags & TAPMP_EXTENDED_STAR)) {
-    static char nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
+    static uint8_t nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
     family = AF_INET6;
     tor_addr_from_ipv6_bytes(addr_out, nil_bytes);
     any_flag = 1;
@@ -814,8 +818,12 @@ tor_addr_is_loopback(const tor_addr_t *addr)
 
 /* Is addr valid?
  * Checks that addr is non-NULL and not tor_addr_is_null().
- * If for_listening is true, IPv4 addr 0.0.0.0 is allowed.
- * It means "bind to all addresses on the local machine". */
+ * If for_listening is true, all IPv4 and IPv6 addresses are valid, including
+ * 0.0.0.0 (for IPv4) and :: (for IPv6). When listening, these addresses mean
+ * "bind to all addresses on the local machine".
+ * Otherwise, 0.0.0.0 and ::  are invalid, because they are null addresses.
+ * All unspecified and unix addresses are invalid, regardless of for_listening.
+ */
 int
 tor_addr_is_valid(const tor_addr_t *addr, int for_listening)
 {
@@ -824,10 +832,11 @@ tor_addr_is_valid(const tor_addr_t *addr, int for_listening)
     return 0;
   }
 
-  /* Only allow IPv4 0.0.0.0 for_listening. */
-  if (for_listening && addr->family == AF_INET
-      && tor_addr_to_ipv4h(addr) == 0) {
-    return 1;
+  /* Allow all IPv4 and IPv6 addresses, when for_listening is true */
+  if (for_listening) {
+    if (addr->family == AF_INET || addr->family == AF_INET6) {
+      return 1;
+    }
   }
 
   /* Otherwise, the address is valid if it's not tor_addr_is_null() */
@@ -879,7 +888,7 @@ tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr)
 /** Set <b>dest</b> to equal the IPv6 address in the 16 bytes at
  * <b>ipv6_bytes</b>. */
 void
-tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *ipv6_bytes)
+tor_addr_from_ipv6_bytes(tor_addr_t *dest, const uint8_t *ipv6_bytes)
 {
   tor_assert(dest);
   tor_assert(ipv6_bytes);
@@ -892,7 +901,21 @@ tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *ipv6_bytes)
 void
 tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6)
 {
-  tor_addr_from_ipv6_bytes(dest, (const char*)in6->s6_addr);
+  tor_addr_from_ipv6_bytes(dest, in6->s6_addr);
+}
+
+/** Set the 16 bytes at <b>dest</b> to equal the IPv6 address <b>src</b>.
+ * <b>src</b> must be an IPv6 address, if it is not, log a warning, and clear
+ * <b>dest</b>. */
+void
+tor_addr_copy_ipv6_bytes(uint8_t *dest, const tor_addr_t *src)
+{
+  tor_assert(dest);
+  tor_assert(src);
+  memset(dest, 0, 16);
+  IF_BUG_ONCE(src->family != AF_INET6)
+    return;
+  memcpy(dest, src->addr.in6_addr.s6_addr, 16);
 }
 
 /** Copy a tor_addr_t from <b>src</b> to <b>dest</b>.
@@ -1166,57 +1189,137 @@ fmt_addr_impl(const tor_addr_t *addr, int decorate)
 const char *
 fmt_addrport(const tor_addr_t *addr, uint16_t port)
 {
-  /* Add space for a colon and up to 5 digits. */
-  static char buf[TOR_ADDR_BUF_LEN + 6];
+  static char buf[TOR_ADDRPORT_BUF_LEN];
   tor_snprintf(buf, sizeof(buf), "%s:%u", fmt_and_decorate_addr(addr), port);
   return buf;
 }
 
 /** Like fmt_addr(), but takes <b>addr</b> as a host-order IPv4
  * addresses. Also not thread-safe, also clobbers its return buffer on
- * repeated calls. */
+ * repeated calls. Clean internal buffer and return empty string on failure. */
 const char *
 fmt_addr32(uint32_t addr)
 {
   static char buf[INET_NTOA_BUF_LEN];
   struct in_addr in;
+  int success;
+
   in.s_addr = htonl(addr);
-  tor_inet_ntoa(&in, buf, sizeof(buf));
+
+  success = tor_inet_ntoa(&in, buf, sizeof(buf));
+  tor_assertf_nonfatal(success >= 0,
+      "Failed to convert IP 0x%08X (HBO) to string", addr);
+
+  IF_BUG_ONCE(success < 0) {
+    memset(buf, 0, INET_NTOA_BUF_LEN);
+  }
+
   return buf;
 }
 
+/** Return a string representing the family of <b>addr</b>.
+ *
+ * This string is a string constant, and must not be freed.
+ * This function is thread-safe.
+ */
+const char *
+fmt_addr_family(const tor_addr_t *addr)
+{
+  static int default_bug_once = 0;
+
+  IF_BUG_ONCE(!addr)
+    return "NULL pointer";
+
+  switch (tor_addr_family(addr)) {
+    case AF_INET6:
+      return "IPv6";
+    case AF_INET:
+      return "IPv4";
+    case AF_UNIX:
+      return "UNIX socket";
+    case AF_UNSPEC:
+      return "unspecified";
+    default:
+      if (!default_bug_once) {
+        log_warn(LD_BUG, "Called with unknown address family %d",
+                 (int)tor_addr_family(addr));
+        default_bug_once = 1;
+      }
+      return "unknown";
+  }
+  //return "(unreachable code)";
+}
+
 /** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
- * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
- * square brackets.
+ * may be an IPv4 address, or an IPv6 address surrounded by square brackets.
  *
- *  Return an address family on success, or -1 if an invalid address string is
- *  provided. */
-int
-tor_addr_parse(tor_addr_t *addr, const char *src)
+ * If <b>allow_ipv6_without_brackets</b> is true, also allow IPv6 addresses
+ * without brackets.
+ *
+ * Always rejects IPv4 addresses with brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+static int
+tor_addr_parse_impl(tor_addr_t *addr, const char *src,
+                    bool allow_ipv6_without_brackets)
 {
   /* Holds substring of IPv6 address after removing square brackets */
   char *tmp = NULL;
-  int result;
+  int result = -1;
   struct in_addr in_tmp;
   struct in6_addr in6_tmp;
+  int brackets_detected = 0;
+
   tor_assert(addr && src);
-  if (src[0] == '[' && src[1])
+
+  size_t len = strlen(src);
+
+  if (len && src[0] == '[' && src[len - 1] == ']') {
+    brackets_detected = 1;
     src = tmp = tor_strndup(src+1, strlen(src)-2);
+  }
 
-  if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
-    result = AF_INET6;
-    tor_addr_from_in6(addr, &in6_tmp);
-  } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
-    result = AF_INET;
-    tor_addr_from_in(addr, &in_tmp);
-  } else {
-    result = -1;
+  /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses
+   * without brackets are allowed */
+  if (brackets_detected || allow_ipv6_without_brackets) {
+    if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
+      result = AF_INET6;
+      tor_addr_from_in6(addr, &in6_tmp);
+    }
+  }
+
+  /* Try to parse an IPv4 address without brackets */
+  if (!brackets_detected) {
+    if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
+      result = AF_INET;
+      tor_addr_from_in(addr, &in_tmp);
+    }
+  }
+
+  /* Clear the address on error, to avoid returning uninitialised or partly
+   * parsed data.
+   */
+  if (result == -1) {
+    memset(addr, 0, sizeof(tor_addr_t));
   }
 
   tor_free(tmp);
   return result;
 }
 
+/** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
+ * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
+ * square brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+int
+tor_addr_parse(tor_addr_t *addr, const char *src)
+{
+  return tor_addr_parse_impl(addr, src, 1);
+}
+
 #ifdef HAVE_IFADDRS_TO_SMARTLIST
 /*
  * Convert a linked list consisting of <b>ifaddrs</b> structures
@@ -1352,7 +1455,7 @@ get_interface_addresses_win32(int severity, sa_family_t family)
 
 /* This is defined on Mac OS X */
 #ifndef _SIZEOF_ADDR_IFREQ
-#define _SIZEOF_ADDR_IFREQ sizeof
+#define _SIZEOF_ADDR_IFREQ(x) sizeof(x)
 #endif
 
 /* Free ifc->ifc_buf safely. */
@@ -1371,10 +1474,10 @@ ifconf_free_ifc_buf(struct ifconf *ifc)
  * into smartlist of <b>tor_addr_t</b> structures.
  */
 STATIC smartlist_t *
-ifreq_to_smartlist(char *buf, size_t buflen)
+ifreq_to_smartlist(const uint8_t *buf, size_t buflen)
 {
   smartlist_t *result = smartlist_new();
-  char *end = buf + buflen;
+  const uint8_t *end = buf + buflen;
 
   /* These acrobatics are due to alignment issues which trigger
    * undefined behaviour traps on OSX. */
@@ -1448,7 +1551,7 @@ get_interface_addresses_ioctl(int severity, sa_family_t family)
     /* Ensure we have least IFREQ_SIZE bytes unused at the end. Otherwise, we
      * don't know if we got everything during ioctl. */
   } while (mult * IFREQ_SIZE - ifc.ifc_len <= IFREQ_SIZE);
-  result = ifreq_to_smartlist(ifc.ifc_buf, ifc.ifc_len);
+  result = ifreq_to_smartlist((const uint8_t *)ifc.ifc_buf, ifc.ifc_len);
 
  done:
   if (fd >= 0)
@@ -1601,11 +1704,15 @@ get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr))
    * Ideally, we want the default route, see #12377 for details */
   SMARTLIST_FOREACH_BEGIN(addrs, tor_addr_t *, a) {
     tor_addr_copy(addr, a);
+    const bool is_internal = tor_addr_is_internal(a, 0);
     rv = 0;
 
+    log_debug(LD_NET, "Found %s interface address '%s'",
+              (is_internal ? "internal" : "external"), fmt_addr(addr));
+
     /* If we found a non-internal address, declare success.  Otherwise,
      * keep looking. */
-    if (!tor_addr_is_internal(a, 0))
+    if (!is_internal)
       break;
   } SMARTLIST_FOREACH_END(a);
 
@@ -1709,6 +1816,11 @@ get_interface_address6_list,(int severity,
  * form "ip" or "ip:0".  Otherwise, accept those forms, and set
  * *<b>port_out</b> to <b>default_port</b>.
  *
+ * This function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
  * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_parse(int severity, const char *addrport,
@@ -1718,6 +1830,7 @@ tor_addr_port_parse(int severity, const char *addrport,
   int retval = -1;
   int r;
   char *addr_tmp = NULL;
+  bool has_port;
 
   tor_assert(addrport);
   tor_assert(address_out);
@@ -1727,28 +1840,47 @@ tor_addr_port_parse(int severity, const char *addrport,
   if (r < 0)
     goto done;
 
-  if (!*port_out) {
+  has_port = !! *port_out;
+  /* If there's no port, use the default port, or fail if there is no default
+   */
+  if (!has_port) {
     if (default_port >= 0)
       *port_out = default_port;
     else
       goto done;
   }
 
-  /* make sure that address_out is an IP address */
-  if (tor_addr_parse(address_out, addr_tmp) < 0)
+  /* Make sure that address_out is an IP address.
+   * If there is no port in addrport, allow IPv6 addresses without brackets. */
+  if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0)
     goto done;
 
   retval = 0;
 
  done:
+  /* Clear the address and port on error, to avoid returning uninitialised or
+   * partly parsed data.
+   */
+  if (retval == -1) {
+    memset(address_out, 0, sizeof(tor_addr_t));
+    *port_out = 0;
+  }
   tor_free(addr_tmp);
   return retval;
 }
 
 /** Given an address of the form "host[:port]", try to divide it into its host
- * and port portions, setting *<b>address_out</b> to a newly allocated string
- * holding the address portion and *<b>port_out</b> to the port (or 0 if no
- * port is given).  Return 0 on success, -1 on failure. */
+ * and port portions.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Sets *<b>address_out</b> to a newly allocated string holding the address
+ * portion, and *<b>port_out</b> to the port (or 0 if no port is given).
+ *
+ * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_split(int severity, const char *addrport,
                     char **address_out, uint16_t *port_out)
@@ -1757,8 +1889,11 @@ tor_addr_port_split(int severity, const char *addrport,
   tor_assert(addrport);
   tor_assert(address_out);
   tor_assert(port_out);
+
   /* We need to check for IPv6 manually because the logic below doesn't
-   * do a good job on IPv6 addresses that lack a port. */
+   * do a good job on IPv6 addresses that lack a port.
+   * If an IPv6 address without square brackets is ambiguous, it gets parsed
+   * here as an address, rather than address:port. */
   if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) {
     *port_out = 0;
     *address_out = tor_strdup(addrport);
@@ -1798,8 +1933,7 @@ tor_addr_port_split(int severity, const char *addrport,
     tor_free(address_);
   }
 
-  if (port_out)
-    *port_out = ok ? ((uint16_t) port_) : 0;
+  *port_out = ok ? ((uint16_t) port_) : 0;
 
   return ok ? 0 : -1;
 }
@@ -1875,17 +2009,24 @@ parse_port_range(const char *port, uint16_t *port_min_out,
 }
 
 /** Given a host-order <b>addr</b>, call tor_inet_ntop() on it
- *  and return a strdup of the resulting address.
+ *  and return a strdup of the resulting address. Return NULL if
+ *  tor_inet_ntop() fails.
  */
 char *
 tor_dup_ip(uint32_t addr)
 {
+  const char *ip_str;
   char buf[TOR_ADDR_BUF_LEN];
   struct in_addr in;
 
   in.s_addr = htonl(addr);
-  tor_inet_ntop(AF_INET, &in, buf, sizeof(buf));
-  return tor_strdup(buf);
+  ip_str = tor_inet_ntop(AF_INET, &in, buf, sizeof(buf));
+
+  tor_assertf_nonfatal(ip_str, "Failed to duplicate IP %08X", addr);
+  if (ip_str)
+    return tor_strdup(buf);
+
+  return NULL;
 }
 
 /**
@@ -1934,7 +2075,7 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port)
   return ap;
 }
 
-/** Return true iff <a>a</b> and <b>b</b> are the same address and port */
+/** Return true iff <b>a</b> and <b>b</b> are the same address and port */
 int
 tor_addr_port_eq(const tor_addr_port_t *a,
                  const tor_addr_port_t *b)
@@ -2018,8 +2159,12 @@ string_is_valid_nonrfc_hostname(const char *string)
 
   smartlist_split_string(components,string,".",0,0);
 
-  if (BUG(smartlist_len(components) == 0))
-    return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks.
+  if (BUG(smartlist_len(components) == 0)) {
+    // LCOV_EXCL_START should be impossible given the earlier checks.
+    smartlist_free(components);
+    return 0;
+    // LCOV_EXCL_STOP
+  }
 
   /* Allow a single terminating '.' used rarely to indicate domains
    * are FQDNs rather than relative. */
diff --git a/src/lib/net/address.h b/src/lib/net/address.h
index 9b826c8359..e5016ee4fe 100644
--- a/src/lib/net/address.h
+++ b/src/lib/net/address.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -62,6 +62,7 @@
 typedef uint8_t maskbits_t;
 
 struct in_addr;
+
 /** Holds an IPv4 or IPv6 address.  (Uses less memory than struct
  * sockaddr_storage.) */
 typedef struct tor_addr_t
@@ -103,6 +104,10 @@ int tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa,
                            uint16_t *port_out);
 void tor_addr_make_unspec(tor_addr_t *a);
 void tor_addr_make_null(tor_addr_t *a, sa_family_t family);
+#define tor_addr_port_make_null(addr, port, family) \
+  (void)(tor_addr_make_null(addr, family), (port) = 0)
+#define tor_addr_port_make_null_ap(ap, family) \
+  tor_addr_port_make_null(&(ap)->addr, (ap)->port, family)
 char *tor_sockaddr_to_str(const struct sockaddr *sa);
 
 /** Return an in6_addr* equivalent to <b>a</b>, or NULL if <b>a</b> is not
@@ -133,6 +138,7 @@ tor_addr_to_in6_assert(const tor_addr_t *a)
  * Requires that <b>x</b> is actually an IPv6 address.
  */
 #define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6_assert(x))
+
 /** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t.
  *
  * Requires that <b>x</b> is actually an IPv6 address.
@@ -146,6 +152,7 @@ tor_addr_to_ipv4n(const tor_addr_t *a)
 {
   return a->family == AF_INET ? a->addr.in_addr.s_addr : 0;
 }
+
 /** Return an IPv4 address in host order for <b>a</b>, or 0 if
  * <b>a</b> is not an IPv4 address. */
 static inline uint32_t
@@ -153,10 +160,11 @@ tor_addr_to_ipv4h(const tor_addr_t *a)
 {
   return ntohl(tor_addr_to_ipv4n(a));
 }
+
 /** Given an IPv6 address, return its mapped IPv4 address in host order, or
  * 0 if <b>a</b> is not an IPv6 address.
  *
- * (Does not check whether the address is really a mapped address */
+ * (Does not check whether the address is really a mapped address.) */
 static inline uint32_t
 tor_addr_to_mapped_ipv4h(const tor_addr_t *a)
 {
@@ -165,21 +173,21 @@ tor_addr_to_mapped_ipv4h(const tor_addr_t *a)
     // Work around an incorrect NULL pointer dereference warning in
     // "clang --analyze" due to limited analysis depth
     addr32 = tor_addr_to_in6_addr32(a);
-    // To improve performance, wrap this assertion in:
-    // #if !defined(__clang_analyzer__) || PARANOIA
     tor_assert(addr32);
     return ntohl(addr32[3]);
   } else {
     return 0;
   }
 }
+
 /** Return the address family of <b>a</b>.  Possible values are:
- * AF_INET6, AF_INET, AF_UNSPEC. */
+ * AF_INET6, AF_INET, AF_UNSPEC, AF_UNIX. */
 static inline sa_family_t
 tor_addr_family(const tor_addr_t *a)
 {
   return a->family;
 }
+
 /** Return an in_addr* equivalent to <b>a</b>, or NULL if <b>a</b> is not
  * an IPv4 address. */
 static inline const struct in_addr *
@@ -187,6 +195,7 @@ tor_addr_to_in(const tor_addr_t *a)
 {
   return a->family == AF_INET ? &a->addr.in_addr : NULL;
 }
+
 /** Return true iff <b>a</b> is an IPv4 address equal to the host-ordered
  * address in <b>u</b>. */
 static inline int
@@ -204,24 +213,39 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u)
  */
 #define TOR_ADDR_BUF_LEN 48
 
+/** Length of a buffer containing an IP address along with a port number and
+ * a seperating colon.
+ *
+ * This allows enough space for
+ *   "[ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]:12345",
+ * plus a terminating NUL.
+ */
+#define TOR_ADDRPORT_BUF_LEN (TOR_ADDR_BUF_LEN + 6)
+
 char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC;
 
 /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6
  *  addresses. */
 #define fmt_addr(a) fmt_addr_impl((a), 0)
+
 /** Wrapper function of fmt_addr_impl(). It decorates IPv6
  *  addresses. */
 #define fmt_and_decorate_addr(a) fmt_addr_impl((a), 1)
+
 const char *fmt_addr_impl(const tor_addr_t *addr, int decorate);
 const char *fmt_addrport(const tor_addr_t *addr, uint16_t port);
-const char * fmt_addr32(uint32_t addr);
+#define fmt_addrport_ap(ap) fmt_addrport(&(ap)->addr, (ap)->port)
+const char *fmt_addr32(uint32_t addr);
+const char *fmt_addr_family(const tor_addr_t *addr);
 
 MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family,
 tor_addr_t *addr));
+
 struct smartlist_t;
-void interface_address6_list_free_(struct smartlist_t * addrs);// XXXX
+void interface_address6_list_free_(struct smartlist_t * addrs);
 #define interface_address6_list_free(addrs) \
   FREE_AND_NULL(struct smartlist_t, interface_address6_list_free_, (addrs))
+
 MOCK_DECL(struct smartlist_t *,get_interface_address6_list,(int severity,
                                                      sa_family_t family,
                                                      int include_internal));
@@ -246,6 +270,7 @@ int tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
 uint64_t tor_addr_hash(const tor_addr_t *addr);
 struct sipkey;
 uint64_t tor_addr_keyed_hash(const struct sipkey *key, const tor_addr_t *addr);
+
 int tor_addr_is_v4(const tor_addr_t *addr);
 int tor_addr_is_internal_(const tor_addr_t *ip, int for_listening,
                           const char *filename, int lineno);
@@ -276,21 +301,25 @@ int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
 int tor_addr_parse_mask_ports(const char *s, unsigned flags,
                               tor_addr_t *addr_out, maskbits_t *mask_out,
                               uint16_t *port_min_out, uint16_t *port_max_out);
+
 const char * tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len,
                              int decorate);
 int tor_addr_parse(tor_addr_t *addr, const char *src);
 void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src);
 void tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src);
+
 void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr);
 /** Set <b>dest</b> to the IPv4 address encoded in <b>v4addr</b> in host
  * order. */
 #define tor_addr_from_ipv4h(dest, v4addr)       \
   tor_addr_from_ipv4n((dest), htonl(v4addr))
-void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *bytes);
+void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const uint8_t *bytes);
 /** Set <b>dest</b> to the IPv4 address incoded in <b>in</b>. */
 #define tor_addr_from_in(dest, in) \
   tor_addr_from_ipv4n((dest), (in)->s_addr);
 void tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6);
+void tor_addr_copy_ipv6_bytes(uint8_t *dest, const tor_addr_t *src);
+
 int tor_addr_is_null(const tor_addr_t *addr);
 int tor_addr_is_loopback(const tor_addr_t *addr);
 
@@ -299,6 +328,7 @@ int tor_addr_is_valid_ipv4n(uint32_t v4n_addr, int for_listening);
 #define tor_addr_is_valid_ipv4h(v4h_addr, for_listening) \
         tor_addr_is_valid_ipv4n(htonl(v4h_addr), (for_listening))
 int tor_port_is_valid(uint16_t port, int for_listening);
+
 /* Are addr and port both valid? */
 #define tor_addr_port_is_valid(addr, port, for_listening) \
         (tor_addr_is_valid((addr), (for_listening)) &&    \
@@ -329,9 +359,11 @@ int parse_port_range(const char *port, uint16_t *port_min_out,
                      uint16_t *port_max_out);
 int addr_mask_get_bits(uint32_t mask);
 char *tor_dup_ip(uint32_t addr) ATTR_MALLOC;
+
 MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr));
 #define interface_address_list_free(lst)\
   interface_address6_list_free(lst)
+
 /** Return a smartlist of the IPv4 addresses of all interfaces on the server.
  * Excludes loopback and multicast addresses. Only includes internal addresses
  * if include_internal is true. (Note that a relay behind NAT may use an
@@ -377,8 +409,8 @@ STATIC struct smartlist_t *get_interface_addresses_win32(int severity,
 #endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
-STATIC struct smartlist_t *ifreq_to_smartlist(char *ifr,
-                                       size_t buflen);
+STATIC struct smartlist_t *ifreq_to_smartlist(const uint8_t *ifr,
+                                              size_t buflen);
 STATIC struct smartlist_t *get_interface_addresses_ioctl(int severity,
                                                   sa_family_t family);
 #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
diff --git a/src/lib/net/alertsock.c b/src/lib/net/alertsock.c
index cc59d7d893..537fdcaee4 100644
--- a/src/lib/net/alertsock.c
+++ b/src/lib/net/alertsock.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h
index c45f42be81..dab4273cf1 100644
--- a/src/lib/net/alertsock.h
+++ b/src/lib/net/alertsock.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,4 +42,4 @@ typedef struct alert_sockets_t {
 int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags);
 void alert_sockets_close(alert_sockets_t *socks);
 
-#endif
+#endif /* !defined(TOR_ALERTSOCK_H) */
diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c
index 3eb0a033d5..4dbf491e1a 100644
--- a/src/lib/net/buffers_net.c
+++ b/src/lib/net/buffers_net.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/net/buffers_net.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/net/nettypes.h"
@@ -22,6 +22,10 @@
 
 #include <stdlib.h>
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 #ifdef PARANOIA
 /** Helper: If PARANOIA is defined, assert that the buffer in local variable
  * <b>buf</b> is well-formed. */
@@ -30,27 +34,36 @@
 #define check() STMT_NIL
 #endif /* defined(PARANOIA) */
 
-/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into
  * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
- * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
- * and the number of bytes read otherwise. */
+ * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b>
+ * is true, otherwise it uses <b>read()</b>.  Return -1 on error (and sets
+ * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read
+ * otherwise. */
 static inline int
 read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
-              int *reached_eof, int *socket_error)
+              int *reached_eof, int *error, bool is_socket)
 {
   ssize_t read_result;
   if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
     at_most = CHUNK_REMAINING_CAPACITY(chunk);
-  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (is_socket)
+    read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+  else
+    read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most);
 
   if (read_result < 0) {
-    int e = tor_socket_errno(fd);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
-        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+        log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?",
+                 is_socket ? "recv" : "read");
 #endif
-      *socket_error = e;
+      if (error)
+        *error = e;
       return -1;
     }
     return 0; /* would block. */
@@ -63,21 +76,22 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
     chunk->datalen += read_result;
     log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
               (int)buf->datalen);
-    tor_assert(read_result < INT_MAX);
+    tor_assert(read_result <= BUF_MAX_LEN);
     return (int)read_result;
   }
 }
 
-/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
- * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
- * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
- * error; else return the number of bytes read.
+/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>.  Read
+ * at most <b>at_most</b> bytes, growing the buffer as necessary.  If recv()
+ * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0.
+ * Return -1 on error; else return the number of bytes read.
  */
 /* XXXX indicate "read blocked" somehow? */
-int
-buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
-                     int *reached_eof,
-                     int *socket_error)
+static int
+buf_read_from_fd(buf_t *buf, int fd, size_t at_most,
+                 int *reached_eof,
+                 int *socket_error,
+                 bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes read" are not mutually exclusive.
@@ -87,11 +101,11 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 
   check();
   tor_assert(reached_eof);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
 
-  if (BUG(buf->datalen >= INT_MAX))
+  if (BUG(buf->datalen > BUF_MAX_LEN))
     return -1;
-  if (BUG(buf->datalen >= INT_MAX - at_most))
+  if (BUG(buf->datalen > BUF_MAX_LEN - at_most))
     return -1;
 
   while (at_most > total_read) {
@@ -108,11 +122,12 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
         readlen = cap;
     }
 
-    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    r = read_to_chunk(buf, chunk, fd, readlen,
+                      reached_eof, socket_error, is_socket);
     check();
     if (r < 0)
       return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
+    tor_assert(total_read+r <= BUF_MAX_LEN);
     total_read += r;
     if ((size_t)r < readlen) { /* eof, block, or no more to read. */
       break;
@@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 }
 
 /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
- * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, 0 on blocking, -1 on failure.
+ * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>.  On
+ * success, deduct the bytes written from *<b>buf_flushlen</b>.  Return the
+ * number of bytes written on success, 0 on blocking, -1 on failure.
  */
 static inline int
-flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
-            size_t *buf_flushlen)
+flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen, bool is_socket)
 {
   ssize_t write_result;
 
   if (sz > chunk->datalen)
     sz = chunk->datalen;
-  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (is_socket)
+    write_result = tor_socket_send(fd, chunk->data, sz, 0);
+  else
+    write_result = write(fd, chunk->data, sz);
 
   if (write_result < 0) {
-    int e = tor_socket_errno(s);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
@@ -150,20 +170,20 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
   } else {
     *buf_flushlen -= write_result;
     buf_drain(buf, write_result);
-    tor_assert(write_result < INT_MAX);
+    tor_assert(write_result <= BUF_MAX_LEN);
     return (int)write_result;
   }
 }
 
-/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+/** Write data from <b>buf</b> to the file descriptor <b>fd</b>.  Write at most
  * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
  * the number of bytes actually written, and remove the written bytes
  * from the buffer.  Return the number of bytes written on success,
  * -1 on failure.  Return 0 if write() would block.
  */
-int
-buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
-                    size_t *buf_flushlen)
+static int
+buf_flush_to_fd(buf_t *buf, int fd, size_t sz,
+                size_t *buf_flushlen, bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes flushed" are not mutually exclusive.
@@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
   int r;
   size_t flushed = 0;
   tor_assert(buf_flushlen);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
   if (BUG(*buf_flushlen > buf->datalen)) {
     *buf_flushlen = buf->datalen;
   }
@@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
     else
       flushlen0 = buf->head->datalen;
 
-    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket);
     check();
     if (r < 0)
       return r;
@@ -197,6 +217,58 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
     if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
       break;
   }
-  tor_assert(flushed < INT_MAX);
+  tor_assert(flushed <= BUF_MAX_LEN);
   return (int)flushed;
 }
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, s, sz, buf_flushlen, true);
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true);
+}
+
+/** Write data from <b>buf</b> to the pipe <b>fd</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_pipe(buf_t *buf, int fd, size_t sz,
+                  size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false);
+}
+
+/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If read() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_pipe(buf_t *buf, int fd, size_t at_most,
+                   int *reached_eof,
+                   int *socket_error)
+{
+  return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false);
+}
diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h
index 5f69bebedf..a45c23a273 100644
--- a/src/lib/net/buffers_net.h
+++ b/src/lib/net/buffers_net.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most,
 int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz,
                         size_t *buf_flushlen);
 
-#endif /* !defined(TOR_BUFFERS_H) */
+int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most,
+                       int *reached_eof,
+                       int *socket_error);
+
+int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz,
+                      size_t *buf_flushlen);
+
+#endif /* !defined(TOR_BUFFERS_NET_H) */
diff --git a/src/lib/net/gethostname.c b/src/lib/net/gethostname.c
index e54a1ea16e..001d95391d 100644
--- a/src/lib/net/gethostname.c
+++ b/src/lib/net/gethostname.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h
index 69b0528bc0..90f8056779 100644
--- a/src/lib/net/gethostname.h
+++ b/src/lib/net/gethostname.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,4 +16,4 @@
 
 MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
 
-#endif
+#endif /* !defined(TOR_GETHOSTNAME_H) */
diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c
index 1a2406ce5f..d50ac2440c 100644
--- a/src/lib/net/inaddr.c
+++ b/src/lib/net/inaddr.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,9 @@
 #include "lib/net/inaddr.h"
 
 #include "lib/cc/torint.h"
+#include "lib/container/smartlist.h"
 #include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
 #include "lib/net/inaddr_st.h"
 #include "lib/string/compat_ctype.h"
 #include "lib/string/compat_string.h"
@@ -35,12 +37,31 @@
  * (Like inet_aton(str,addr), but works on Windows and Solaris.)
  */
 int
-tor_inet_aton(const char *str, struct in_addr* addr)
+tor_inet_aton(const char *str, struct in_addr *addr)
 {
-  unsigned a,b,c,d;
+  unsigned a, b, c, d;
   char more;
-  if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a,&b,&c,&d,&more) != 4)
+  bool is_octal = false;
+  smartlist_t *sl = NULL;
+
+  if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a, &b, &c, &d, &more) != 4)
+    return 0;
+
+  /* Parse the octets and check them for leading zeros. */
+  sl = smartlist_new();
+  smartlist_split_string(sl, str, ".", 0, 0);
+  SMARTLIST_FOREACH(sl, const char *, octet, {
+    is_octal = (strlen(octet) > 1 && octet[0] == '0');
+    if (is_octal) {
+        break;
+    }
+  });
+  SMARTLIST_FOREACH(sl, char *, octet, tor_free(octet));
+  smartlist_free(sl);
+
+  if (is_octal)
     return 0;
+
   if (a > 255) return 0;
   if (b > 255) return 0;
   if (c > 255) return 0;
@@ -168,6 +189,13 @@ tor_inet_pton(int af, const char *src, void *dst)
   if (af == AF_INET) {
     return tor_inet_aton(src, dst);
   } else if (af == AF_INET6) {
+    ssize_t len = strlen(src);
+
+    /* Reject if src has needless trailing ':'. */
+    if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') {
+      return 0;
+    }
+
     struct in6_addr *out = dst;
     uint16_t words[8];
     int gapPos = -1, i, setWords=0;
@@ -207,7 +235,6 @@ tor_inet_pton(int af, const char *src, void *dst)
         return 0;
       if (TOR_ISXDIGIT(*src)) {
         char *next;
-        ssize_t len;
         long r = strtol(src, &next, 16);
         if (next == NULL || next == src) {
           /* The 'next == src' error case can happen on versions of openbsd
diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h
index 36352b65ea..8d6766eb5d 100644
--- a/src/lib/net/inaddr.h
+++ b/src/lib/net/inaddr.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,4 +24,4 @@ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
 const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len);
 int tor_inet_pton(int af, const char *src, void *dst);
 
-#endif
+#endif /* !defined(TOR_INADDR_H) */
diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h
index 806f2c096a..b9ee2b86cf 100644
--- a/src/lib/net/inaddr_st.h
+++ b/src/lib/net/inaddr_st.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -104,4 +104,4 @@ struct sockaddr_in6 {
 };
 #endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */
 
-#endif /* TOR_INADDR_ST_H */
+#endif /* !defined(TOR_INADDR_ST_H) */
diff --git a/src/lib/net/include.am b/src/lib/net/include.am
index ff0967e786..485019f4b7 100644
--- a/src/lib/net/include.am
+++ b/src/lib/net/include.am
@@ -5,12 +5,14 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-net-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_net_a_SOURCES =			\
 	src/lib/net/address.c			\
 	src/lib/net/alertsock.c                 \
 	src/lib/net/buffers_net.c		\
 	src/lib/net/gethostname.c		\
 	src/lib/net/inaddr.c			\
+	src/lib/net/network_sys.c		\
 	src/lib/net/resolve.c			\
 	src/lib/net/socket.c			\
 	src/lib/net/socketpair.c
@@ -20,6 +22,7 @@ src_lib_libtor_net_testing_a_SOURCES = \
 src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/net/address.h			\
 	src/lib/net/alertsock.h                 \
@@ -28,6 +31,7 @@ noinst_HEADERS +=				\
 	src/lib/net/inaddr.h			\
 	src/lib/net/inaddr_st.h			\
 	src/lib/net/nettypes.h			\
+	src/lib/net/network_sys.h		\
 	src/lib/net/resolve.h			\
 	src/lib/net/socket.h			\
 	src/lib/net/socketpair.h		\
diff --git a/src/lib/net/lib_net.md b/src/lib/net/lib_net.md
new file mode 100644
index 0000000000..b61878d827
--- /dev/null
+++ b/src/lib/net/lib_net.md
@@ -0,0 +1,6 @@
+@dir /lib/net
+@brief lib/net: Low-level network-related code.
+
+This module includes address manipulation, compatibility wrappers,
+convenience functions, and so on.
+
diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h
index 6209bbe18a..953673d4c3 100644
--- a/src/lib/net/nettypes.h
+++ b/src/lib/net/nettypes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,7 +31,7 @@ typedef int socklen_t;
 #define TOR_SOCKET_T_FORMAT "%"PRIuPTR
 #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
 #define TOR_INVALID_SOCKET INVALID_SOCKET
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 /** Type used for a network socket. */
 #define tor_socket_t int
 #define TOR_SOCKET_T_FORMAT "%d"
@@ -41,4 +41,4 @@ typedef int socklen_t;
 #define TOR_INVALID_SOCKET (-1)
 #endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_NET_TYPES_H) */
diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c
new file mode 100644
index 0000000000..e95c3ba819
--- /dev/null
+++ b/src/lib/net/network_sys.c
@@ -0,0 +1,47 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file network_sys.c
+ * \brief Subsystem object for networking setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/net/network_sys.h"
+#include "lib/net/resolve.h"
+#include "lib/net/socket.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#endif
+
+static int
+subsys_network_initialize(void)
+{
+  if (network_init() < 0)
+    return -1;
+
+  return 0;
+}
+
+static void
+subsys_network_shutdown(void)
+{
+#ifdef _WIN32
+  WSACleanup();
+#endif
+  tor_free_getaddrinfo_cache();
+}
+
+const subsys_fns_t sys_network = {
+  .name = "network",
+  SUBSYS_DECLARE_LOCATION(),
+  /* Network depends on logging, and a lot of other modules depend on network.
+   */
+  .level = -55,
+  .supported = true,
+  .initialize = subsys_network_initialize,
+  .shutdown = subsys_network_shutdown,
+};
diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h
new file mode 100644
index 0000000000..734533c7e8
--- /dev/null
+++ b/src/lib/net/network_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file network_sys.h
+ * \brief Declare subsystem object for the network module.
+ **/
+
+#ifndef TOR_NETWORK_SYS_H
+#define TOR_NETWORK_SYS_H
+
+extern const struct subsys_fns_t sys_network;
+
+#endif /* !defined(TOR_NETWORK_SYS_H) */
diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c
index 8cee29df37..68a8c01ef4 100644
--- a/src/lib/net/resolve.c
+++ b/src/lib/net/resolve.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,6 +8,7 @@
  * \brief Use the libc DNS resolver to convert hostnames into addresses.
  **/
 
+#define RESOLVE_PRIVATE
 #include "lib/net/resolve.h"
 
 #include "lib/net/address.h"
@@ -16,8 +17,8 @@
 #include "lib/string/parse_int.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
-#include "ht.h"
+#include "ext/siphash.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
@@ -35,6 +36,8 @@
  * *<b>addr</b> to the proper IP address, in host byte order.  Returns 0
  * on success, -1 on failure; 1 on transient failure.
  *
+ * This function only accepts IPv4 addresses.
+ *
  * (This function exists because standard windows gethostbyname
  * doesn't treat raw IP addresses properly.)
  */
@@ -45,6 +48,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   tor_addr_t myaddr;
   int ret;
 
+  if (BUG(!addr))
+    return -1;
+
+  *addr = 0;
+
   if ((ret = tor_addr_lookup(name, AF_INET, &myaddr)))
     return ret;
 
@@ -56,183 +64,257 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   return -1;
 }
 
-/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
- * *<b>addr</b> to the proper IP address and family. The <b>family</b>
- * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
- * <i>preferred</i> family, though another one may be returned if only one
- * family is implemented for this address.
+#ifdef HAVE_GETADDRINFO
+
+/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is
+ * available on this system.
  *
- * Return 0 on success, -1 on failure; 1 on transient failure.
+ * See tor_addr_lookup() for details.
  */
-MOCK_IMPL(int,
-tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                          tor_addr_t *addr))
 {
-  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
-   * something.
-   */
-  struct in_addr iaddr;
-  struct in6_addr iaddr6;
-  tor_assert(name);
-  tor_assert(addr);
-  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
-  if (!*name) {
-    /* Empty address is an error. */
-    return -1;
-  } else if (tor_inet_pton(AF_INET, name, &iaddr)) {
-    /* It's an IPv4 IP. */
-    if (family == AF_INET6)
-      return -1;
-    tor_addr_from_in(addr, &iaddr);
-    return 0;
-  } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) {
-    if (family == AF_INET)
-      return -1;
-    tor_addr_from_in6(addr, &iaddr6);
-    return 0;
-  } else {
-#ifdef HAVE_GETADDRINFO
-    int err;
-    struct addrinfo *res=NULL, *res_p;
-    struct addrinfo *best=NULL;
-    struct addrinfo hints;
-    int result = -1;
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = family;
-    hints.ai_socktype = SOCK_STREAM;
-    err = tor_getaddrinfo(name, NULL, &hints, &res);
-    /* The check for 'res' here shouldn't be necessary, but it makes static
-     * analysis tools happy. */
-    if (!err && res) {
-      best = NULL;
-      for (res_p = res; res_p; res_p = res_p->ai_next) {
-        if (family == AF_UNSPEC) {
-          if (res_p->ai_family == AF_INET) {
-            best = res_p;
-            break;
-          } else if (res_p->ai_family == AF_INET6 && !best) {
-            best = res_p;
-          }
-        } else if (family == res_p->ai_family) {
+  int err;
+  struct addrinfo *res=NULL, *res_p;
+  struct addrinfo *best=NULL;
+  struct addrinfo hints;
+  int result = -1;
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = family;
+  hints.ai_socktype = SOCK_STREAM;
+  err = tor_getaddrinfo(name, NULL, &hints, &res);
+  /* The check for 'res' here shouldn't be necessary, but it makes static
+   * analysis tools happy. */
+  if (!err && res) {
+    best = NULL;
+    for (res_p = res; res_p; res_p = res_p->ai_next) {
+      if (family == AF_UNSPEC) {
+        if (res_p->ai_family == AF_INET) {
           best = res_p;
           break;
+        } else if (res_p->ai_family == AF_INET6 && !best) {
+          best = res_p;
         }
+      } else if (family == res_p->ai_family) {
+        best = res_p;
+        break;
       }
-      if (!best)
-        best = res;
-      if (best->ai_family == AF_INET) {
-        tor_addr_from_in(addr,
-                         &((struct sockaddr_in*)best->ai_addr)->sin_addr);
-        result = 0;
-      } else if (best->ai_family == AF_INET6) {
-        tor_addr_from_in6(addr,
-                          &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
-        result = 0;
-      }
-      tor_freeaddrinfo(res);
-      return result;
     }
-    return (err == EAI_AGAIN) ? 1 : -1;
-#else /* !(defined(HAVE_GETADDRINFO)) */
-    struct hostent *ent;
-    int err;
+    if (!best)
+      best = res;
+    if (best->ai_family == AF_INET) {
+      tor_addr_from_in(addr,
+                       &((struct sockaddr_in*)best->ai_addr)->sin_addr);
+      result = 0;
+    } else if (best->ai_family == AF_INET6) {
+      tor_addr_from_in6(addr,
+                        &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
+      result = 0;
+    }
+    tor_freeaddrinfo(res);
+    return result;
+  }
+  return (err == EAI_AGAIN) ? 1 : -1;
+}
+
+#else /* !defined(HAVE_GETADDRINFO) */
+
+/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname().
+ * Used when getaddrinfo() is not available on this system.
+ *
+ * See tor_addr_lookup() for details.
+ */
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                           tor_addr_t *addr))
+{
+  (void) family;
+  struct hostent *ent;
+  int err;
 #ifdef HAVE_GETHOSTBYNAME_R_6_ARG
-    char buf[2048];
-    struct hostent hostent;
-    int r;
-    r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
+  char buf[2048];
+  struct hostent hostent;
+  int r;
+  r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_5_ARG)
-    char buf[2048];
-    struct hostent hostent;
-    ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
+  char buf[2048];
+  struct hostent hostent;
+  ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_3_ARG)
-    struct hostent_data data;
-    struct hostent hent;
-    memset(&data, 0, sizeof(data));
-    err = gethostbyname_r(name, &hent, &data);
-    ent = err ? NULL : &hent;
+  struct hostent_data data;
+  struct hostent hent;
+  memset(&data, 0, sizeof(data));
+  err = gethostbyname_r(name, &hent, &data);
+  ent = err ? NULL : &hent;
 #else
-    ent = gethostbyname(name);
+  ent = gethostbyname(name);
 #ifdef _WIN32
-    err = WSAGetLastError();
+  err = WSAGetLastError();
 #else
-    err = h_errno;
-#endif
+  err = h_errno;
+#endif /* defined(_WIN32) */
 #endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */
-    if (ent) {
-      if (ent->h_addrtype == AF_INET) {
-        tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
-      } else if (ent->h_addrtype == AF_INET6) {
-        tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
-      } else {
-        tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
-      }
-      return 0;
+  if (ent) {
+    if (ent->h_addrtype == AF_INET) {
+      tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
+    } else if (ent->h_addrtype == AF_INET6) {
+      tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
+    } else {
+      tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
     }
+    return 0;
+  }
 #ifdef _WIN32
-    return (err == WSATRY_AGAIN) ? 1 : -1;
+  return (err == WSATRY_AGAIN) ? 1 : -1;
 #else
-    return (err == TRY_AGAIN) ? 1 : -1;
+  return (err == TRY_AGAIN) ? 1 : -1;
 #endif
+}
 #endif /* defined(HAVE_GETADDRINFO) */
+
+/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
+ * *<b>addr</b> to the proper IP address and family. The <b>family</b>
+ * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
+ * <i>preferred</i> family, though another one may be returned if only one
+ * family is implemented for this address.
+ *
+ * Like tor_addr_parse(), this function accepts IPv6 addresses with or without
+ * square brackets.
+ *
+ * Return 0 on success, -1 on failure; 1 on transient failure.
+ */
+MOCK_IMPL(int,
+tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+{
+  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
+   * something.
+   */
+  int parsed_family = 0;
+  int result = -1;
+
+  tor_assert(name);
+  tor_assert(addr);
+  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
+
+  if (!*name) {
+    /* Empty address is an error. */
+    goto permfail;
+  }
+
+  /* Is it an IP address? */
+  parsed_family = tor_addr_parse(addr, name);
+
+  if (parsed_family >= 0) {
+    /* If the IP address family matches, or was unspecified */
+    if (parsed_family == family || family == AF_UNSPEC) {
+      goto success;
+    } else {
+      goto permfail;
+    }
+  } else {
+    /* Clear the address after a failed tor_addr_parse(). */
+    memset(addr, 0, sizeof(tor_addr_t));
+    result = tor_addr_lookup_host_impl(name, family, addr);
+    goto done;
   }
+
+ /* If we weren't successful, and haven't already set the result,
+  * assume it's a permanent failure */
+ permfail:
+  result = -1;
+  goto done;
+ success:
+  result = 0;
+
+ /* We have set the result, now it's time to clean up */
+ done:
+  if (result) {
+    /* Clear the address on error */
+    memset(addr, 0, sizeof(tor_addr_t));
+  }
+  return result;
 }
 
 /** Parse an address or address-port combination from <b>s</b>, resolve the
  * address as needed, and put the result in <b>addr_out</b> and (optionally)
- * <b>port_out</b>.  Return 0 on success, negative on failure. */
+ * <b>port_out</b>.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Return 0 on success, negative on failure. */
 int
 tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out)
 {
-  const char *port;
   tor_addr_t addr;
-  uint16_t portval;
+  uint16_t portval = 0;
   char *tmp = NULL;
+  int rv = 0;
+  int result;
 
   tor_assert(s);
   tor_assert(addr_out);
 
   s = eat_whitespace(s);
 
-  if (*s == '[') {
-    port = strstr(s, "]");
-    if (!port)
-      goto err;
-    tmp = tor_strndup(s+1, port-(s+1));
-    port = port+1;
-    if (*port == ':')
-      port++;
-    else
-      port = NULL;
-  } else {
-    port = strchr(s, ':');
-    if (port)
-      tmp = tor_strndup(s, port-s);
-    else
-      tmp = tor_strdup(s);
-    if (port)
-      ++port;
+  /* Try parsing s as an address:port first, so we don't have to duplicate
+   * the logic that rejects IPv6:Port with no square brackets. */
+  rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0);
+  /* That was easy, no DNS required. */
+  if (rv == 0)
+    goto success;
+
+  /* Now let's check for malformed IPv6 addresses and ports:
+   * tor_addr_port_parse() requires squared brackes if there is a port,
+   * and we want tor_addr_port_lookup() to have the same requirement.
+   * But we strip the port using tor_addr_port_split(), so tor_addr_lookup()
+   * only sees the address, and will accept it without square brackets. */
+  int family = tor_addr_parse(&addr, s);
+  /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need
+   * to reject this address as malformed. */
+  if (family >= 0) {
+    /* Double-check it's an IPv6 address. If not, we have a parsing bug.
+     */
+    tor_assertf_nonfatal(family == AF_INET6,
+                         "Wrong family: %d (should be IPv6: %d) which "
+                         "failed IP:port parsing, but passed IP parsing. "
+                         "input string: '%s'; parsed address: '%s'.",
+                         family, AF_INET6, s, fmt_addr(&addr));
+    goto err;
   }
 
-  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+  /* Now we have a hostname. Let's split off the port, if any. */
+  rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval);
+  if (rv < 0)
     goto err;
-  tor_free(tmp);
 
-  if (port) {
-    portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL);
-    if (!portval)
-      goto err;
-  } else {
-    portval = 0;
-  }
+  /* And feed the hostname to the lookup function. */
+  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+    goto err;
 
+ success:
   if (port_out)
     *port_out = portval;
   tor_addr_copy(addr_out, &addr);
+  result = 0;
+  goto done;
 
-  return 0;
  err:
+  /* Clear the address and port on error */
+  memset(addr_out, 0, sizeof(tor_addr_t));
+  if (port_out)
+    *port_out = 0;
+  result = -1;
+
+ /* We have set the result, now it's time to clean up */
+ done:
   tor_free(tmp);
-  return -1;
+  return result;
 }
 
 #ifdef USE_SANDBOX_GETADDRINFO
@@ -290,11 +372,11 @@ static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t)
 
 HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
              cached_getaddrinfo_item_hash,
-             cached_getaddrinfo_items_eq)
+             cached_getaddrinfo_items_eq);
 HT_GENERATE2(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
              cached_getaddrinfo_item_hash,
              cached_getaddrinfo_items_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 /** If true, don't try to cache getaddrinfo results. */
 static int sandbox_getaddrinfo_cache_disabled = 0;
@@ -421,4 +503,13 @@ tor_make_getaddrinfo_cache_active(void)
 {
   sandbox_getaddrinfo_is_active = 1;
 }
-#endif
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
+void
+sandbox_disable_getaddrinfo_cache(void)
+{
+}
+void
+tor_make_getaddrinfo_cache_active(void)
+{
+}
+#endif /* defined(USE_SANDBOX_GETADDRINFO) */
diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h
index 47a283c81c..ef3d9fa176 100644
--- a/src/lib/net/resolve.h
+++ b/src/lib/net/resolve.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,12 +24,18 @@
 
 struct tor_addr_t;
 
+/*
+ * Primary lookup functions.
+ */
 MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr));
 MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
                                 struct tor_addr_t *addr_out));
 int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out,
                          uint16_t *port_out);
 
+/*
+ * Sandbox helpers
+ */
 struct addrinfo;
 #ifdef USE_SANDBOX_GETADDRINFO
 /** Pre-calls getaddrinfo in order to pre-record result. */
@@ -42,8 +48,7 @@ int tor_getaddrinfo(const char *name, const char *servname,
                         struct addrinfo **res);
 void tor_freeaddrinfo(struct addrinfo *addrinfo);
 void tor_free_getaddrinfo_cache(void);
-void tor_make_getaddrinfo_cache_active(void);
-#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
 #define tor_getaddrinfo(name, servname, hints, res)  \
   getaddrinfo((name),(servname), (hints),(res))
 #define tor_add_addrinfo(name) \
@@ -54,5 +59,15 @@ void tor_make_getaddrinfo_cache_active(void);
 #endif /* defined(USE_SANDBOX_GETADDRINFO) */
 
 void sandbox_disable_getaddrinfo_cache(void);
+void tor_make_getaddrinfo_cache_active(void);
 
+/*
+ * Internal resolver wrapper; exposed for mocking.
+ */
+#ifdef RESOLVE_PRIVATE
+MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name,
+                                                  uint16_t family,
+                                                  struct tor_addr_t *addr));
 #endif
+
+#endif /* !defined(TOR_RESOLVE_H) */
diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c
index fba90b7506..adc060a735 100644
--- a/src/lib/net/socket.c
+++ b/src/lib/net/socket.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,7 +9,6 @@
  *    sockets.
  **/
 
-#define SOCKET_PRIVATE
 #include "lib/net/socket.h"
 #include "lib/net/socketpair.h"
 #include "lib/net/address.h"
@@ -31,6 +30,9 @@
 #endif
 #include <stddef.h>
 #include <string.h>
+#ifdef __FreeBSD__
+#include <sys/sysctl.h>
+#endif
 
 /** Called before we make any calls to network-related functions.
  * (Some operating systems require their network libraries to be
@@ -60,6 +62,32 @@ network_init(void)
   return 0;
 }
 
+/**
+ * Warn the user if any system network parameters should be changed.
+ */
+void
+check_network_configuration(bool server_mode)
+{
+#ifdef __FreeBSD__
+  if (server_mode) {
+    int random_id_state;
+    size_t state_size = sizeof(random_id_state);
+
+    if (sysctlbyname("net.inet.ip.random_id", &random_id_state,
+                     &state_size, NULL, 0)) {
+      log_warn(LD_CONFIG,
+               "Failed to figure out if IP ids are randomized.");
+    } else if (random_id_state == 0) {
+      log_warn(LD_CONFIG, "Looks like IP ids are not randomized. "
+               "Please consider setting the net.inet.ip.random_id sysctl, "
+               "so your relay makes it harder to figure out how busy it is.");
+    }
+  }
+#else /* !defined(__FreeBSD__) */
+  (void) server_mode;
+#endif /* defined(__FreeBSD__) */
+}
+
 /* When set_max_file_sockets() is called, update this with the max file
  * descriptor value so we can use it to check the limit when opening a new
  * socket. Default value is what Debian sets as the default hard limit. */
@@ -177,7 +205,7 @@ mark_socket_closed(tor_socket_t s)
     bitarray_clear(open_sockets, s);
   }
 }
-#else /* !(defined(DEBUG_SOCKET_COUNTING)) */
+#else /* !defined(DEBUG_SOCKET_COUNTING) */
 #define mark_socket_open(s) ((void) (s))
 #define mark_socket_closed(s) ((void) (s))
 #endif /* defined(DEBUG_SOCKET_COUNTING) */
@@ -279,7 +307,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -389,7 +417,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -429,7 +457,9 @@ get_n_open_sockets(void)
  * localhost is inaccessible (for example, if the networking
  * stack is down). And even if it succeeds, the socket pair will not
  * be able to read while localhost is down later (the socket pair may
- * even close, depending on OS-specific timeouts).
+ * even close, depending on OS-specific timeouts). The socket pair
+ * should work on IPv4-only, IPv6-only, and dual-stack systems, as long
+ * as they have the standard localhost addresses.
  *
  * Returns 0 on success and -errno on failure; do not rely on the value
  * of errno or WSAGetLastError().
@@ -456,11 +486,11 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
   r = socketpair(family, type, protocol, fd);
   if (r < 0)
     return -errno;
-#else
+#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */
   r = tor_ersatz_socketpair(family, type, protocol, fd);
   if (r < 0)
     return -r;
-#endif
+#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */
 
 #if defined(FD_CLOEXEC)
   if (SOCKET_OK(fd[0])) {
diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h
index 0909619510..46735fdef0 100644
--- a/src/lib/net/socket.h
+++ b/src/lib/net/socket.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
 int set_socket_nonblocking(tor_socket_t socket);
 int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]);
 int network_init(void);
+void check_network_configuration(bool server_mode);
 
 int get_max_sockets(void);
 void set_max_sockets(int);
@@ -91,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count);
 #define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
 int tor_socket_errno(tor_socket_t sock);
 const char *tor_socket_strerror(int e);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 #define SOCK_ERRNO(e) e
 #if EAGAIN == EWOULDBLOCK
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
@@ -115,4 +116,4 @@ const char *tor_socket_strerror(int e);
 #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKET_H) */
diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c
index 10eb749735..d4310020cb 100644
--- a/src/lib/net/socketpair.c
+++ b/src/lib/net/socketpair.c
@@ -1,6 +1,11 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+
+/**
+ * @file socketpair.c
+ * @brief Replacement socketpair() for systems that lack it
+ **/
 
 #include "lib/cc/torint.h"
 #include "lib/net/socketpair.h"
@@ -22,11 +27,11 @@
 #include <windows.h>
 #define socket_errno() (WSAGetLastError())
 #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT
-#else
+#else /* !defined(_WIN32) */
 #define closesocket(x) close(x)
 #define socket_errno() (errno)
 #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef NEED_ERSATZ_SOCKETPAIR
 
@@ -105,7 +110,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2)
 /**
  * Helper used to implement socketpair on systems that lack it, by
  * making a direct connection to localhost.
- */
+ *
+ * See tor_socketpair() for details.
+ *
+ * The direct connection defaults to IPv4, but falls back to IPv6 if
+ * IPv4 is not supported.
+ **/
 int
 tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
 {
diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h
index 6be0803881..b07016ab94 100644
--- a/src/lib/net/socketpair.h
+++ b/src/lib/net/socketpair.h
@@ -1,11 +1,16 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SOCKETPAIR_H
 #define TOR_SOCKETPAIR_H
 
+/**
+ * @file socketpair.h
+ * @brief Header for socketpair.c
+ **/
+
 #include "orconfig.h"
 #include "lib/testsupport/testsupport.h"
 #include "lib/net/nettypes.h"
@@ -16,4 +21,4 @@ int tor_ersatz_socketpair(int family, int type, int protocol,
                           tor_socket_t fd[2]);
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKETPAIR_H) */
diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h
index e55242ce66..90c4305d29 100644
--- a/src/lib/net/socks5_status.h
+++ b/src/lib/net/socks5_status.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,6 +27,17 @@ typedef enum {
   SOCKS5_TTL_EXPIRED                = 0x06,
   SOCKS5_COMMAND_NOT_SUPPORTED      = 0x07,
   SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08,
+
+  /* Extended error code (see prop304). Only used if the SocksPort flag
+   * "ExtendedErrors" is set. */
+  SOCKS5_HS_NOT_FOUND               = 0xF0,
+  SOCKS5_HS_IS_INVALID              = 0xF1,
+  SOCKS5_HS_INTRO_FAILED            = 0xF2,
+  SOCKS5_HS_REND_FAILED             = 0xF3,
+  SOCKS5_HS_MISSING_CLIENT_AUTH     = 0xF4,
+  SOCKS5_HS_BAD_CLIENT_AUTH         = 0xF5,
+  SOCKS5_HS_BAD_ADDRESS             = 0xF6,
+  SOCKS5_HS_INTRO_TIMEDOUT          = 0xF7,
 } socks5_reply_status_t;
 
-#endif
+#endif /* !defined(TOR_SOCKS5_STATUS_H) */
diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am
index 16c5812604..84bd7feb00 100644
--- a/src/lib/osinfo/include.am
+++ b/src/lib/osinfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_osinfo_a_SOURCES =			\
 	src/lib/osinfo/uname.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_osinfo_testing_a_SOURCES = \
 src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/osinfo/uname.h
diff --git a/src/lib/osinfo/lib_osinfo.md b/src/lib/osinfo/lib_osinfo.md
new file mode 100644
index 0000000000..0678ecc21e
--- /dev/null
+++ b/src/lib/osinfo/lib_osinfo.md
@@ -0,0 +1,8 @@
+@dir /lib/osinfo
+@brief lib/osinfo: For inspecting the OS version and capabilities.
+
+In general, we use this module when we're telling the user what operating
+system they are running.  We shouldn't make decisions based on the output of
+these checks: instead, we should have more specific checks, either at compile
+time or run time, based on the observed system behavior.
+
diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c
index 2b37ff136c..f7f5ede307 100644
--- a/src/lib/osinfo/uname.c
+++ b/src/lib/osinfo/uname.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,6 +27,40 @@ static char uname_result[256];
 /** True iff uname_result is set. */
 static int uname_result_is_set = 0;
 
+#ifdef _WIN32
+/** Table to map claimed windows versions into human-readable windows
+ * versions. */
+static struct {
+  unsigned major;
+  unsigned minor;
+  const char *client_version;
+  const char *server_version;
+} win_version_table[] = {
+  /* This table must be sorted in descending order.
+   * Sources:
+   *   https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions
+   *   https://docs.microsoft.com/en-us/windows/desktop/api/winnt/
+   *     ns-winnt-_osversioninfoexa#remarks
+   */
+   /* Windows Server 2019 is indistinguishable from Windows Server 2016
+    * using GetVersionEx().
+   { 10,  0, NULL,                        "Windows Server 2019" }, */
+   // clang-format off
+   { 10,  0, "Windows 10",                "Windows Server 2016" },
+   {  6,  3, "Windows 8.1",               "Windows Server 2012 R2" },
+   {  6,  2, "Windows 8",                 "Windows Server 2012" },
+   {  6,  1, "Windows 7",                 "Windows Server 2008 R2" },
+   {  6,  0, "Windows Vista",             "Windows Server 2008" },
+   {  5,  2, "Windows XP Professional",   "Windows Server 2003" },
+   /* Windows XP did not have a server version, but we need something here */
+   {  5,  1, "Windows XP",                "Windows XP Server" },
+   {  5,  0, "Windows 2000 Professional", "Windows 2000 Server" },
+   /* Earlier versions are not supported by GetVersionEx(). */
+   {  0,  0, NULL,                        NULL }
+   // clang-format on
+};
+#endif /* defined(_WIN32) */
+
 /** Return a pointer to a description of our platform.
  */
 MOCK_IMPL(const char *,
@@ -49,31 +83,6 @@ get_uname,(void))
         int is_client = 0;
         int is_server = 0;
         const char *plat = NULL;
-        static struct {
-          unsigned major; unsigned minor;
-          const char *client_version; const char *server_version;
-        } win_version_table[] = {
-    /* This table must be sorted in descending order.
-     * Sources:
-     *   https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions
-     *   https://docs.microsoft.com/en-us/windows/desktop/api/winnt/
-     *     ns-winnt-_osversioninfoexa#remarks
-     */
-    /* Windows Server 2019 is indistinguishable from Windows Server 2016
-     * using GetVersionEx().
-    { 10,  0, NULL,                        "Windows Server 2019" }, */
-    { 10,  0, "Windows 10",                "Windows Server 2016" },
-    {  6,  3, "Windows 8.1",               "Windows Server 2012 R2" },
-    {  6,  2, "Windows 8",                 "Windows Server 2012" },
-    {  6,  1, "Windows 7",                 "Windows Server 2008 R2" },
-    {  6,  0, "Windows Vista",             "Windows Server 2008" },
-    {  5,  2, "Windows XP Professional",   "Windows Server 2003" },
-    /* Windows XP did not have a server version, but we need something here */
-    {  5,  1, "Windows XP",                "Windows XP Server" },
-    {  5,  0, "Windows 2000 Professional", "Windows 2000 Server" },
-    /* Earlier versions are not supported by GetVersionEx(). */
-    {  0,  0, NULL,                        NULL }
-        };
         memset(&info, 0, sizeof(info));
         info.dwOSVersionInfoSize = sizeof(info);
         if (! GetVersionEx((LPOSVERSIONINFO)&info)) {
@@ -137,7 +146,7 @@ get_uname,(void))
         if (!is_server && !is_client) {
           strlcat(uname_result, " [client or server]", sizeof(uname_result));
         }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
         /* LCOV_EXCL_START -- can't provoke uname failure */
         strlcpy(uname_result, "Unknown platform", sizeof(uname_result));
         /* LCOV_EXCL_STOP */
diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h
index fcce629074..c6b1f43235 100644
--- a/src/lib/osinfo/uname.h
+++ b/src/lib/osinfo/uname.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,4 +15,4 @@
 
 MOCK_DECL(const char *, get_uname,(void));
 
-#endif
+#endif /* !defined(HAVE_TOR_UNAME_H) */
diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include
index 05414d2a96..ce1b6ecf59 100644
--- a/src/lib/process/.may_include
+++ b/src/lib/process/.may_include
@@ -1,17 +1,20 @@
 orconfig.h
 
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
 lib/err/*.h
-lib/intmath/*.h
+lib/evloop/*.h
 lib/fs/*.h
+lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/process/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 
-ht.h
\ No newline at end of file
+ext/ht.h
diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c
index 3b90bef671..b3b98a297e 100644
--- a/src/lib/process/daemon.c
+++ b/src/lib/process/daemon.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd)
 
   return 0;
 }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
 /* defined(_WIN32) */
 int
 start_daemon(void)
diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h
index 20920e0aae..23f3117898 100644
--- a/src/lib/process/daemon.h
+++ b/src/lib/process/daemon.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,4 +18,4 @@ int finish_daemon(const char *desired_cwd);
 
 bool start_daemon_has_been_called(void);
 
-#endif
+#endif /* !defined(TOR_DAEMON_H) */
diff --git a/src/lib/process/env.c b/src/lib/process/env.c
index 0060200ba1..517b655a5e 100644
--- a/src/lib/process/env.c
+++ b/src/lib/process/env.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,6 @@
 #include "lib/container/smartlist.h"
 #include "lib/log/util_bug.h"
 #include "lib/log/log.h"
-#include "lib/malloc/malloc.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -47,7 +46,7 @@ get_environment(void)
    * when we do a mostly-static build on OSX 10.7, the resulting binary won't
    * work on OSX 10.6. */
   return *_NSGetEnviron();
-#else /* !(defined(HAVE__NSGETENVIRON)) */
+#else /* !defined(HAVE__NSGETENVIRON) */
   return environ;
 #endif /* defined(HAVE__NSGETENVIRON) */
 }
diff --git a/src/lib/process/env.h b/src/lib/process/env.h
index 15d59351e0..7838dcaa90 100644
--- a/src/lib/process/env.h
+++ b/src/lib/process/env.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,4 +38,4 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
                                            const char *new_var,
                                            void (*free_old)(void*),
                                            int free_p);
-#endif
+#endif /* !defined(TOR_ENV_H) */
diff --git a/src/lib/process/include.am b/src/lib/process/include.am
index c6cc3a6699..18876b3f54 100644
--- a/src/lib/process/include.am
+++ b/src/lib/process/include.am
@@ -5,13 +5,17 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-process-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_process_a_SOURCES =		\
 	src/lib/process/daemon.c		\
 	src/lib/process/env.c			\
 	src/lib/process/pidfile.c		\
+	src/lib/process/process.c		\
+	src/lib/process/process_sys.c		\
+	src/lib/process/process_unix.c		\
+	src/lib/process/process_win32.c		\
 	src/lib/process/restrict.c		\
 	src/lib/process/setuid.c		\
-	src/lib/process/subprocess.c		\
 	src/lib/process/waitpid.c
 
 src_lib_libtor_process_testing_a_SOURCES = \
@@ -19,11 +23,15 @@ src_lib_libtor_process_testing_a_SOURCES = \
 src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/process/daemon.h		\
 	src/lib/process/env.h			\
 	src/lib/process/pidfile.h		\
+	src/lib/process/process.h		\
+	src/lib/process/process_sys.h		\
+	src/lib/process/process_unix.h		\
+	src/lib/process/process_win32.h		\
 	src/lib/process/restrict.h		\
 	src/lib/process/setuid.h		\
-	src/lib/process/subprocess.h		\
 	src/lib/process/waitpid.h
diff --git a/src/lib/process/lib_process.md b/src/lib/process/lib_process.md
new file mode 100644
index 0000000000..354129e70e
--- /dev/null
+++ b/src/lib/process/lib_process.md
@@ -0,0 +1,2 @@
+@dir /lib/process
+@brief lib/process: Launch and manage subprocesses.
diff --git a/src/lib/process/pidfile.c b/src/lib/process/pidfile.c
index 1b9d1c6d25..e7d9d2c47a 100644
--- a/src/lib/process/pidfile.c
+++ b/src/lib/process/pidfile.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h
index dfeb39e046..d04302df3a 100644
--- a/src/lib/process/pidfile.h
+++ b/src/lib/process/pidfile.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,4 +13,4 @@
 
 int write_pidfile(const char *filename);
 
-#endif
+#endif /* !defined(TOR_PIDFILE_H) */
diff --git a/src/lib/process/process.c b/src/lib/process/process.c
new file mode 100644
index 0000000000..12c1f9a772
--- /dev/null
+++ b/src/lib/process/process.c
@@ -0,0 +1,798 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.c
+ * \brief Module for working with other processes.
+ **/
+
+#define PROCESS_PRIVATE
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_STDDEF_H
+#include <stddef.h>
+#endif
+
+/** A list of all <b>process_t</b> instances currently allocated. */
+static smartlist_t *processes;
+
+/**
+ * Boolean.  If true, then Tor may call execve or CreateProcess via
+ * tor_spawn_background.
+ **/
+static int may_spawn_background_process = 1;
+
+/** Structure to represent a child process. */
+struct process_t {
+  /** Process status. */
+  process_status_t status;
+
+  /** Which protocol is the process using? */
+  process_protocol_t protocol;
+
+  /** Which function to call when we have data ready from stdout? */
+  process_read_callback_t stdout_read_callback;
+
+  /** Which function to call when we have data ready from stderr? */
+  process_read_callback_t stderr_read_callback;
+
+  /** Which function call when our process terminated? */
+  process_exit_callback_t exit_callback;
+
+  /** Our exit code when the process have terminated. */
+  process_exit_code_t exit_code;
+
+  /** Name of the command we want to execute (for example: /bin/ls). */
+  char *command;
+
+  /** The arguments used for the new process. The format here is one argument
+   * per element of the smartlist_t. On Windows these arguments are combined
+   * together using the <b>tor_join_win_cmdline</b> function. On Unix the
+   * process name (argv[0]) and the trailing NULL is added automatically before
+   * the process is executed. */
+  smartlist_t *arguments;
+
+  /** The environment used for the new process. */
+  smartlist_t *environment;
+
+  /** Buffer to store data from stdout when it is read. */
+  buf_t *stdout_buffer;
+
+  /** Buffer to store data from stderr when it is read. */
+  buf_t *stderr_buffer;
+
+  /** Buffer to store data to stdin before it is written. */
+  buf_t *stdin_buffer;
+
+  /** Do we need to store some custom data with the process? */
+  void *data;
+
+#ifndef _WIN32
+  /** Our Unix process handle. */
+  process_unix_t *unix_process;
+#else
+  /** Our Win32 process handle. */
+  process_win32_t *win32_process;
+#endif /* !defined(_WIN32) */
+};
+
+/** Convert a given process status in <b>status</b> to its string
+ * representation. */
+const char *
+process_status_to_string(process_status_t status)
+{
+  switch (status) {
+  case PROCESS_STATUS_NOT_RUNNING:
+    return "not running";
+  case PROCESS_STATUS_RUNNING:
+    return "running";
+  case PROCESS_STATUS_ERROR:
+    return "error";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/** Convert a given process protocol in <b>protocol</b> to its string
+ * representation. */
+const char *
+process_protocol_to_string(process_protocol_t protocol)
+{
+  switch (protocol) {
+  case PROCESS_PROTOCOL_LINE:
+    return "Line";
+  case PROCESS_PROTOCOL_RAW:
+    return "Raw";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * Turn off may_spawn_background_process, so that all future calls to
+ * tor_spawn_background are guaranteed to fail.
+ **/
+void
+tor_disable_spawning_background_processes(void)
+{
+  may_spawn_background_process = 0;
+}
+
+/** Initialize the Process subsystem.  This function initializes the Process
+ * subsystem's global state. For cleaning up, <b>process_free_all()</b> should
+ * be called. */
+void
+process_init(void)
+{
+  processes = smartlist_new();
+
+#ifdef _WIN32
+  process_win32_init();
+#endif
+}
+
+/** Free up all resources that is handled by the Process subsystem. Note that
+ * this call does not terminate already running processes. */
+void
+process_free_all(void)
+{
+#ifdef _WIN32
+  process_win32_deinit();
+#endif
+
+  SMARTLIST_FOREACH(processes, process_t *, x, process_free(x));
+  smartlist_free(processes);
+}
+
+/** Get a list of all processes.  This function returns a smartlist of
+ * <b>process_t</b> containing all the currently allocated processes. */
+const smartlist_t *
+process_get_all_processes(void)
+{
+  return processes;
+}
+
+/** Allocate and initialize a new process.  This function returns a newly
+ * allocated and initialized process data, which can be used to configure and
+ * later run a subprocess of Tor. Use the various <b>process_set_*()</b>
+ * methods to configure it and run the process using <b>process_exec()</b>. Use
+ * <b>command</b> to specify the path to the command to run. You can either
+ * specify an absolute path to the command or relative where Tor will use the
+ * underlying operating system's functionality for finding the command to run.
+ * */
+process_t *
+process_new(const char *command)
+{
+  tor_assert(command);
+
+  process_t *process;
+  process = tor_malloc_zero(sizeof(process_t));
+
+  /* Set our command. */
+  process->command = tor_strdup(command);
+
+  /* By default we are not running. */
+  process->status = PROCESS_STATUS_NOT_RUNNING;
+
+  /* Prepare process environment. */
+  process->arguments = smartlist_new();
+  process->environment = smartlist_new();
+
+  /* Prepare the buffers. */
+  process->stdout_buffer = buf_new();
+  process->stderr_buffer = buf_new();
+  process->stdin_buffer = buf_new();
+
+#ifndef _WIN32
+  /* Prepare our Unix process handle. */
+  process->unix_process = process_unix_new();
+#else
+  /* Prepare our Win32 process handle. */
+  process->win32_process = process_win32_new();
+#endif /* !defined(_WIN32) */
+
+  smartlist_add(processes, process);
+
+  return process;
+}
+
+/** Deallocate the given process in <b>process</b>. */
+void
+process_free_(process_t *process)
+{
+  if (! process)
+    return;
+
+  /* Cleanup parameters. */
+  tor_free(process->command);
+
+  /* Cleanup arguments and environment. */
+  SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x));
+  smartlist_free(process->arguments);
+
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+
+  /* Cleanup the buffers. */
+  buf_free(process->stdout_buffer);
+  buf_free(process->stderr_buffer);
+  buf_free(process->stdin_buffer);
+
+#ifndef _WIN32
+  /* Cleanup our Unix process handle. */
+  process_unix_free(process->unix_process);
+#else
+  /* Cleanup our Win32 process handle. */
+  process_win32_free(process->win32_process);
+#endif /* !defined(_WIN32) */
+
+  smartlist_remove(processes, process);
+
+  tor_free(process);
+}
+
+/** Execute the given process.  This function executes the given process as a
+ * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_exec(process_t *process)
+{
+  tor_assert(process);
+
+  if (BUG(may_spawn_background_process == 0))
+    return PROCESS_STATUS_ERROR;
+
+  process_status_t status = PROCESS_STATUS_NOT_RUNNING;
+
+  log_info(LD_PROCESS, "Starting new process: %s", process->command);
+
+#ifndef _WIN32
+  status = process_unix_exec(process);
+#else
+  status = process_win32_exec(process);
+#endif
+
+  /* Update our state. */
+  process_set_status(process, status);
+
+  if (status != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_PROCESS, "Failed to start process: %s",
+             process_get_command(process));
+  }
+
+  return status;
+}
+
+/** Terminate the given process. Returns true on success,
+ * otherwise false. */
+bool
+process_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  /* Terminating a non-running process isn't going to work. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  log_debug(LD_PROCESS, "Terminating process");
+
+#ifndef _WIN32
+  return process_unix_terminate(process);
+#else
+  return process_win32_terminate(process);
+#endif
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+#ifndef _WIN32
+  return process_unix_get_pid(process);
+#else
+  return process_win32_get_pid(process);
+#endif
+}
+
+/** Set the callback function for output from the child process's standard out
+ * handle.  This function sets the callback function which is called every time
+ * the child process have written output to its standard out file handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stdout_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stdout_read_callback = callback;
+}
+
+/** Set the callback function for output from the child process's standard
+ * error handle.  This function sets the callback function which is called
+ * every time the child process have written output to its standard error file
+ * handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stderr_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stderr_read_callback = callback;
+}
+
+/** Set the callback function for process exit notification.  The
+ * <b>callback</b> function will be called every time your child process have
+ * terminated. */
+void
+process_set_exit_callback(process_t *process,
+                          process_exit_callback_t callback)
+{
+  tor_assert(process);
+  process->exit_callback = callback;
+}
+
+/** Get the current command of the given process. */
+const char *
+process_get_command(const process_t *process)
+{
+  tor_assert(process);
+  return process->command;
+}
+
+void
+process_set_protocol(process_t *process, process_protocol_t protocol)
+{
+  tor_assert(process);
+  process->protocol = protocol;
+}
+
+/** Get the currently used protocol of the given process. */
+process_protocol_t
+process_get_protocol(const process_t *process)
+{
+  tor_assert(process);
+  return process->protocol;
+}
+
+/** Set opague pointer to data.  This function allows you to store a pointer to
+ * your own data in the given process. Use <b>process_get_data()</b> in the
+ * various callback functions to retrieve the data again.
+ *
+ * Note that the given process does NOT take ownership of the data and you are
+ * responsible for freeing up any resources allocated by the given data.
+ * */
+void
+process_set_data(process_t *process, void *data)
+{
+  tor_assert(process);
+  process->data = data;
+}
+
+/** Get the opaque pointer to callback data from the given process.  This
+ * function allows you get the data you stored with <b>process_set_data()</b>
+ * in the different callback functions. */
+void *
+process_get_data(const process_t *process)
+{
+  tor_assert(process);
+  return process->data;
+}
+
+/** Set the status of a given process. */
+void
+process_set_status(process_t *process, process_status_t status)
+{
+  tor_assert(process);
+  process->status = status;
+}
+
+/** Get the status of the given process. */
+process_status_t
+process_get_status(const process_t *process)
+{
+  tor_assert(process);
+  return process->status;
+}
+
+/** Append an argument to the list of arguments in the given process. */
+void
+process_append_argument(process_t *process, const char *argument)
+{
+  tor_assert(process);
+  tor_assert(argument);
+
+  smartlist_add(process->arguments, tor_strdup(argument));
+}
+
+/** Returns a list of arguments (excluding the command itself) from the
+ * given process. */
+const smartlist_t *
+process_get_arguments(const process_t *process)
+{
+  tor_assert(process);
+  return process->arguments;
+}
+
+/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b>
+ * to deallocate it after use. */
+char **
+process_get_argv(const process_t *process)
+{
+  tor_assert(process);
+
+  /** Generate a Unix style process argument vector from our process's
+   * arguments smartlist_t. */
+  char **argv = NULL;
+
+  char *filename = process->command;
+  const smartlist_t *arguments = process->arguments;
+  const size_t size = smartlist_len(arguments);
+
+  /* Make space for the process filename as argv[0] and a trailing NULL. */
+  argv = tor_malloc_zero(sizeof(char *) * (size + 2));
+
+  /* Set our filename as first argument. */
+  argv[0] = filename;
+
+  /* Put in the rest of the values from arguments. */
+  SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) {
+    tor_assert(arg_val != NULL);
+
+    argv[arg_val_sl_idx + 1] = arg_val;
+  } SMARTLIST_FOREACH_END(arg_val);
+
+  return argv;
+}
+
+/** This function clears the internal environment and copies over every string
+ * from <b>env</b> as the new environment. */
+void
+process_reset_environment(process_t *process, const smartlist_t *env)
+{
+  tor_assert(process);
+  tor_assert(env);
+
+  /* Cleanup old environment. */
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+  process->environment = smartlist_new();
+
+  SMARTLIST_FOREACH(env, char *, x,
+                    smartlist_add(process->environment, tor_strdup(x)));
+}
+
+/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the
+ * given process. */
+void
+process_set_environment(process_t *process,
+                        const char *key,
+                        const char *value)
+{
+  tor_assert(process);
+  tor_assert(key);
+  tor_assert(value);
+
+  smartlist_add_asprintf(process->environment, "%s=%s", key, value);
+}
+
+/** Returns a newly allocated <b>process_environment_t</b> containing the
+ * environment variables for the given process. */
+process_environment_t *
+process_get_environment(const process_t *process)
+{
+  tor_assert(process);
+  return process_environment_make(process->environment);
+}
+
+#ifndef _WIN32
+/** Get the internal handle for the Unix backend. */
+process_unix_t *
+process_get_unix_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->unix_process);
+  return process->unix_process;
+}
+#else /* defined(_WIN32) */
+/** Get the internal handle for Windows backend. */
+process_win32_t *
+process_get_win32_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->win32_process);
+  return process->win32_process;
+}
+#endif /* !defined(_WIN32) */
+
+/** Write <b>size</b> bytes of <b>data</b> to the given process's standard
+ * input. */
+void
+process_write(process_t *process,
+              const uint8_t *data, size_t size)
+{
+  tor_assert(process);
+  tor_assert(data);
+
+  buf_add(process->stdin_buffer, (char *)data, size);
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** As tor_vsnprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_vprintf(process_t *process,
+                const char *format, va_list args)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  int size;
+  char *data;
+
+  size = tor_vasprintf(&data, format, args);
+  tor_assert(data != NULL);
+  process_write(process, (uint8_t *)data, size);
+  tor_free(data);
+}
+
+/** As tor_snprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_printf(process_t *process,
+               const char *format, ...)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  va_list ap;
+  va_start(ap, format);
+  process_vprintf(process, format, ap);
+  va_end(ap);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard output
+ * handle. */
+void
+process_notify_event_stdout(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stdout(process, process->stdout_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stdout_buffer,
+                      process->stdout_read_callback);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard error
+ * handle. */
+void
+process_notify_event_stderr(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stderr(process, process->stderr_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stderr_buffer,
+                      process->stderr_read_callback);
+}
+
+/** This function is called by the Process backend when a given process is
+ * allowed to begin writing data to the standard input of the child process. */
+void
+process_notify_event_stdin(process_t *process)
+{
+  tor_assert(process);
+
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** This function is called by the Process backend when a given process have
+ * terminated. The exit status code is passed in <b>exit_code</b>. We mark the
+ * process as no longer running and calls the <b>exit_callback</b> with
+ * information about the process termination. The given <b>process</b> is
+ * free'd iff the exit_callback returns true. */
+void
+process_notify_event_exit(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_debug(LD_PROCESS,
+            "Process terminated with exit code: %"PRIu64, exit_code);
+
+  /* Update our state. */
+  process_set_status(process, PROCESS_STATUS_NOT_RUNNING);
+  process->exit_code = exit_code;
+
+  /* Call our exit callback, if it exists. */
+  bool free_process_handle = false;
+
+  /* The exit callback will tell us if we should process_free() our handle. */
+  if (process->exit_callback)
+    free_process_handle = process->exit_callback(process, exit_code);
+
+  if (free_process_handle)
+    process_free(process);
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard out handle. Returns the number of
+ * bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stdout(process, buffer);
+#else
+  return process_win32_read_stdout(process, buffer);
+#endif
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard error handle. Returns the number
+ * of bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stderr(process, buffer);
+#else
+  return process_win32_read_stderr(process, buffer);
+#endif
+}
+
+/** This function calls the backend function for the given process whenever
+ * there is data to be written to the backends' file handles. */
+MOCK_IMPL(STATIC void, process_write_stdin,
+          (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  process_unix_write(process, buffer);
+#else
+  process_win32_write(process, buffer);
+#endif
+}
+
+/** This function calls the protocol handlers based on the value of
+ * <b>process_get_protocol(process)</b>. Currently we call
+ * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and
+ * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */
+STATIC void
+process_read_data(process_t *process,
+                  buf_t *buffer,
+                  process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  switch (process_get_protocol(process)) {
+  case PROCESS_PROTOCOL_RAW:
+    process_read_buffer(process, buffer, callback);
+    break;
+  case PROCESS_PROTOCOL_LINE:
+    process_read_lines(process, buffer, callback);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_unreached();
+    return;
+    /* LCOV_EXCL_STOP */
+  }
+}
+
+/** This function takes the content of the given <b>buffer</b> and passes it to
+ * the given <b>callback</b> function, but ensures that an additional zero byte
+ * is added to the end of the data such that the given callback implementation
+ * can threat the content as a ASCIIZ string. */
+STATIC void
+process_read_buffer(process_t *process,
+                    buf_t *buffer,
+                    process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer);
+
+  /* We allocate an extra byte for the zero byte in the end. */
+  char *data = tor_malloc_zero(size + 1);
+
+  buf_get_bytes(buffer, data, size);
+  log_debug(LD_PROCESS, "Read data from process");
+
+  if (callback)
+    callback(process, data, size);
+
+  tor_free(data);
+}
+
+/** This function tries to extract complete lines from the given <b>buffer</b>
+ * and calls the given <b>callback</b> function whenever it has a complete
+ * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n"
+ * from the line. If we are unable to extract a complete line we leave the data
+ * in the buffer for next call. */
+STATIC void
+process_read_lines(process_t *process,
+                   buf_t *buffer,
+                   process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer) + 1;
+  size_t line_size = 0;
+  char *data = tor_malloc_zero(size);
+  int ret;
+
+  while (true) {
+    line_size = size;
+    ret = buf_get_line(buffer, data, &line_size);
+
+    /* A complete line should always be smaller than the size of our
+     * buffer. */
+    tor_assert(ret != -1);
+
+    /* Remove \n from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\n') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    /* Remove \r from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\r') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    if (ret == 1) {
+      log_debug(LD_PROCESS, "Read line from process: \"%s\"", data);
+
+      if (callback)
+        callback(process, data, line_size);
+
+      /* We have read a whole line, let's see if there is more lines to read.
+       * */
+      continue;
+    }
+
+    /* No complete line for us to read. We are done for now. */
+    tor_assert_nonfatal(ret == 0);
+    break;
+  }
+
+  tor_free(data);
+}
diff --git a/src/lib/process/process.h b/src/lib/process/process.h
new file mode 100644
index 0000000000..8879ec4f21
--- /dev/null
+++ b/src/lib/process/process.h
@@ -0,0 +1,148 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.h
+ * \brief Header for process.c
+ **/
+
+#ifndef TOR_PROCESS_H
+#define TOR_PROCESS_H
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdbool.h>
+
+/** Maximum number of bytes to write to a process' stdin. */
+#define PROCESS_MAX_WRITE (1024)
+
+/** Maximum number of bytes to read from a process' stdout/stderr. */
+#define PROCESS_MAX_READ  (1024)
+
+typedef enum {
+  /** The process is not running. */
+  PROCESS_STATUS_NOT_RUNNING,
+
+  /** The process is running. */
+  PROCESS_STATUS_RUNNING,
+
+  /** The process is in an erroneous state. */
+  PROCESS_STATUS_ERROR
+} process_status_t;
+
+const char *process_status_to_string(process_status_t status);
+
+typedef enum {
+  /** Pass complete newline-terminated lines to the
+   *  callback (with the LF or CRLF removed). */
+  PROCESS_PROTOCOL_LINE,
+
+  /** Pass the raw response from read() to the callback. */
+  PROCESS_PROTOCOL_RAW
+} process_protocol_t;
+
+const char *process_protocol_to_string(process_protocol_t protocol);
+
+void tor_disable_spawning_background_processes(void);
+
+struct smartlist_t;
+
+struct process_t;
+typedef struct process_t process_t;
+
+typedef uint64_t process_exit_code_t;
+typedef uint64_t process_pid_t;
+
+typedef void (*process_read_callback_t)(process_t *,
+                                        const char *,
+                                        size_t);
+typedef bool
+(*process_exit_callback_t)(process_t *, process_exit_code_t);
+
+void process_init(void);
+void process_free_all(void);
+const struct smartlist_t *process_get_all_processes(void);
+
+process_t *process_new(const char *command);
+void process_free_(process_t *process);
+#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s))
+
+process_status_t process_exec(process_t *process);
+bool process_terminate(process_t *process);
+
+process_pid_t process_get_pid(process_t *process);
+
+void process_set_stdout_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_stderr_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_exit_callback(process_t *,
+                               process_exit_callback_t);
+
+const char *process_get_command(const process_t *process);
+
+void process_append_argument(process_t *process, const char *argument);
+const struct smartlist_t *process_get_arguments(const process_t *process);
+char **process_get_argv(const process_t *process);
+
+void process_reset_environment(process_t *process,
+                               const struct smartlist_t *env);
+void process_set_environment(process_t *process,
+                             const char *key,
+                             const char *value);
+
+struct process_environment_t;
+struct process_environment_t *process_get_environment(const process_t *);
+
+void process_set_protocol(process_t *process, process_protocol_t protocol);
+process_protocol_t process_get_protocol(const process_t *process);
+
+void process_set_data(process_t *process, void *data);
+void *process_get_data(const process_t *process);
+
+void process_set_status(process_t *process, process_status_t status);
+process_status_t process_get_status(const process_t *process);
+
+#ifndef _WIN32
+struct process_unix_t;
+struct process_unix_t *process_get_unix_process(const process_t *process);
+#else
+struct process_win32_t;
+struct process_win32_t *process_get_win32_process(const process_t *process);
+#endif /* !defined(_WIN32) */
+
+void process_write(process_t *process,
+                   const uint8_t *data, size_t size);
+void process_vprintf(process_t *process,
+                     const char *format, va_list args) CHECK_PRINTF(2, 0);
+void process_printf(process_t *process,
+                    const char *format, ...) CHECK_PRINTF(2, 3);
+
+void process_notify_event_stdout(process_t *process);
+void process_notify_event_stderr(process_t *process);
+void process_notify_event_stdin(process_t *process);
+void process_notify_event_exit(process_t *process,
+                               process_exit_code_t);
+
+#ifdef PROCESS_PRIVATE
+struct buf_t;
+MOCK_DECL(STATIC int, process_read_stdout, (process_t *, struct buf_t *));
+MOCK_DECL(STATIC int, process_read_stderr, (process_t *, struct buf_t *));
+MOCK_DECL(STATIC void, process_write_stdin, (process_t *, struct buf_t *));
+
+STATIC void process_read_data(process_t *process,
+                              struct buf_t *buffer,
+                              process_read_callback_t callback);
+STATIC void process_read_buffer(process_t *process,
+                                struct buf_t *buffer,
+                                process_read_callback_t callback);
+STATIC void process_read_lines(process_t *process,
+                               struct buf_t *buffer,
+                               process_read_callback_t callback);
+#endif /* defined(PROCESS_PRIVATE) */
+
+#endif /* !defined(TOR_PROCESS_H) */
diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c
new file mode 100644
index 0000000000..c8332ba91e
--- /dev/null
+++ b/src/lib/process/process_sys.c
@@ -0,0 +1,34 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.c
+ * \brief Subsystem object for process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/process/process_sys.h"
+#include "lib/process/process.h"
+
+static int
+subsys_process_initialize(void)
+{
+  process_init();
+  return 0;
+}
+
+static void
+subsys_process_shutdown(void)
+{
+  process_free_all();
+}
+
+const subsys_fns_t sys_process = {
+  .name = "process",
+  SUBSYS_DECLARE_LOCATION(),
+  .level = -18,
+  .supported = true,
+  .initialize = subsys_process_initialize,
+  .shutdown = subsys_process_shutdown
+};
diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h
new file mode 100644
index 0000000000..97b3aaebd0
--- /dev/null
+++ b/src/lib/process/process_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.h
+ * \brief Declare subsystem object for the process module.
+ **/
+
+#ifndef TOR_PROCESS_SYS_H
+#define TOR_PROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_process;
+
+#endif /* !defined(TOR_PROCESS_SYS_H) */
diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c
new file mode 100644
index 0000000000..2b47e1874d
--- /dev/null
+++ b/src/lib/process/process_unix.c
@@ -0,0 +1,698 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.c
+ * \brief Module for working with Unix processes.
+ **/
+
+#define PROCESS_UNIX_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/waitpid.h"
+#include "lib/process/env.h"
+
+#include <stdio.h>
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+#include <sys/prctl.h>
+#endif
+
+#if HAVE_SIGNAL_H
+#include <signal.h>
+#endif
+
+#ifndef _WIN32
+
+/** Maximum number of file descriptors, if we cannot get it via sysconf() */
+#define DEFAULT_MAX_FD 256
+
+/** Internal state for Unix handles. */
+struct process_unix_handle_t {
+  /** Unix File Descriptor. */
+  int fd;
+
+  /** Have we reached end of file? */
+  bool reached_eof;
+
+  /** Event structure for libevent. */
+  struct event *event;
+
+  /** Are we writing? */
+  bool is_writing;
+};
+
+/** Internal state for our Unix process. */
+struct process_unix_t {
+  /** Standard in handle. */
+  process_unix_handle_t stdin_handle;
+
+  /** Standard out handle. */
+  process_unix_handle_t stdout_handle;
+
+  /** Standard error handle. */
+  process_unix_handle_t stderr_handle;
+
+  /** The process identifier of our process. */
+  pid_t pid;
+
+  /** Waitpid Callback structure. */
+  waitpid_callback_t *waitpid;
+};
+
+/** Returns a newly allocated <b>process_unix_t</b>. */
+process_unix_t *
+process_unix_new(void)
+{
+  process_unix_t *unix_process;
+  unix_process = tor_malloc_zero(sizeof(process_unix_t));
+
+  unix_process->stdin_handle.fd = -1;
+  unix_process->stderr_handle.fd = -1;
+  unix_process->stdout_handle.fd = -1;
+
+  return unix_process;
+}
+
+/** Deallocates the given <b>unix_process</b>. */
+void
+process_unix_free_(process_unix_t *unix_process)
+{
+  if (! unix_process)
+    return;
+
+  /* Clean up our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+
+  /* FIXME(ahf): Refactor waitpid code? */
+  unix_process->waitpid = NULL;
+
+  /* Close all our file descriptors. */
+  process_unix_close_file_descriptors(unix_process);
+
+  tor_event_free(unix_process->stdout_handle.event);
+  tor_event_free(unix_process->stderr_handle.event);
+  tor_event_free(unix_process->stdin_handle.event);
+
+  tor_free(unix_process);
+}
+
+/** Executes the given process as a child process of Tor.  This function is
+ * responsible for setting up the child process and run it. This includes
+ * setting up pipes for interprocess communication, initialize the waitpid
+ * callbacks, and finally run fork() followed by execve(). Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_unix_exec(process_t *process)
+{
+  static int max_fd = -1;
+
+  process_unix_t *unix_process;
+  pid_t pid;
+  int stdin_pipe[2];
+  int stdout_pipe[2];
+  int stderr_pipe[2];
+  int retval, fd;
+
+  unix_process = process_get_unix_process(process);
+
+  /* Create standard in pipe. */
+  retval = pipe(stdin_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdin "
+             "communication with process: %s",
+             strerror(errno));
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard out pipe. */
+  retval = pipe(stdout_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdout "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard error pipe. */
+  retval = pipe(stderr_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stderr "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+#ifdef _SC_OPEN_MAX
+  if (-1 == max_fd) {
+    max_fd = (int)sysconf(_SC_OPEN_MAX);
+
+    if (max_fd == -1) {
+      max_fd = DEFAULT_MAX_FD;
+      log_warn(LD_PROCESS,
+               "Cannot find maximum file descriptor, assuming: %d", max_fd);
+    }
+  }
+#else /* !defined(_SC_OPEN_MAX) */
+  max_fd = DEFAULT_MAX_FD;
+#endif /* defined(_SC_OPEN_MAX) */
+
+  pid = fork();
+
+  if (0 == pid) {
+    /* This code is running in the child process context. */
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+    /* Attempt to have the kernel issue a SIGTERM if the parent
+     * goes away. Certain attributes of the binary being execve()ed
+     * will clear this during the execve() call, but it's better
+     * than nothing.
+     */
+    prctl(PR_SET_PDEATHSIG, SIGTERM);
+#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
+
+    /* Link process stdout to the write end of the pipe. */
+    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stderr to the write end of the pipe. */
+    retval = dup2(stderr_pipe[1], STDERR_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stdin to the read end of the pipe */
+    retval = dup2(stdin_pipe[0], STDIN_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Close our pipes now after they have been dup2()'ed. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /* Close all other fds, including the read end of the pipe.  XXX: We should
+     * now be doing enough FD_CLOEXEC setting to make this needless.
+     */
+    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++)
+      close(fd);
+
+    /* Create the argv value for our new process. */
+    char **argv = process_get_argv(process);
+
+    /* Create the env value for our new process. */
+    process_environment_t *env = process_get_environment(process);
+
+    /* Call the requested program. */
+    execve(argv[0], argv, env->unixoid_environment_block);
+
+    /* If we made it here it is because execve failed :-( */
+    tor_free(argv);
+    process_environment_free(env);
+
+ error:
+    fprintf(stderr, "Error from child process: %s", strerror(errno));
+    _exit(1);
+  }
+
+  /* We are in the parent process. */
+  if (-1 == pid) {
+    log_warn(LD_PROCESS,
+             "Failed to create child process: %s", strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    /** Cleanup standard error pipe. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Register our PID. */
+  unix_process->pid = pid;
+
+  /* Setup waitpid callbacks. */
+  unix_process->waitpid = set_waitpid_callback(pid,
+                                               process_unix_waitpid_callback,
+                                               process);
+
+  /* Handle standard out. */
+  unix_process->stdout_handle.fd = stdout_pipe[0];
+  retval = close(stdout_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard error. */
+  unix_process->stderr_handle.fd = stderr_pipe[0];
+  retval = close(stderr_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Failed to close write end of standard error pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard in. */
+  unix_process->stdin_handle.fd = stdin_pipe[1];
+  retval = close(stdin_pipe[0]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s",
+             strerror(errno));
+  }
+
+  /* Setup our handles. */
+  process_unix_setup_handle(process,
+                            &unix_process->stdout_handle,
+                            EV_READ|EV_PERSIST,
+                            stdout_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stderr_handle,
+                            EV_READ|EV_PERSIST,
+                            stderr_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stdin_handle,
+                            EV_WRITE|EV_PERSIST,
+                            stdin_write_callback);
+
+  /* Start reading from standard out and standard error. */
+  process_unix_start_reading(&unix_process->stdout_handle);
+  process_unix_start_reading(&unix_process->stderr_handle);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_unix_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* All running processes should have a waitpid. */
+  if (BUG(unix_process->waitpid == NULL))
+    return false;
+
+  bool success = true;
+
+  /* Send a SIGTERM to our child process. */
+  int ret;
+
+  ret = kill(unix_process->pid, SIGTERM);
+
+  if (ret == -1) {
+    log_warn(LD_PROCESS, "Unable to terminate process: %s",
+             strerror(errno));
+    success = false;
+  }
+
+  /* Close all our FD's. */
+  if (! process_unix_close_file_descriptors(unix_process))
+    success = false;
+
+  return success;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_unix_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+  return (process_pid_t)unix_process->pid;
+}
+
+/** Write the given <b>buffer</b> as input to the given <b>process</b>'s
+ * standard input. Returns the number of bytes written. */
+int
+process_unix_write(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  size_t buffer_flush_len = buf_datalen(buffer);
+  const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len);
+
+  /* If we have data to write (when buffer_flush_len > 0) and we are not
+   * currently getting file descriptor events from the kernel, we tell the
+   * kernel to start notifying us about when we can write to our file
+   * descriptor and return. */
+  if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) {
+    process_unix_start_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We don't have any data to write, but the kernel is currently notifying us
+   * about whether we are able to write or not. Tell the kernel to stop
+   * notifying us until we have data to write. */
+  if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) {
+    process_unix_stop_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We have data to write and the kernel have told us to write it. */
+  return buf_flush_to_pipe(buffer,
+                           process_get_unix_process(process)->stdin_handle.fd,
+                           max_to_write, &buffer_flush_len);
+}
+
+/** Read data from the given process's standard output and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stdout(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stdout_handle,
+                                  buffer);
+}
+
+/** Read data from the given process's standard error and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stderr(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stderr_handle,
+                                  buffer);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard output. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stdout_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdout(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard error. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stderr_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stderr(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * written the child process's standard input. We notify the Process subsystem,
+ * which is then responsible for calling back to us for doing the actual write
+ * of the data. */
+STATIC void
+stdin_write_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdin(process);
+}
+
+/** This function tells libevent that we are interested in receiving read
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * read events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+}
+
+/** This function tells libevent that we are interested in receiving write
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+
+  handle->is_writing = true;
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * write events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+
+  handle->is_writing = false;
+}
+
+/** This function is called when the waitpid system have detected that our
+ * process have terminated. We disable the waitpid system and notify the
+ * Process subsystem that we have terminated. */
+STATIC void
+process_unix_waitpid_callback(int status, void *data)
+{
+  tor_assert(data);
+
+  process_t *process = data;
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* Remove our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+  unix_process->waitpid = NULL;
+
+  /* Notify our process. */
+  process_notify_event_exit(process, status);
+
+  /* Make sure you don't modify the process after we have called
+   * process_notify_event_exit() on it, to allow users to process_free() it in
+   * the exit callback. */
+}
+
+/** This function sets the file descriptor in the <b>handle</b> as non-blocking
+ * and configures the libevent event structure based on the given <b>flags</b>
+ * to ensure that <b>callback</b> is called whenever we have events on the
+ * given <b>handle</b>. */
+STATIC void
+process_unix_setup_handle(process_t *process,
+                          process_unix_handle_t *handle,
+                          short flags,
+                          event_callback_fn callback)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(callback);
+
+  /* Put our file descriptor into non-blocking mode. */
+  if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) {
+    log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s",
+             strerror(errno));
+  }
+
+  /* Setup libevent event. */
+  handle->event = tor_event_new(tor_libevent_get_base(),
+                                handle->fd,
+                                flags,
+                                callback,
+                                process);
+}
+
+/** This function reads data from the given <b>handle</b> and puts it into
+ * <b>buffer</b>. Returns the number of bytes read this way. */
+STATIC int
+process_unix_read_handle(process_t *process,
+                         process_unix_handle_t *handle,
+                         buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(buffer);
+
+  int ret = 0;
+  int eof = 0;
+  int error = 0;
+
+  ret = buf_read_from_pipe(buffer,
+                           handle->fd,
+                           PROCESS_MAX_READ,
+                           &eof,
+                           &error);
+
+  if (error)
+    log_warn(LD_PROCESS,
+             "Unable to read data: %s", strerror(error));
+
+  if (eof) {
+    handle->reached_eof = true;
+    process_unix_stop_reading(handle);
+  }
+
+  return ret;
+}
+
+/** Close the standard in, out, and error handles of the given
+ * <b>unix_process</b>. */
+STATIC bool
+process_unix_close_file_descriptors(process_unix_t *unix_process)
+{
+  tor_assert(unix_process);
+
+  int ret;
+  bool success = true;
+
+  /* Stop reading and writing before we close() our
+   * file descriptors. */
+  if (! unix_process->stdout_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stdout_handle);
+
+  if (! unix_process->stderr_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stderr_handle);
+
+  if (unix_process->stdin_handle.is_writing)
+    process_unix_stop_writing(&unix_process->stdin_handle);
+
+  if (unix_process->stdin_handle.fd != -1) {
+    ret = close(unix_process->stdin_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard in");
+      success = false;
+    }
+
+    unix_process->stdin_handle.fd = -1;
+  }
+
+  if (unix_process->stdout_handle.fd != -1) {
+    ret = close(unix_process->stdout_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard out");
+      success = false;
+    }
+
+    unix_process->stdout_handle.fd = -1;
+  }
+
+  if (unix_process->stderr_handle.fd != -1) {
+    ret = close(unix_process->stderr_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard error");
+      success = false;
+    }
+
+    unix_process->stderr_handle.fd = -1;
+  }
+
+  return success;
+}
+
+#endif /* !defined(_WIN32) */
diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h
new file mode 100644
index 0000000000..a6b8304d48
--- /dev/null
+++ b/src/lib/process/process_unix.h
@@ -0,0 +1,68 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.h
+ * \brief Header for process_unix.c
+ **/
+
+#ifndef TOR_PROCESS_UNIX_H
+#define TOR_PROCESS_UNIX_H
+
+#ifndef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+
+#include <event2/event.h>
+
+struct process_t;
+
+struct process_unix_t;
+typedef struct process_unix_t process_unix_t;
+
+process_unix_t *process_unix_new(void);
+void process_unix_free_(process_unix_t *unix_process);
+#define process_unix_free(s) \
+  FREE_AND_NULL(process_unix_t, process_unix_free_, (s))
+
+process_status_t process_unix_exec(struct process_t *process);
+bool process_unix_terminate(struct process_t *process);
+
+process_pid_t process_unix_get_pid(struct process_t *process);
+
+int process_unix_write(struct process_t *process, buf_t *buffer);
+int process_unix_read_stdout(struct process_t *process, buf_t *buffer);
+int process_unix_read_stderr(struct process_t *process, buf_t *buffer);
+
+#ifdef PROCESS_UNIX_PRIVATE
+struct process_unix_handle_t;
+typedef struct process_unix_handle_t process_unix_handle_t;
+
+STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data);
+
+STATIC void process_unix_start_reading(process_unix_handle_t *);
+STATIC void process_unix_stop_reading(process_unix_handle_t *);
+
+STATIC void process_unix_start_writing(process_unix_handle_t *);
+STATIC void process_unix_stop_writing(process_unix_handle_t *);
+
+STATIC void process_unix_waitpid_callback(int status, void *data);
+
+STATIC void process_unix_setup_handle(process_t *process,
+                                      process_unix_handle_t *handle,
+                                      short flags,
+                                      event_callback_fn callback);
+STATIC int process_unix_read_handle(process_t *,
+                                    process_unix_handle_t *,
+                                    buf_t *);
+STATIC bool process_unix_close_file_descriptors(process_unix_t *);
+#endif /* defined(PROCESS_UNIX_PRIVATE) */
+
+#endif /* !defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_UNIX_H) */
diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c
new file mode 100644
index 0000000000..8683e17fc2
--- /dev/null
+++ b/src/lib/process/process_win32.c
@@ -0,0 +1,1105 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.c
+ * \brief Module for working with Windows processes.
+ **/
+
+#define PROCESS_WIN32_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/process/process.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef _WIN32
+
+/** The size of our intermediate buffers. */
+#define BUFFER_SIZE (1024)
+
+/** Timer that ticks once a second and calls the process_win32_timer_callback()
+ * function. */
+static periodic_timer_t *periodic_timer;
+
+/** Structure to represent the state around the pipe HANDLE.
+ *
+ * This structure is used to store state about a given HANDLE, including
+ * whether we have reached end of file, its intermediate buffers, and how much
+ * data that is available in the intermediate buffer. */
+struct process_win32_handle_t {
+  /** Standard out pipe handle. */
+  HANDLE pipe;
+
+  /** True iff we have reached EOF from the pipe. */
+  bool reached_eof;
+
+  /** How much data is available in buffer. */
+  size_t data_available;
+
+  /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */
+  char buffer[BUFFER_SIZE];
+
+  /** Overlapped structure for ReadFileEx() and WriteFileEx(). */
+  OVERLAPPED overlapped;
+
+  /** Are we waiting for another I/O operation to complete? */
+  bool busy;
+};
+
+/** Structure to represent the Windows specific implementation details of this
+ * Process backend.
+ *
+ * This structure is attached to <b>process_t</b> (see process.h) and is
+ * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b>
+ * method. */
+struct process_win32_t {
+  /** Standard in state. */
+  process_win32_handle_t stdin_handle;
+
+  /** Standard out state. */
+  process_win32_handle_t stdout_handle;
+
+  /** Standard error state. */
+  process_win32_handle_t stderr_handle;
+
+  /** Process Information. */
+  PROCESS_INFORMATION process_information;
+};
+
+/** Create a new <b>process_win32_t</b>.
+ *
+ * This function constructs a new <b>process_win32_t</b> and initializes the
+ * default values. */
+process_win32_t *
+process_win32_new(void)
+{
+  process_win32_t *win32_process;
+  win32_process = tor_malloc_zero(sizeof(process_win32_t));
+
+  win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE;
+
+  return win32_process;
+}
+
+/** Free a given <b>process_win32_t</b>.
+ *
+ * This function deinitializes and frees up the resources allocated for the
+ * given <b>process_win32_t</b>. */
+void
+process_win32_free_(process_win32_t *win32_process)
+{
+  if (! win32_process)
+    return;
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  tor_free(win32_process);
+}
+
+/** Initialize the Windows backend of the Process subsystem. */
+void
+process_win32_init(void)
+{
+  /* We don't start the periodic timer here because it makes no sense to have
+   * the timer running until we have some processes that benefits from the
+   * timer timer ticks. */
+}
+
+/** Deinitialize the Windows backend of the Process subsystem. */
+void
+process_win32_deinit(void)
+{
+  /* Stop our timer, but only if it's running. */
+  if (process_win32_timer_running())
+    process_win32_timer_stop();
+}
+
+/** Execute the given process. This function is responsible for setting up
+ * named pipes for I/O between the child process and the Tor process. Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_win32_exec(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  HANDLE stdout_pipe_read = NULL;
+  HANDLE stdout_pipe_write = NULL;
+  HANDLE stderr_pipe_read = NULL;
+  HANDLE stderr_pipe_write = NULL;
+  HANDLE stdin_pipe_read = NULL;
+  HANDLE stdin_pipe_write = NULL;
+  BOOL ret = FALSE;
+
+  /* Setup our security attributes. */
+  SECURITY_ATTRIBUTES security_attributes;
+  security_attributes.nLength = sizeof(security_attributes);
+  security_attributes.bInheritHandle = TRUE;
+  /* FIXME: should we set explicit security attributes?
+   * (See Ticket #2046, comment 5) */
+  security_attributes.lpSecurityDescriptor = NULL;
+
+  /* Create our standard out pipe. */
+  if (! process_win32_create_pipe(&stdout_pipe_read,
+                                  &stdout_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create our standard error pipe. */
+  if (! process_win32_create_pipe(&stderr_pipe_read,
+                                  &stderr_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create out standard in pipe. */
+  if (! process_win32_create_pipe(&stdin_pipe_read,
+                                  &stdin_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_WRITER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Configure startup info for our child process. */
+  STARTUPINFOA startup_info;
+
+  memset(&startup_info, 0, sizeof(startup_info));
+  startup_info.cb = sizeof(startup_info);
+  startup_info.hStdError = stderr_pipe_write;
+  startup_info.hStdOutput = stdout_pipe_write;
+  startup_info.hStdInput = stdin_pipe_read;
+  startup_info.dwFlags |= STARTF_USESTDHANDLES;
+
+  /* Create the env value for our new process. */
+  process_environment_t *env = process_get_environment(process);
+
+  /* Create the argv value for our new process. */
+  char **argv = process_get_argv(process);
+
+  /* Windows expects argv to be a whitespace delimited string, so join argv up
+   */
+  char *joined_argv = tor_join_win_cmdline((const char **)argv);
+
+  /* Create the child process */
+  ret = CreateProcessA(NULL,
+                       joined_argv,
+                       NULL,
+                       NULL,
+                       TRUE,
+                       CREATE_NO_WINDOW,
+                       env->windows_environment_block[0] == '\0' ?
+                         NULL : env->windows_environment_block,
+                       NULL,
+                       &startup_info,
+                       &win32_process->process_information);
+
+  tor_free(argv);
+  tor_free(joined_argv);
+  process_environment_free(env);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "CreateProcessA() failed: %s",
+      format_win32_error(GetLastError()));
+
+    /* Cleanup our handles. */
+    CloseHandle(stdout_pipe_read);
+    CloseHandle(stdout_pipe_write);
+    CloseHandle(stderr_pipe_read);
+    CloseHandle(stderr_pipe_write);
+    CloseHandle(stdin_pipe_read);
+    CloseHandle(stdin_pipe_write);
+
+    /* In the Unix backend, we do not get an error in the Tor process when a
+     * child process fails to spawn its target executable since we need to
+     * first do the fork() call in the Tor process and then the child process
+     * is responsible for doing the call to execve().
+     *
+     * This means that the user of the process_exec() API must check for
+     * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on
+     * Unix, but will happen for error cases on Windows where it does not
+     * happen on Unix. For example: when the target executable does not exist
+     * on the file system.
+     *
+     * To have somewhat feature compatibility between the Unix and the Windows
+     * backend, we here notify the process_t owner that the process have exited
+     * (even though it never managed to run) to ensure that the exit callback
+     * is executed.
+     */
+    process_notify_event_exit(process, 0);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* TODO: Should we close hProcess and hThread in
+   * process_handle->process_information? */
+  win32_process->stdout_handle.pipe = stdout_pipe_read;
+  win32_process->stderr_handle.pipe = stderr_pipe_read;
+  win32_process->stdin_handle.pipe = stdin_pipe_write;
+
+  /* Close our ends of the pipes that is now owned by the child process. */
+  CloseHandle(stdout_pipe_write);
+  CloseHandle(stderr_pipe_write);
+  CloseHandle(stdin_pipe_read);
+
+  /* Used by the callback functions from ReadFileEx() and WriteFileEx() such
+   * that we can figure out which process_t that was responsible for the event.
+   *
+   * Warning, here be dragons:
+   *
+   *   MSDN says that the hEvent member of the overlapped structure is unused
+   *   for ReadFileEx() and WriteFileEx, which allows us to store a pointer to
+   *   our process state there.
+   */
+  win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process;
+
+  /* Start our timer if it is not already running. */
+  if (! process_win32_timer_running())
+    process_win32_timer_start();
+
+  /* We use Windows Extended I/O functions, so our completion callbacks are
+   * called automatically for us when there is data to read. Because of this
+   * we start the read of standard out and error right away. */
+  process_notify_event_stdout(process);
+  process_notify_event_stderr(process);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_win32_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Terminate our process. */
+  BOOL ret;
+
+  ret = TerminateProcess(win32_process->process_information.hProcess, 0);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "TerminateProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  return true;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_win32_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  return (process_pid_t)win32_process->process_information.dwProcessId;
+}
+
+/** Schedule an async write of the data found in <b>buffer</b> for the given
+ * process.  This function runs an async write operation of the content of
+ * buffer, if we are not already waiting for a pending I/O request. Returns the
+ * number of bytes that Windows will hopefully write for us in the background.
+ * */
+int
+process_win32_write(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD error_code = 0;
+  const size_t buffer_size = buf_datalen(buffer);
+
+  /* Windows is still writing our buffer. */
+  if (win32_process->stdin_handle.busy)
+    return 0;
+
+  /* Nothing for us to do right now. */
+  if (buffer_size == 0)
+    return 0;
+
+  /* We have reached end of file already? */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return 0;
+
+  /* Figure out how much data we should read. */
+  const size_t write_size = MIN(buffer_size,
+                                sizeof(win32_process->stdin_handle.buffer));
+
+  /* Read data from the process_t buffer into our intermediate buffer. */
+  buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size);
+
+  /* Because of the slightly weird API for WriteFileEx() we must set this to 0
+   * before we call WriteFileEx() because WriteFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Schedule our write. */
+  ret = WriteFileEx(win32_process->stdin_handle.pipe,
+                    win32_process->stdin_handle.buffer,
+                    write_size,
+                    &win32_process->stdin_handle.overlapped,
+                    process_win32_stdin_write_done);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "WriteFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for WriteFileEx() we
+   * should check GetLastError() after a call to WriteFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* This cast should be safe since our buffer can maximum be BUFFER_SIZE
+   * large. */
+  return (int)write_size;
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard output of the child process
+ * and asks Windows to call process_win32_stdout_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stdout(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stdout_handle,
+                                        buffer,
+                                        process_win32_stdout_read_done);
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard error of the child process
+ * and asks Windows to call process_win32_stderr_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stderr(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stderr_handle,
+                                        buffer,
+                                        process_win32_stderr_read_done);
+}
+
+/** This function is responsible for moving the Tor process into what Microsoft
+ * calls an "alertable" state. Once the process is in an alertable state the
+ * Windows kernel will notify us when our background I/O requests have finished
+ * and the callbacks will be executed. */
+void
+process_win32_trigger_completion_callbacks(void)
+{
+  DWORD ret;
+
+  /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep
+   * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause
+   * the process to enter alertable state, where the Windows kernel will notify
+   * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which
+   * will cause our completion callbacks to be executed.
+   *
+   * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION
+   * if one or more I/O callbacks were executed. */
+  ret = SleepEx(0, TRUE);
+
+  /* Warn us if the function returned something we did not anticipate. */
+  if (ret != 0 && ret != WAIT_IO_COMPLETION) {
+    log_warn(LD_PROCESS, "SleepEx() returned %lu", ret);
+  }
+}
+
+/** Start the periodic timer which is reponsible for checking whether processes
+ * are still alive and to make sure that the Tor process is periodically being
+ * moved into an alertable state. */
+void
+process_win32_timer_start(void)
+{
+  /* Make sure we never start our timer if it's already running. */
+  if (BUG(process_win32_timer_running()))
+    return;
+
+  /* Wake up once a second. */
+  static const struct timeval interval = {1, 0};
+
+  log_info(LD_PROCESS, "Starting Windows Process I/O timer");
+  periodic_timer = periodic_timer_new(tor_libevent_get_base(),
+                                      &interval,
+                                      process_win32_timer_callback,
+                                      NULL);
+}
+
+/** Stops the periodic timer. */
+void
+process_win32_timer_stop(void)
+{
+  if (BUG(periodic_timer == NULL))
+    return;
+
+  log_info(LD_PROCESS, "Stopping Windows Process I/O timer");
+  periodic_timer_free(periodic_timer);
+}
+
+/** Returns true iff the periodic timer is running. */
+bool
+process_win32_timer_running(void)
+{
+  return periodic_timer != NULL;
+}
+
+/** This function is called whenever the periodic_timer ticks. The function is
+ * responsible for moving the Tor process into an alertable state once a second
+ * and checking for whether our child processes have terminated since the last
+ * tick. */
+STATIC void
+process_win32_timer_callback(periodic_timer_t *timer, void *data)
+{
+  tor_assert(timer == periodic_timer);
+  tor_assert(data == NULL);
+
+  /* Move the process into an alertable state. */
+  process_win32_trigger_completion_callbacks();
+
+  /* Check if our processes are still alive. */
+
+  /* Since the call to process_win32_timer_test_process() might call
+   * process_notify_event_exit() which again might call process_free() which
+   * updates the list of processes returned by process_get_all_processes() it
+   * is important here that we make sure to not touch the list of processes if
+   * the call to process_win32_timer_test_process() returns true. */
+  bool done;
+
+  do {
+    const smartlist_t *processes = process_get_all_processes();
+    done = true;
+
+    SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) {
+      /* If process_win32_timer_test_process() returns true, it means that
+       * smartlist_remove() might have been called on the list returned by
+       * process_get_all_processes(). We start the loop over again until we
+       * have a succesful run over the entire list where the list was not
+       * modified. */
+      if (process_win32_timer_test_process(process)) {
+        done = false;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(process);
+  } while (! done);
+}
+
+/** Test whether a given process is still alive. Notify the Process subsystem
+ * if our process have died. Returns true iff the given process have
+ * terminated. */
+STATIC bool
+process_win32_timer_test_process(process_t *process)
+{
+  tor_assert(process);
+
+  /* No need to look at processes that don't claim they are running. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD exit_code = 0;
+
+  /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error
+   * message until some time after the process have actually terminated. We
+   * make sure that our ReadFileEx() calls for the process have *all* returned
+   * and both standard out and error have been marked as EOF before we try to
+   * see if the process terminated.
+   *
+   * This ensures that we *never* call the exit callback of the `process_t`,
+   * which potentially ends up calling `process_free()` on our `process_t`,
+   * before all data have been received from the process.
+   *
+   * We do NOT have a check here for whether standard in reached EOF since
+   * standard in's WriteFileEx() function is only called on-demand when we have
+   * something to write and is thus usually not awaiting to finish any
+   * operations. If we WriteFileEx() to a file that has terminated we'll simply
+   * get an error from ReadFileEx() or its completion routine and move on with
+   * life.  */
+  if (! win32_process->stdout_handle.reached_eof)
+    return false;
+
+  if (! win32_process->stderr_handle.reached_eof)
+    return false;
+
+  /* We start by testing whether our process is still running. */
+  ret = GetExitCodeProcess(win32_process->process_information.hProcess,
+                           &exit_code);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Notify our process_t that our process have terminated. Since our
+   * exit_callback might decide to process_free() our process handle it is very
+   * important that we do not touch the process_t after the call to
+   * process_notify_event_exit(). */
+  if (exit_code != STILL_ACTIVE) {
+    process_notify_event_exit(process, exit_code);
+    return true;
+  }
+
+  return false;
+}
+
+/** Create a new overlapped named pipe. This function creates a new connected,
+ * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is
+ * succesful. Returns true on sucess, false on failure. */
+STATIC bool
+process_win32_create_pipe(HANDLE *read_pipe,
+                          HANDLE *write_pipe,
+                          SECURITY_ATTRIBUTES *attributes,
+                          process_win32_pipe_type_t pipe_type)
+{
+  tor_assert(read_pipe);
+  tor_assert(write_pipe);
+  tor_assert(attributes);
+
+  BOOL ret = FALSE;
+
+  /* Buffer size. */
+  const size_t size = 4096;
+
+  /* Our additional read/write modes that depends on which pipe type we are
+   * creating. */
+  DWORD read_mode = 0;
+  DWORD write_mode = 0;
+
+  /* Generate the unique pipe name. */
+  char pipe_name[MAX_PATH];
+  static DWORD process_id = 0;
+  static DWORD counter = 0;
+
+  if (process_id == 0)
+    process_id = GetCurrentProcessId();
+
+  tor_snprintf(pipe_name, sizeof(pipe_name),
+               "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu",
+               process_id, counter++);
+
+  /* Only one of our handles can be overlapped. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    read_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    write_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Setup our read and write handles. */
+  HANDLE read_handle;
+  HANDLE write_handle;
+
+  /* Create our named pipe. */
+  read_handle = CreateNamedPipeA(pipe_name,
+                                 (PIPE_ACCESS_INBOUND|read_mode),
+                                 (PIPE_TYPE_BYTE|PIPE_WAIT),
+                                 1,
+                                 size,
+                                 size,
+                                 1000,
+                                 attributes);
+
+  if (read_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Create our file in the pipe namespace. */
+  write_handle = CreateFileA(pipe_name,
+                             GENERIC_WRITE,
+                             0,
+                             attributes,
+                             OPEN_EXISTING,
+                             (FILE_ATTRIBUTE_NORMAL|write_mode),
+                             NULL);
+
+  if (write_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateFileA() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+
+    return false;
+  }
+
+  /* Set the inherit flag for our pipe. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "SetHandleInformation() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+    CloseHandle(write_handle);
+
+    return false;
+  }
+
+  /* Everything is good. */
+  *read_pipe = read_handle;
+  *write_pipe = write_handle;
+
+  return true;
+}
+
+/** Cleanup a given <b>handle</b>. */
+STATIC void
+process_win32_cleanup_handle(process_win32_handle_t *handle)
+{
+  tor_assert(handle);
+
+#if 0
+  BOOL ret;
+  DWORD error_code;
+
+  /* Cancel any pending I/O requests: This means that instead of getting
+   * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem
+   * like this is needed. */
+  ret = CancelIo(handle->pipe);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* There was no pending I/O requests for our handle. */
+    if (error_code != ERROR_NOT_FOUND) {
+      log_warn(LD_PROCESS, "CancelIo() failed: %s",
+               format_win32_error(error_code));
+    }
+  }
+#endif /* 0 */
+
+  /* Close our handle. */
+  if (handle->pipe != INVALID_HANDLE_VALUE) {
+    CloseHandle(handle->pipe);
+    handle->pipe = INVALID_HANDLE_VALUE;
+    handle->reached_eof = true;
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard output. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stdout_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stdout_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stdout(process);
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard error. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stderr_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stderr_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stderr(process);
+  }
+}
+
+/** This function is called when WriteFileEx() completes its background write
+ * to the child process's standard input. We notify the Process subsystem that
+ * it can write data to us again. */
+STATIC VOID WINAPI
+process_win32_stdin_write_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  (void)byte_count;
+
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  win32_process->stdin_handle.busy = false;
+
+  /* Check if we have been asked to write to the handle that have been marked
+   * as having reached EOF. */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return;
+
+  if (error_code == 0) {
+    /** Our data have been succesfully written. Clear our state and schedule
+     * the next write. */
+    win32_process->stdin_handle.data_available = 0;
+    memset(win32_process->stdin_handle.buffer, 0,
+           sizeof(win32_process->stdin_handle.buffer));
+
+    /* Schedule the next write. */
+    process_notify_event_stdin(process);
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our WriteFileEx() call was succesful, but we reached the end of our
+     * file.  We mark our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    win32_process->stdin_handle.reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from WriteFileEx(): %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+  }
+}
+
+/** This function reads data from the given <b>handle</b>'s internal buffer and
+ * moves it into the given <b>buffer</b>. Additionally, we start the next
+ * ReadFileEx() background operation with the given <b>callback</b> as
+ * completion callback. Returns the number of bytes written to the buffer. */
+STATIC int
+process_win32_read_from_handle(process_win32_handle_t *handle,
+                               buf_t *buffer,
+                               LPOVERLAPPED_COMPLETION_ROUTINE callback)
+{
+  tor_assert(handle);
+  tor_assert(buffer);
+  tor_assert(callback);
+
+  BOOL ret = FALSE;
+  int bytes_available = 0;
+  DWORD error_code = 0;
+
+  /* We already have a request to read data that isn't complete yet. */
+  if (BUG(handle->busy))
+    return 0;
+
+  /* Check if we have been asked to read from a handle that have already told
+   * us that we have reached the end of the file. */
+  if (BUG(handle->reached_eof))
+    return 0;
+
+  /* This cast should be safe since our buffer can be at maximum up to
+   * BUFFER_SIZE in size. */
+  bytes_available = (int)handle->data_available;
+
+  if (handle->data_available > 0) {
+    /* Read data from our intermediate buffer into the process_t buffer. */
+    buf_add(buffer, handle->buffer, handle->data_available);
+
+    /* Reset our read state. */
+    handle->data_available = 0;
+    memset(handle->buffer, 0, sizeof(handle->buffer));
+  }
+
+  /* Because of the slightly weird API for ReadFileEx() we must set this to 0
+   * before we call ReadFileEx() because ReadFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Ask the Windows kernel to read data from our pipe into our buffer and call
+   * the callback function when it is done. */
+  ret = ReadFileEx(handle->pipe,
+                   handle->buffer,
+                   sizeof(handle->buffer),
+                   &handle->overlapped,
+                   callback);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "ReadFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    handle->reached_eof = true;
+    return bytes_available;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for ReadFileEx() we
+   * should check GetLastError() after a call to ReadFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    handle->reached_eof = true;
+    return bytes_available;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* We mark our handle as having a pending I/O request. */
+  handle->busy = true;
+
+  return bytes_available;
+}
+
+/** This function checks the callback values from ReadFileEx() in
+ * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true
+ * iff our caller should request more data from ReadFileEx(). */
+STATIC bool
+process_win32_handle_read_completion(process_win32_handle_t *handle,
+                                     DWORD error_code,
+                                     DWORD byte_count)
+{
+  tor_assert(handle);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  handle->busy = false;
+
+  if (error_code == 0) {
+    /* Our ReadFileEx() call was succesful and there is data for us. */
+
+    /* This cast should be safe since byte_count should never be larger than
+     * BUFFER_SIZE. */
+    tor_assert(byte_count <= BUFFER_SIZE);
+    handle->data_available = (size_t)byte_count;
+
+    /* Tell our caller to schedule the next read. */
+    return true;
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our ReadFileEx() finished, but we reached the end of our file.  We mark
+     * our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    handle->reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from ReadFileEx(): %s",
+             format_win32_error(error_code));
+
+    handle->reached_eof = true;
+  }
+
+  /* Our caller should NOT schedule the next read. */
+  return false;
+}
+
+/** Format a single argument for being put on a Windows command line.
+ * Returns a newly allocated string */
+STATIC char *
+format_win_cmdline_argument(const char *arg)
+{
+  char *formatted_arg;
+  char need_quotes;
+  const char *c;
+  int i;
+  int bs_counter = 0;
+  /* Backslash we can point to when one is inserted into the string */
+  const char backslash = '\\';
+
+  /* Smartlist of *char */
+  smartlist_t *arg_chars;
+  arg_chars = smartlist_new();
+
+  /* Quote string if it contains whitespace or is empty */
+  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
+
+  /* Build up smartlist of *chars */
+  for (c=arg; *c != '\0'; c++) {
+    if ('"' == *c) {
+      /* Double up backslashes preceding a quote */
+      for (i=0; i<(bs_counter*2); i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      /* Escape the quote */
+      smartlist_add(arg_chars, (void*)&backslash);
+      smartlist_add(arg_chars, (void*)c);
+    } else if ('\\' == *c) {
+      /* Count backslashes until we know whether to double up */
+      bs_counter++;
+    } else {
+      /* Don't double up slashes preceding a non-quote */
+      for (i=0; i<bs_counter; i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      smartlist_add(arg_chars, (void*)c);
+    }
+  }
+  /* Don't double up trailing backslashes */
+  for (i=0; i<bs_counter; i++)
+    smartlist_add(arg_chars, (void*)&backslash);
+
+  /* Allocate space for argument, quotes (if needed), and terminator */
+  const size_t formatted_arg_len = smartlist_len(arg_chars) +
+    (need_quotes ? 2 : 0) + 1;
+  formatted_arg = tor_malloc_zero(formatted_arg_len);
+
+  /* Add leading quote */
+  i=0;
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+
+  /* Add characters */
+  SMARTLIST_FOREACH(arg_chars, char*, ch,
+  {
+    formatted_arg[i++] = *ch;
+  });
+
+  /* Add trailing quote */
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+  formatted_arg[i] = '\0';
+
+  smartlist_free(arg_chars);
+  return formatted_arg;
+}
+
+/** Format a command line for use on Windows, which takes the command as a
+ * string rather than string array. Follows the rules from "Parsing C++
+ * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
+ * Python subprocess module. Returns a newly allocated string */
+STATIC char *
+tor_join_win_cmdline(const char *argv[])
+{
+  smartlist_t *argv_list;
+  char *joined_argv;
+  int i;
+
+  /* Format each argument and put the result in a smartlist */
+  argv_list = smartlist_new();
+  for (i=0; argv[i] != NULL; i++) {
+    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
+  }
+
+  /* Join the arguments with whitespace */
+  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
+
+  /* Free the newly allocated arguments, and the smartlist */
+  SMARTLIST_FOREACH(argv_list, char *, arg,
+  {
+    tor_free(arg);
+  });
+  smartlist_free(argv_list);
+
+  return joined_argv;
+}
+
+#endif /* defined(_WIN32) */
diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h
new file mode 100644
index 0000000000..0f264c8710
--- /dev/null
+++ b/src/lib/process/process_win32.h
@@ -0,0 +1,97 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.h
+ * \brief Header for process_win32.c
+ **/
+
+#ifndef TOR_PROCESS_WIN32_H
+#define TOR_PROCESS_WIN32_H
+
+#ifdef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include <windows.h>
+
+struct process_t;
+
+struct process_win32_t;
+typedef struct process_win32_t process_win32_t;
+
+process_win32_t *process_win32_new(void);
+void process_win32_free_(process_win32_t *win32_process);
+#define process_win32_free(s) \
+  FREE_AND_NULL(process_win32_t, process_win32_free_, (s))
+
+void process_win32_init(void);
+void process_win32_deinit(void);
+
+process_status_t process_win32_exec(struct process_t *process);
+bool process_win32_terminate(struct process_t *process);
+
+process_pid_t process_win32_get_pid(struct process_t *process);
+
+int process_win32_write(struct process_t *process, buf_t *buffer);
+int process_win32_read_stdout(struct process_t *process, buf_t *buffer);
+int process_win32_read_stderr(struct process_t *process, buf_t *buffer);
+
+void process_win32_trigger_completion_callbacks(void);
+
+/* Timer handling. */
+void process_win32_timer_start(void);
+void process_win32_timer_stop(void);
+bool process_win32_timer_running(void);
+
+#ifdef PROCESS_WIN32_PRIVATE
+STATIC void process_win32_timer_callback(periodic_timer_t *, void *);
+STATIC bool process_win32_timer_test_process(process_t *);
+
+/* I/O pipe handling. */
+struct process_win32_handle_t;
+typedef struct process_win32_handle_t process_win32_handle_t;
+
+typedef enum process_win32_pipe_type_t {
+  /** This pipe is used for reading. */
+  PROCESS_WIN32_PIPE_TYPE_READER,
+
+  /** This pipe is used for writing. */
+  PROCESS_WIN32_PIPE_TYPE_WRITER
+} process_win32_pipe_type_t;
+
+STATIC bool process_win32_create_pipe(HANDLE *,
+                                      HANDLE *,
+                                      SECURITY_ATTRIBUTES *,
+                                      process_win32_pipe_type_t);
+
+STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle);
+
+STATIC VOID WINAPI process_win32_stdout_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stderr_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stdin_write_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+
+STATIC int process_win32_read_from_handle(process_win32_handle_t *,
+                                          buf_t *,
+                                          LPOVERLAPPED_COMPLETION_ROUTINE);
+STATIC bool process_win32_handle_read_completion(process_win32_handle_t *,
+                                                 DWORD,
+                                                 DWORD);
+
+STATIC char *format_win_cmdline_argument(const char *arg);
+STATIC char *tor_join_win_cmdline(const char *argv[]);
+#endif /* defined(PROCESS_WIN32_PRIVATE) */
+
+#endif /* defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_WIN32_H) */
diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c
index 534b39d101..cd2a1c57b5 100644
--- a/src/lib/process/restrict.c
+++ b/src/lib/process/restrict.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -85,7 +85,7 @@ tor_set_max_memlock(void)
 {
   /* Future consideration for Windows is probably SetProcessWorkingSetSize
    * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK
-   * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx
+   * https://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx
    */
 
   struct rlimit limit;
@@ -128,7 +128,7 @@ tor_mlockall(void)
    * Future consideration for Windows may be VirtualLock
    * VirtualLock appears to implement mlock() but not mlockall()
    *
-   * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx
+   * https://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx
    */
 
 #ifdef HAVE_UNIX_MLOCKALL
@@ -152,7 +152,7 @@ tor_mlockall(void)
                            "pages: %s", strerror(errno));
     return -1;
   }
-#else /* !(defined(HAVE_UNIX_MLOCKALL)) */
+#else /* !defined(HAVE_UNIX_MLOCKALL) */
   log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?");
   return -1;
 #endif /* defined(HAVE_UNIX_MLOCKALL) */
@@ -190,7 +190,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
 
   /* Define some maximum connections values for systems where we cannot
    * automatically determine a limit. Re Cygwin, see
-   * http://archives.seul.org/or/talk/Aug-2006/msg00210.html
+   * https://archives.seul.org/or/talk/Aug-2006/msg00210.html
    * For an iPhone, 9999 should work. For Windows and all other unknown
    * systems we use 15000 as the default. */
 #ifndef HAVE_GETRLIMIT
@@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
     return -1;
   }
   limit = MAX_CONNECTIONS;
-#else /* !(!defined(HAVE_GETRLIMIT)) */
+#else /* defined(HAVE_GETRLIMIT) */
   struct rlimit rlim;
 
   if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
diff --git a/src/lib/process/restrict.h b/src/lib/process/restrict.h
index 8491c99044..6173d5a438 100644
--- a/src/lib/process/restrict.h
+++ b/src/lib/process/restrict.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c
index 6e8258f279..3cfd520a4f 100644
--- a/src/lib/process/setuid.c
+++ b/src/lib/process/setuid.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -64,7 +64,7 @@ log_credential_status(void)
 
   /* log UIDs */
 #ifdef HAVE_GETRESUID
-  if (getresuid(&ruid, &euid, &suid) != 0 ) {
+  if (getresuid(&ruid, &euid, &suid) != 0) {
     log_warn(LD_GENERAL, "Error getting changed UIDs: %s", strerror(errno));
     return -1;
   } else {
@@ -72,7 +72,7 @@ log_credential_status(void)
            "UID is %u (real), %u (effective), %u (saved)",
            (unsigned)ruid, (unsigned)euid, (unsigned)suid);
   }
-#else /* !(defined(HAVE_GETRESUID)) */
+#else /* !defined(HAVE_GETRESUID) */
   /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */
   ruid = getuid();
   euid = geteuid();
@@ -85,7 +85,7 @@ log_credential_status(void)
 
   /* log GIDs */
 #ifdef HAVE_GETRESGID
-  if (getresgid(&rgid, &egid, &sgid) != 0 ) {
+  if (getresgid(&rgid, &egid, &sgid) != 0) {
     log_warn(LD_GENERAL, "Error getting changed GIDs: %s", strerror(errno));
     return -1;
   } else {
@@ -93,7 +93,7 @@ log_credential_status(void)
            "GID is %u (real), %u (effective), %u (saved)",
            (unsigned)rgid, (unsigned)egid, (unsigned)sgid);
   }
-#else /* !(defined(HAVE_GETRESGID)) */
+#else /* !defined(HAVE_GETRESGID) */
   /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */
   rgid = getgid();
   egid = getegid();
@@ -154,7 +154,7 @@ have_capability_support(void)
     return 0;
   cap_free(caps);
   return 1;
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   return 0;
 #endif /* defined(HAVE_LINUX_CAPABILITIES) */
 }
@@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags)
     if (drop_capabilities(1))
       return -1;
   }
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   (void) keep_bindlow;
   if (warn_if_no_caps) {
     log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support "
@@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags)
 #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */
   return 0;
 
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void)user;
   (void)flags;
 
diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h
index 7d03e1f025..fec35a1216 100644
--- a/src/lib/process/setuid.h
+++ b/src/lib/process/setuid.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ int have_capability_support(void);
 #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1)
 int switch_id(const char *user, unsigned flags);
 
-#endif
+#endif /* !defined(TOR_SETUID_H) */
diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c
deleted file mode 100644
index f4429d7f76..0000000000
--- a/src/lib/process/subprocess.c
+++ /dev/null
@@ -1,1236 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.c
- * \brief Launch and monitor other processes.
- **/
-
-#define SUBPROCESS_PRIVATE
-#include "lib/process/subprocess.h"
-
-#include "lib/container/smartlist.h"
-#include "lib/err/torerr.h"
-#include "lib/log/log.h"
-#include "lib/log/util_bug.h"
-#include "lib/log/win32err.h"
-#include "lib/malloc/malloc.h"
-#include "lib/process/env.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/compat_ctype.h"
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_PRCTL_H
-#include <sys/prctl.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-#include <errno.h>
-#include <string.h>
-
-/** Format a single argument for being put on a Windows command line.
- * Returns a newly allocated string */
-static char *
-format_win_cmdline_argument(const char *arg)
-{
-  char *formatted_arg;
-  char need_quotes;
-  const char *c;
-  int i;
-  int bs_counter = 0;
-  /* Backslash we can point to when one is inserted into the string */
-  const char backslash = '\\';
-
-  /* Smartlist of *char */
-  smartlist_t *arg_chars;
-  arg_chars = smartlist_new();
-
-  /* Quote string if it contains whitespace or is empty */
-  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
-
-  /* Build up smartlist of *chars */
-  for (c=arg; *c != '\0'; c++) {
-    if ('"' == *c) {
-      /* Double up backslashes preceding a quote */
-      for (i=0; i<(bs_counter*2); i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      /* Escape the quote */
-      smartlist_add(arg_chars, (void*)&backslash);
-      smartlist_add(arg_chars, (void*)c);
-    } else if ('\\' == *c) {
-      /* Count backslashes until we know whether to double up */
-      bs_counter++;
-    } else {
-      /* Don't double up slashes preceding a non-quote */
-      for (i=0; i<bs_counter; i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      smartlist_add(arg_chars, (void*)c);
-    }
-  }
-  /* Don't double up trailing backslashes */
-  for (i=0; i<bs_counter; i++)
-    smartlist_add(arg_chars, (void*)&backslash);
-
-  /* Allocate space for argument, quotes (if needed), and terminator */
-  const size_t formatted_arg_len = smartlist_len(arg_chars) +
-    (need_quotes ? 2 : 0) + 1;
-  formatted_arg = tor_malloc_zero(formatted_arg_len);
-
-  /* Add leading quote */
-  i=0;
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-
-  /* Add characters */
-  SMARTLIST_FOREACH(arg_chars, char*, ch,
-  {
-    formatted_arg[i++] = *ch;
-  });
-
-  /* Add trailing quote */
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-  formatted_arg[i] = '\0';
-
-  smartlist_free(arg_chars);
-  return formatted_arg;
-}
-
-/** Format a command line for use on Windows, which takes the command as a
- * string rather than string array. Follows the rules from "Parsing C++
- * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
- * Python subprocess module. Returns a newly allocated string */
-char *
-tor_join_win_cmdline(const char *argv[])
-{
-  smartlist_t *argv_list;
-  char *joined_argv;
-  int i;
-
-  /* Format each argument and put the result in a smartlist */
-  argv_list = smartlist_new();
-  for (i=0; argv[i] != NULL; i++) {
-    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
-  }
-
-  /* Join the arguments with whitespace */
-  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
-
-  /* Free the newly allocated arguments, and the smartlist */
-  SMARTLIST_FOREACH(argv_list, char *, arg,
-  {
-    tor_free(arg);
-  });
-  smartlist_free(argv_list);
-
-  return joined_argv;
-}
-
-#ifndef _WIN32
-/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in
- * <b>hex_errno</b>.  Called between fork and _exit, so must be signal-handler
- * safe.
- *
- * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available.
- *
- * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded
- * with spaces. CHILD_STATE indicates where
- * in the process of starting the child process did the failure occur (see
- * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of
- * errno when the failure occurred.
- *
- * On success return the number of characters added to hex_errno, not counting
- * the terminating NUL; return -1 on error.
- */
-STATIC int
-format_helper_exit_status(unsigned char child_state, int saved_errno,
-                          char *hex_errno)
-{
-  unsigned int unsigned_errno;
-  int written, left;
-  char *cur;
-  size_t i;
-  int res = -1;
-
-  /* Fill hex_errno with spaces, and a trailing newline (memset may
-     not be signal handler safe, so we can't use it) */
-  for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++)
-    hex_errno[i] = ' ';
-  hex_errno[HEX_ERRNO_SIZE - 1] = '\n';
-
-  /* Convert errno to be unsigned for hex conversion */
-  if (saved_errno < 0) {
-    // Avoid overflow on the cast to unsigned int when result is INT_MIN
-    // by adding 1 to the signed int negative value,
-    // then, after it has been negated and cast to unsigned,
-    // adding the original 1 back (the double-addition is intentional).
-    // Otherwise, the cast to signed could cause a temporary int
-    // to equal INT_MAX + 1, which is undefined.
-    unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1;
-  } else {
-    unsigned_errno = (unsigned int) saved_errno;
-  }
-
-  /*
-   * Count how many chars of space we have left, and keep a pointer into the
-   * current point in the buffer.
-   */
-  left = HEX_ERRNO_SIZE+1;
-  cur = hex_errno;
-
-  /* Emit child_state */
-  written = format_hex_number_sigsafe(child_state, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-  if (left <= 0)
-    goto err;
-
-  /* Now the '/' */
-  *cur = '/';
-
-  /* Adjust left and cur */
-  ++cur;
-  --left;
-  if (left <= 0)
-    goto err;
-
-  /* Need minus? */
-  if (saved_errno < 0) {
-    *cur = '-';
-    ++cur;
-    --left;
-    if (left <= 0)
-      goto err;
-  }
-
-  /* Emit unsigned_errno */
-  written = format_hex_number_sigsafe(unsigned_errno, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-
-  /* Check that we have enough space left for a newline and a NUL */
-  if (left <= 1)
-    goto err;
-
-  /* Emit the newline and NUL */
-  *cur++ = '\n';
-  *cur++ = '\0';
-
-  res = (int)(cur - hex_errno - 1);
-
-  goto done;
-
- err:
-  /*
-   * In error exit, just write a '\0' in the first char so whatever called
-   * this at least won't fall off the end.
-   */
-  *hex_errno = '\0';
-
- done:
-  return res;
-}
-#endif /* !defined(_WIN32) */
-
-/* Maximum number of file descriptors, if we cannot get it via sysconf() */
-#define DEFAULT_MAX_FD 256
-
-/** Terminate the process of <b>process_handle</b>, if that process has not
- * already exited.
- *
- * Return 0 if we succeeded in terminating the process (or if the process
- * already exited), and -1 if we tried to kill the process but failed.
- *
- * Based on code originally borrowed from Python's os.kill. */
-int
-tor_terminate_process(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) {
-    HANDLE handle = process_handle->pid.hProcess;
-
-    if (!TerminateProcess(handle, 0))
-      return -1;
-    else
-      return 0;
-  }
-#else /* !(defined(_WIN32)) */
-  if (process_handle->waitpid_cb) {
-    /* We haven't got a waitpid yet, so we can just kill off the process. */
-    return kill(process_handle->pid, SIGTERM);
-  }
-#endif /* defined(_WIN32) */
-
-  return 0; /* We didn't need to kill the process, so report success */
-}
-
-/** Return the Process ID of <b>process_handle</b>. */
-int
-tor_process_get_pid(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  return (int) process_handle->pid.dwProcessId;
-#else
-  return (int) process_handle->pid;
-#endif
-}
-
-#ifdef _WIN32
-HANDLE
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#else /* !(defined(_WIN32)) */
-/* DOCDOC tor_process_get_stdout_pipe */
-int
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#endif /* defined(_WIN32) */
-
-/* DOCDOC process_handle_new */
-static process_handle_t *
-process_handle_new(void)
-{
-  process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t));
-
-#ifdef _WIN32
-  out->stdin_pipe = INVALID_HANDLE_VALUE;
-  out->stdout_pipe = INVALID_HANDLE_VALUE;
-  out->stderr_pipe = INVALID_HANDLE_VALUE;
-#else
-  out->stdin_pipe = -1;
-  out->stdout_pipe = -1;
-  out->stderr_pipe = -1;
-#endif /* defined(_WIN32) */
-
-  return out;
-}
-
-#ifndef _WIN32
-/** Invoked when a process that we've launched via tor_spawn_background() has
- * been found to have terminated.
- */
-static void
-process_handle_waitpid_cb(int status, void *arg)
-{
-  process_handle_t *process_handle = arg;
-
-  process_handle->waitpid_exit_status = status;
-  clear_waitpid_callback(process_handle->waitpid_cb);
-  if (process_handle->status == PROCESS_STATUS_RUNNING)
-    process_handle->status = PROCESS_STATUS_NOTRUNNING;
-  process_handle->waitpid_cb = 0;
-}
-#endif /* !defined(_WIN32) */
-
-/**
- * @name child-process states
- *
- * Each of these values represents a possible state that a child process can
- * be in.  They're used to determine what to say when telling the parent how
- * far along we were before failure.
- *
- * @{
- */
-#define CHILD_STATE_INIT 0
-#define CHILD_STATE_PIPE 1
-#define CHILD_STATE_MAXFD 2
-#define CHILD_STATE_FORK 3
-#define CHILD_STATE_DUPOUT 4
-#define CHILD_STATE_DUPERR 5
-#define CHILD_STATE_DUPIN 6
-#define CHILD_STATE_CLOSEFD 7
-#define CHILD_STATE_EXEC 8
-#define CHILD_STATE_FAILEXEC 9
-/** @} */
-/**
- * Boolean.  If true, then Tor may call execve or CreateProcess via
- * tor_spawn_background.
- **/
-static int may_spawn_background_process = 1;
-/**
- * Turn off may_spawn_background_process, so that all future calls to
- * tor_spawn_background are guaranteed to fail.
- **/
-void
-tor_disable_spawning_background_processes(void)
-{
-  may_spawn_background_process = 0;
-}
-/** Start a program in the background. If <b>filename</b> contains a '/', then
- * it will be treated as an absolute or relative path.  Otherwise, on
- * non-Windows systems, the system path will be searched for <b>filename</b>.
- * On Windows, only the current directory will be searched. Here, to search the
- * system path (as well as the application directory, current working
- * directory, and system directories), set filename to NULL.
- *
- * The strings in <b>argv</b> will be passed as the command line arguments of
- * the child program (following convention, argv[0] should normally be the
- * filename of the executable, and this must be the case if <b>filename</b> is
- * NULL). The last element of argv must be NULL. A handle to the child process
- * will be returned in process_handle (which must be non-NULL). Read
- * process_handle.status to find out if the process was successfully launched.
- * For convenience, process_handle.status is returned by this function.
- *
- * Some parts of this code are based on the POSIX subprocess module from
- * Python, and example code from
- * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx.
- */
-int
-tor_spawn_background(const char *const filename, const char **argv,
-                     process_environment_t *env,
-                     process_handle_t **process_handle_out)
-{
-  if (BUG(may_spawn_background_process == 0)) {
-    /* We should never reach this point if we're forbidden to spawn
-     * processes. Instead we should have caught the attempt earlier. */
-    return PROCESS_STATUS_ERROR;
-  }
-
-#ifdef _WIN32
-  HANDLE stdout_pipe_read = NULL;
-  HANDLE stdout_pipe_write = NULL;
-  HANDLE stderr_pipe_read = NULL;
-  HANDLE stderr_pipe_write = NULL;
-  HANDLE stdin_pipe_read = NULL;
-  HANDLE stdin_pipe_write = NULL;
-  process_handle_t *process_handle;
-  int status;
-
-  STARTUPINFOA siStartInfo;
-  BOOL retval = FALSE;
-
-  SECURITY_ATTRIBUTES saAttr;
-  char *joined_argv;
-
-  saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
-  saAttr.bInheritHandle = TRUE;
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-  saAttr.lpSecurityDescriptor = NULL;
-
-  /* Assume failure to start process */
-  status = PROCESS_STATUS_ERROR;
-
-  /* Set up pipe for stdout */
-  if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdout communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdout communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stderr */
-  if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stderr communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stderr communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stdin */
-  if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdin communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdin communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Create the child process */
-
-  /* Windows expects argv to be a whitespace delimited string, so join argv up
-   */
-  joined_argv = tor_join_win_cmdline(argv);
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-
-  ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION));
-  ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
-  siStartInfo.cb = sizeof(STARTUPINFO);
-  siStartInfo.hStdError = stderr_pipe_write;
-  siStartInfo.hStdOutput = stdout_pipe_write;
-  siStartInfo.hStdInput = stdin_pipe_read;
-  siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
-
-  /* Create the child process */
-
-  retval = CreateProcessA(filename,      // module name
-                 joined_argv,   // command line
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-                 NULL,          // process security attributes
-                 NULL,          // primary thread security attributes
-                 TRUE,          // handles are inherited
-  /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess()
-   * work?) */
-                 CREATE_NO_WINDOW,             // creation flags
-                 (env==NULL) ? NULL : env->windows_environment_block,
-                 NULL,          // use parent's current directory
-                 &siStartInfo,  // STARTUPINFO pointer
-                 &(process_handle->pid));  // receives PROCESS_INFORMATION
-
-  tor_free(joined_argv);
-
-  if (!retval) {
-    log_warn(LD_GENERAL,
-      "Failed to create child process %s: %s", filename?filename:argv[0],
-      format_win32_error(GetLastError()));
-    tor_free(process_handle);
-  } else  {
-    /* TODO: Close hProcess and hThread in process_handle->pid? */
-    process_handle->stdout_pipe = stdout_pipe_read;
-    process_handle->stderr_pipe = stderr_pipe_read;
-    process_handle->stdin_pipe = stdin_pipe_write;
-    status = process_handle->status = PROCESS_STATUS_RUNNING;
-  }
-
-  /* TODO: Close pipes on exit */
-  *process_handle_out = process_handle;
-  return status;
-#else /* !(defined(_WIN32)) */
-  pid_t pid;
-  int stdout_pipe[2];
-  int stderr_pipe[2];
-  int stdin_pipe[2];
-  int fd, retval;
-  process_handle_t *process_handle;
-  int status;
-
-  const char *error_message = SPAWN_ERROR_MESSAGE;
-  size_t error_message_length;
-
-  /* Represents where in the process of spawning the program is;
-     this is used for printing out the error message */
-  unsigned char child_state = CHILD_STATE_INIT;
-
-  char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */
-
-  static int max_fd = -1;
-
-  status = PROCESS_STATUS_ERROR;
-
-  /* We do the strlen here because strlen() is not signal handler safe,
-     and we are not allowed to use unsafe functions between fork and exec */
-  error_message_length = strlen(error_message);
-
-  // child_state = CHILD_STATE_PIPE;
-
-  /* Set up pipe for redirecting stdout, stderr, and stdin of child */
-  retval = pipe(stdout_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdout communication with child process: %s",
-       strerror(errno));
-    return status;
-  }
-
-  retval = pipe(stderr_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stderr communication with child process: %s",
-      strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-
-    return status;
-  }
-
-  retval = pipe(stdin_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdin communication with child process: %s",
-       strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-
-    return status;
-  }
-
-  // child_state = CHILD_STATE_MAXFD;
-
-#ifdef _SC_OPEN_MAX
-  if (-1 == max_fd) {
-    max_fd = (int) sysconf(_SC_OPEN_MAX);
-    if (max_fd == -1) {
-      max_fd = DEFAULT_MAX_FD;
-      log_warn(LD_GENERAL,
-               "Cannot find maximum file descriptor, assuming %d", max_fd);
-    }
-  }
-#else /* !(defined(_SC_OPEN_MAX)) */
-  max_fd = DEFAULT_MAX_FD;
-#endif /* defined(_SC_OPEN_MAX) */
-
-  // child_state = CHILD_STATE_FORK;
-
-  pid = fork();
-  if (0 == pid) {
-    /* In child */
-
-#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
-    /* Attempt to have the kernel issue a SIGTERM if the parent
-     * goes away. Certain attributes of the binary being execve()ed
-     * will clear this during the execve() call, but it's better
-     * than nothing.
-     */
-    prctl(PR_SET_PDEATHSIG, SIGTERM);
-#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
-
-    child_state = CHILD_STATE_DUPOUT;
-
-    /* Link child stdout to the write end of the pipe */
-    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPERR;
-
-    /* Link child stderr to the write end of the pipe */
-    retval = dup2(stderr_pipe[1], STDERR_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPIN;
-
-    /* Link child stdin to the read end of the pipe */
-    retval = dup2(stdin_pipe[0], STDIN_FILENO);
-    if (-1 == retval)
-      goto error;
-
-    // child_state = CHILD_STATE_CLOSEFD;
-
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-
-    /* Close all other fds, including the read end of the pipe */
-    /* XXX: We should now be doing enough FD_CLOEXEC setting to make
-     * this needless. */
-    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) {
-      close(fd);
-    }
-
-    // child_state = CHILD_STATE_EXEC;
-
-    /* Call the requested program. We need the cast because
-       execvp doesn't define argv as const, even though it
-       does not modify the arguments */
-    if (env)
-      execve(filename, (char *const *) argv, env->unixoid_environment_block);
-    else {
-      static char *new_env[] = { NULL };
-      execve(filename, (char *const *) argv, new_env);
-    }
-
-    /* If we got here, the exec or open(/dev/null) failed */
-
-    child_state = CHILD_STATE_FAILEXEC;
-
-  error:
-    {
-      /* XXX: are we leaking fds from the pipe? */
-      int n, err=0;
-      ssize_t nbytes;
-
-      n = format_helper_exit_status(child_state, errno, hex_errno);
-
-      if (n >= 0) {
-        /* Write the error message. GCC requires that we check the return
-           value, but there is nothing we can do if it fails */
-        /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */
-        nbytes = write(STDOUT_FILENO, error_message, error_message_length);
-        err = (nbytes < 0);
-        nbytes = write(STDOUT_FILENO, hex_errno, n);
-        err += (nbytes < 0);
-      }
-
-      _exit(err?254:255); // exit ok: in child.
-    }
-
-    /* Never reached, but avoids compiler warning */
-    return status; // LCOV_EXCL_LINE
-  }
-
-  /* In parent */
-
-  if (-1 == pid) {
-    log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno));
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    return status;
-  }
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-  process_handle->pid = pid;
-
-  /* TODO: If the child process forked but failed to exec, waitpid it */
-
-  /* Return read end of the pipes to caller, and close write end */
-  process_handle->stdout_pipe = stdout_pipe[0];
-  retval = close(stdout_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stdout pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  process_handle->waitpid_cb = set_waitpid_callback(pid,
-                                                    process_handle_waitpid_cb,
-                                                    process_handle);
-
-  process_handle->stderr_pipe = stderr_pipe[0];
-  retval = close(stderr_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stderr pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  /* Return write end of the stdin pipe to caller, and close the read end */
-  process_handle->stdin_pipe = stdin_pipe[1];
-  retval = close(stdin_pipe[0]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close read end of stdin pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  status = process_handle->status = PROCESS_STATUS_RUNNING;
-  /* Set stdin/stdout/stderr pipes to be non-blocking */
-  if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) {
-    log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
-             "nonblocking in parent process: %s", strerror(errno));
-  }
-
-  *process_handle_out = process_handle;
-  return status;
-#endif /* defined(_WIN32) */
-}
-
-/** Destroy all resources allocated by the process handle in
- *  <b>process_handle</b>.
- *  If <b>also_terminate_process</b> is true, also terminate the
- *  process of the process handle. */
-MOCK_IMPL(void,
-tor_process_handle_destroy,(process_handle_t *process_handle,
-                            int also_terminate_process))
-{
-  if (!process_handle)
-    return;
-
-  if (also_terminate_process) {
-    if (tor_terminate_process(process_handle) < 0) {
-      const char *errstr =
-#ifdef _WIN32
-        format_win32_error(GetLastError());
-#else
-        strerror(errno);
-#endif
-      log_notice(LD_GENERAL, "Failed to terminate process with "
-                 "PID '%d' ('%s').", tor_process_get_pid(process_handle),
-                 errstr);
-    } else {
-      log_info(LD_GENERAL, "Terminated process with PID '%d'.",
-               tor_process_get_pid(process_handle));
-    }
-  }
-
-  process_handle->status = PROCESS_STATUS_NOTRUNNING;
-
-#ifdef _WIN32
-  if (process_handle->stdout_pipe)
-    CloseHandle(process_handle->stdout_pipe);
-
-  if (process_handle->stderr_pipe)
-    CloseHandle(process_handle->stderr_pipe);
-
-  if (process_handle->stdin_pipe)
-    CloseHandle(process_handle->stdin_pipe);
-#else /* !(defined(_WIN32)) */
-  close(process_handle->stdout_pipe);
-  close(process_handle->stderr_pipe);
-  close(process_handle->stdin_pipe);
-
-  clear_waitpid_callback(process_handle->waitpid_cb);
-#endif /* defined(_WIN32) */
-
-  memset(process_handle, 0x0f, sizeof(process_handle_t));
-  tor_free(process_handle);
-}
-
-/** Get the exit code of a process specified by <b>process_handle</b> and store
- * it in <b>exit_code</b>, if set to a non-NULL value.  If <b>block</b> is set
- * to true, the call will block until the process has exited.  Otherwise if
- * the process is still running, the function will return
- * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns
- * PROCESS_EXIT_EXITED if the process did exit. If there is a failure,
- * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if
- * non-NULL) will be undefined. N.B. Under *nix operating systems, this will
- * probably not work in Tor, because waitpid() is called in main.c to reap any
- * terminated child processes.*/
-int
-tor_get_exit_code(process_handle_t *process_handle,
-                  int block, int *exit_code)
-{
-#ifdef _WIN32
-  DWORD retval;
-  BOOL success;
-
-  if (block) {
-    /* Wait for the process to exit */
-    retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE);
-    if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-              (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  } else {
-    retval = WaitForSingleObject(process_handle->pid.hProcess, 0);
-    if (WAIT_TIMEOUT == retval) {
-      /* Process has not exited */
-      return PROCESS_EXIT_RUNNING;
-    } else if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-               (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-
-  if (exit_code != NULL) {
-    success = GetExitCodeProcess(process_handle->pid.hProcess,
-                                 (PDWORD)exit_code);
-    if (!success) {
-      log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s",
-               format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-#else /* !(defined(_WIN32)) */
-  int stat_loc;
-  int retval;
-
-  if (process_handle->waitpid_cb) {
-    /* We haven't processed a SIGCHLD yet. */
-    retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG);
-    if (retval == process_handle->pid) {
-      clear_waitpid_callback(process_handle->waitpid_cb);
-      process_handle->waitpid_cb = NULL;
-      process_handle->waitpid_exit_status = stat_loc;
-    }
-  } else {
-    /* We already got a SIGCHLD for this process, and handled it. */
-    retval = process_handle->pid;
-    stat_loc = process_handle->waitpid_exit_status;
-  }
-
-  if (!block && 0 == retval) {
-    /* Process has not exited */
-    return PROCESS_EXIT_RUNNING;
-  } else if (retval != process_handle->pid) {
-    log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s",
-             (int)process_handle->pid, strerror(errno));
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (!WIFEXITED(stat_loc)) {
-    log_warn(LD_GENERAL, "Process %d did not exit normally",
-             (int)process_handle->pid);
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (exit_code != NULL)
-    *exit_code = WEXITSTATUS(stat_loc);
-#endif /* defined(_WIN32) */
-
-  return PROCESS_EXIT_EXITED;
-}
-
-#ifdef _WIN32
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>hProcess</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle
- * to the process owning the <b>h</b>. In this case, the function will exit
- * only once the process has exited, or <b>count</b> bytes are read. Returns
- * the number of bytes read, or -1 on error. */
-ssize_t
-tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                    const process_handle_t *process)
-{
-  size_t numread = 0;
-  BOOL retval;
-  DWORD byte_count;
-  BOOL process_exited = FALSE;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    /* Check if there is anything to read */
-    retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
-    if (!retval) {
-      log_warn(LD_GENERAL,
-        "Failed to peek from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* Nothing available: process exited or it is busy */
-
-      /* Exit if we don't know whether the process is running */
-      if (NULL == process)
-        break;
-
-      /* The process exited and there's nothing left to read from it */
-      if (process_exited)
-        break;
-
-      /* If process is not running, check for output one more time in case
-         it wrote something after the peek was performed. Otherwise keep on
-         waiting for output */
-      tor_assert(process != NULL);
-      byte_count = WaitForSingleObject(process->pid.hProcess, 0);
-      if (WAIT_TIMEOUT != byte_count)
-        process_exited = TRUE;
-
-      continue;
-    }
-
-    /* There is data to read; read it */
-    retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL);
-    tor_assert(byte_count + numread <= count);
-    if (!retval) {
-      log_warn(LD_GENERAL, "Failed to read from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* End of file */
-      break;
-    }
-    numread += byte_count;
-  }
-  return (ssize_t)numread;
-}
-#else /* !(defined(_WIN32)) */
-/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>process</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise data will be read until end of file, or
- * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
- * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
- * file has been reached. */
-ssize_t
-tor_read_all_handle(int fd, char *buf, size_t count,
-                    const process_handle_t *process,
-                    int *eof)
-{
-  size_t numread = 0;
-  ssize_t result;
-
-  if (eof)
-    *eof = 0;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    result = read(fd, buf+numread, count-numread);
-
-    if (result == 0) {
-      log_debug(LD_GENERAL, "read() reached end of file");
-      if (eof)
-        *eof = 1;
-      break;
-    } else if (result < 0 && errno == EAGAIN) {
-      if (process)
-        continue;
-      else
-        break;
-    } else if (result < 0) {
-      log_warn(LD_GENERAL, "read() failed: %s", strerror(errno));
-      return -1;
-    }
-
-    numread += result;
-  }
-
-  log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread);
-  return (ssize_t)numread;
-}
-#endif /* defined(_WIN32) */
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stdout(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stderr(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Return a string corresponding to <b>stream_status</b>. */
-const char *
-stream_status_to_string(enum stream_status stream_status)
-{
-  switch (stream_status) {
-    case IO_STREAM_OKAY:
-      return "okay";
-    case IO_STREAM_EAGAIN:
-      return "temporarily unavailable";
-    case IO_STREAM_TERM:
-      return "terminated";
-    case IO_STREAM_CLOSED:
-      return "closed";
-    default:
-      tor_fragile_assert();
-      return "unknown";
-  }
-}
-
-/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be
- * modified. The resulting smartlist will consist of pointers to buf, so there
- * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated
- * string. <b>len</b> should be set to the length of the buffer excluding the
- * NUL. Non-printable characters (including NUL) will be replaced with "." */
-int
-tor_split_lines(smartlist_t *sl, char *buf, int len)
-{
-  /* Index in buf of the start of the current line */
-  int start = 0;
-  /* Index in buf of the current character being processed */
-  int cur = 0;
-  /* Are we currently in a line */
-  char in_line = 0;
-
-  /* Loop over string */
-  while (cur < len) {
-    /* Loop until end of line or end of string */
-    for (; cur < len; cur++) {
-      if (in_line) {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* End of line */
-          buf[cur] = '\0';
-          /* Point cur to the next line */
-          cur++;
-          /* Line starts at start and ends with a nul */
-          break;
-        } else {
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      } else {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* Skip leading vertical space */
-          ;
-        } else {
-          in_line = 1;
-          start = cur;
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      }
-    }
-    /* We are at the end of the line or end of string. If in_line is true there
-     * is a line which starts at buf+start and ends at a NUL. cur points to
-     * the character after the NUL. */
-    if (in_line)
-      smartlist_add(sl, (void *)(buf+start));
-    in_line = 0;
-  }
-  return smartlist_len(sl);
-}
-
-#ifdef _WIN32
-
-/** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (HANDLE *handle,
-                            enum stream_status *stream_status_out))
-{
-  int pos;
-  char stdout_buf[600] = {0};
-  smartlist_t *lines = NULL;
-
-  tor_assert(stream_status_out);
-
-  *stream_status_out = IO_STREAM_TERM;
-
-  pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL);
-  if (pos < 0) {
-    *stream_status_out = IO_STREAM_TERM;
-    return NULL;
-  }
-  if (pos == 0) {
-    *stream_status_out = IO_STREAM_EAGAIN;
-    return NULL;
-  }
-
-  /* End with a null even if there isn't a \r\n at the end */
-  /* TODO: What if this is a partial line? */
-  stdout_buf[pos] = '\0';
-
-  /* Split up the buffer */
-  lines = smartlist_new();
-  tor_split_lines(lines, stdout_buf, pos);
-
-  /* Currently 'lines' is populated with strings residing on the
-     stack. Replace them with their exact copies on the heap: */
-  SMARTLIST_FOREACH(lines, char *, line,
-                    SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line)));
-
-  *stream_status_out = IO_STREAM_OKAY;
-
-  return lines;
-}
-
-#else /* !(defined(_WIN32)) */
-
-/** Return a smartlist containing lines outputted from
- *  <b>fd</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
-{
-  enum stream_status stream_status;
-  char stdout_buf[400];
-  smartlist_t *lines = NULL;
-
-  while (1) {
-    memset(stdout_buf, 0, sizeof(stdout_buf));
-
-    stream_status = get_string_from_pipe(fd,
-                                         stdout_buf, sizeof(stdout_buf) - 1);
-    if (stream_status != IO_STREAM_OKAY)
-      goto done;
-
-    if (!lines) lines = smartlist_new();
-    smartlist_split_string(lines, stdout_buf, "\n", 0, 0);
-  }
-
- done:
-  *stream_status_out = stream_status;
-  return lines;
-}
-
-#endif /* defined(_WIN32) */
-
-/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
- *  sure it's below <b>count</b> bytes.
- *  If the string has a trailing newline, we strip it off.
- *
- * This function is specifically created to handle input from managed
- * proxies, according to the pluggable transports spec. Make sure it
- * fits your needs before using it.
- *
- * Returns:
- * IO_STREAM_CLOSED: If the stream is closed.
- * IO_STREAM_EAGAIN: If there is nothing to read and we should check back
- *  later.
- * IO_STREAM_TERM: If something is wrong with the stream.
- * IO_STREAM_OKAY: If everything went okay and we got a string
- *  in <b>buf_out</b>. */
-enum stream_status
-get_string_from_pipe(int fd, char *buf_out, size_t count)
-{
-  ssize_t ret;
-
-  tor_assert(count <= INT_MAX);
-
-  ret = read(fd, buf_out, count);
-
-  if (ret == 0)
-    return IO_STREAM_CLOSED;
-  else if (ret < 0 && errno == EAGAIN)
-    return IO_STREAM_EAGAIN;
-  else if (ret < 0)
-    return IO_STREAM_TERM;
-
-  if (buf_out[ret - 1] == '\n') {
-    /* Remove the trailing newline */
-    buf_out[ret - 1] = '\0';
-  } else
-    buf_out[ret] = '\0';
-
-  return IO_STREAM_OKAY;
-}
diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h
deleted file mode 100644
index aa3127d62d..0000000000
--- a/src/lib/process/subprocess.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.h
- * \brief Header for subprocess.c
- **/
-
-#ifndef TOR_SUBPROCESS_H
-#define TOR_SUBPROCESS_H
-
-#include "lib/cc/torint.h"
-#include "lib/testsupport/testsupport.h"
-#include <stddef.h>
-#ifdef _WIN32
-#include <windows.h>
-#endif
-
-struct smartlist_t;
-
-void tor_disable_spawning_background_processes(void);
-
-typedef struct process_handle_t process_handle_t;
-struct process_environment_t;
-int tor_spawn_background(const char *const filename, const char **argv,
-                         struct process_environment_t *env,
-                         process_handle_t **process_handle_out);
-
-#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code "
-
-/** Status of an I/O stream. */
-enum stream_status {
-  IO_STREAM_OKAY,
-  IO_STREAM_EAGAIN,
-  IO_STREAM_TERM,
-  IO_STREAM_CLOSED
-};
-
-const char *stream_status_to_string(enum stream_status stream_status);
-
-enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
-
-/* Values of process_handle_t.status. */
-#define PROCESS_STATUS_NOTRUNNING 0
-#define PROCESS_STATUS_RUNNING 1
-#define PROCESS_STATUS_ERROR -1
-
-#ifdef SUBPROCESS_PRIVATE
-struct waitpid_callback_t;
-
-/** Structure to represent the state of a process with which Tor is
- * communicating. The contents of this structure are private to util.c */
-struct process_handle_t {
-  /** One of the PROCESS_STATUS_* values */
-  int status;
-#ifdef _WIN32
-  HANDLE stdin_pipe;
-  HANDLE stdout_pipe;
-  HANDLE stderr_pipe;
-  PROCESS_INFORMATION pid;
-#else /* !(defined(_WIN32)) */
-  int stdin_pipe;
-  int stdout_pipe;
-  int stderr_pipe;
-  pid_t pid;
-  /** If the process has not given us a SIGCHLD yet, this has the
-   * waitpid_callback_t that gets invoked once it has. Otherwise this
-   * contains NULL. */
-  struct waitpid_callback_t *waitpid_cb;
-  /** The exit status reported by waitpid. */
-  int waitpid_exit_status;
-#endif /* defined(_WIN32) */
-};
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-/* Return values of tor_get_exit_code() */
-#define PROCESS_EXIT_RUNNING 1
-#define PROCESS_EXIT_EXITED 0
-#define PROCESS_EXIT_ERROR -1
-int tor_get_exit_code(process_handle_t *process_handle,
-                      int block, int *exit_code);
-int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
-#ifdef _WIN32
-ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                            const process_handle_t *process);
-#else
-ssize_t tor_read_all_handle(int fd, char *buf, size_t count,
-                            const process_handle_t *process,
-                            int *eof);
-#endif /* defined(_WIN32) */
-ssize_t tor_read_all_from_process_stdout(
-    const process_handle_t *process_handle, char *buf, size_t count);
-ssize_t tor_read_all_from_process_stderr(
-    const process_handle_t *process_handle, char *buf, size_t count);
-char *tor_join_win_cmdline(const char *argv[]);
-
-int tor_process_get_pid(process_handle_t *process_handle);
-#ifdef _WIN32
-HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#else
-int tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#endif
-
-#ifdef _WIN32
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle,
-                           enum stream_status *stream_status));
-#else
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd,
-                                       enum stream_status *stream_status));
-#endif /* defined(_WIN32) */
-
-int tor_terminate_process(process_handle_t *process_handle);
-
-MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle,
-                                            int also_terminate_process));
-
-#ifdef SUBPROCESS_PRIVATE
-/* Prototypes for private functions only used by util.c (and unit tests) */
-
-#ifndef _WIN32
-STATIC int format_helper_exit_status(unsigned char child_state,
-                              int saved_errno, char *hex_errno);
-
-/* Space for hex values of child state, a slash, saved_errno (with
-   leading minus) and newline (no null) */
-#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \
-                        1 + sizeof(int) * 2 + 1)
-#endif /* !defined(_WIN32) */
-
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-#endif
diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c
index 9b626394d2..33798f65f0 100644
--- a/src/lib/process/waitpid.c
+++ b/src/lib/process/waitpid.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
-#include "ht.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
@@ -58,9 +58,9 @@ process_map_entries_eq_(const waitpid_callback_t *a,
 static HT_HEAD(process_map, waitpid_callback_t) process_map = HT_INITIALIZER();
 
 HT_PROTOTYPE(process_map, waitpid_callback_t, node, process_map_entry_hash_,
-             process_map_entries_eq_)
+             process_map_entries_eq_);
 HT_GENERATE2(process_map, waitpid_callback_t, node, process_map_entry_hash_,
-             process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_)
+             process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_);
 
 /**
  * Begin monitoring the child pid <b>pid</b> to see if we get a SIGCHLD for
diff --git a/src/lib/process/waitpid.h b/src/lib/process/waitpid.h
index 5faef468c1..171cf206fb 100644
--- a/src/lib/process/waitpid.h
+++ b/src/lib/process/waitpid.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/pubsub/.may_include b/src/lib/pubsub/.may_include
new file mode 100644
index 0000000000..5623492f00
--- /dev/null
+++ b/src/lib/pubsub/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/pubsub/*.h
+lib/string/*.h
diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am
new file mode 100644
index 0000000000..e2abebcd40
--- /dev/null
+++ b/src/lib/pubsub/include.am
@@ -0,0 +1,28 @@
+
+noinst_LIBRARIES += src/lib/libtor-pubsub.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_pubsub_a_SOURCES =			\
+	src/lib/pubsub/pubsub_build.c			\
+	src/lib/pubsub/pubsub_check.c			\
+	src/lib/pubsub/pubsub_publish.c
+
+src_lib_libtor_pubsub_testing_a_SOURCES = \
+	$(src_lib_libtor_pubsub_a_SOURCES)
+src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/pubsub/pub_binding_st.h			\
+	src/lib/pubsub/pubsub.h				\
+	src/lib/pubsub/pubsub_build.h			\
+	src/lib/pubsub/pubsub_builder_st.h		\
+	src/lib/pubsub/pubsub_connect.h			\
+	src/lib/pubsub/pubsub_flags.h			\
+	src/lib/pubsub/pubsub_macros.h			\
+	src/lib/pubsub/pubsub_publish.h
diff --git a/src/lib/pubsub/lib_pubsub.md b/src/lib/pubsub/lib_pubsub.md
new file mode 100644
index 0000000000..3f4c473436
--- /dev/null
+++ b/src/lib/pubsub/lib_pubsub.md
@@ -0,0 +1,14 @@
+@dir /lib/pubsub
+@brief lib/pubsub: Publish-subscribe message passing.
+
+This module wraps the \refdir{lib/dispatch} module, to provide a more
+ergonomic and type-safe approach to message passing.
+
+In general, we favor this mechanism for cases where higher-level modules
+need to be notified when something happens in lower-level modules. (The
+alternative would be calling up from the lower-level modules, which
+would be error-prone; or maintaining lists of function-pointers, which
+would be clumsy and tend to complicate the call graph.)
+
+See pubsub.c for more information.
+
diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h
new file mode 100644
index 0000000000..d7c562fc35
--- /dev/null
+++ b/src/lib/pubsub/pub_binding_st.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pub_binding_st.h
+ * @brief Declaration of pub_binding_t.
+ *
+ * This is an internal type for the pubsub implementation.
+ */
+
+#ifndef TOR_PUB_BINDING_ST_H
+#define TOR_PUB_BINDING_ST_H
+
+#include "lib/dispatch/msgtypes.h"
+struct dispatch_t;
+
+/**
+ * A pub_binding_t is an opaque object that subsystems use to publish
+ * messages.  The DISPATCH_ADD_PUB*() macros set it up.
+ **/
+typedef struct pub_binding_t {
+  /**
+   * A pointer to a configured dispatch_t object.  This is filled in
+   * when the dispatch_t is finally constructed.
+   **/
+  struct dispatch_t *dispatch_ptr;
+  /**
+   * A template for the msg_t fields that are filled in for this message.
+   * This is copied into outgoing messages, ensuring that their fields are set
+   * corretly.
+   **/
+  msg_t msg_template;
+} pub_binding_t;
+
+#endif /* !defined(TOR_PUB_BINDING_ST_H) */
diff --git a/src/lib/pubsub/publish_subscribe.md b/src/lib/pubsub/publish_subscribe.md
new file mode 100644
index 0000000000..bb05b100b1
--- /dev/null
+++ b/src/lib/pubsub/publish_subscribe.md
@@ -0,0 +1,144 @@
+
+@page publish_subscribe Publish-subscribe message passing in Tor
+
+@tableofcontents
+
+## Introduction
+
+Tor has introduced a generic publish-subscribe mechanism for delivering
+messages internally.  It is meant to help us improve the modularity of
+our code, by avoiding direct coupling between modules that don't
+actually need to invoke one another.
+
+This publish-subscribe mechanism is *not* meant for handing
+multithreading or multiprocess issues, thought we hope that eventually
+it might be extended and adapted for that purpose.  Instead, we use
+publish-subscribe today to decouple modules that shouldn't be calling
+each other directly.
+
+For example, there are numerous parts of our code that might need to
+take action when a circuit is completed: a controller might need to be
+informed, an onion service negotiation might need to be attached, a
+guard might need to be marked as working, or a client connection might
+need to be attached.  But many of those actions occur at a higher layer
+than circuit completion: calling them directly is a layering violation,
+and makes our code harder to understand and analyze.
+
+But with message-passing, we can invert this layering violation: circuit
+completion can become a "message" that the circuit code publishes, and
+to which higher-level layers subscribe.  This means that circuit
+handling can be decoupled from higher-level modules, and stay nice and
+simple. (@ref pubsub_notyet "1")
+
+> @anchor pubsub_notyet 1. Unfortunately, like most of our code, circuit
+> handling is _not_ yet refactored to use publish-subscribe throughout.
+> Instead, layer violations of the type described here are pretty common
+> in Tor today.  To see a small part of what happens when a circuit is
+> completed today, have a look at circuit_build_no_more_hops() and its
+> associated code.
+
+## Channels and delivery policies
+
+To work with messages, especially when refactoring existing code, you'll
+need to understand "channels" and "delivery policies".
+
+Every message is delivered on a "message channel".  Each channel
+(conceptually) a queue-like structure that can support an arbitrarily
+number of message types.  Where channels vary is their delivery
+mechanisms, and their guarantees about when messages are processed.
+
+Currently, three delivery policies are possible:
+
+   - `DELIV_PROMPT` -- causes messages to be processed via a callback in
+      Tor's event loop.  This is generally the best choice, since it
+      avoids unexpected growth of the stack.
+
+   - `DELIV_IMMEDIATE` -- causes messages to be processed immediately
+      on the call stack when they are published.  This choice grows the
+      stack, and can lead to unexpected complexity in the call graph.
+      We should only use it when necessary.
+
+   - `DELIV_NEVER` -- causes messages not to be delivered by the message
+      dispatch system at all. Instead, some other part of the code must
+      call dispatch_flush() to get the messages delivered.
+
+See mainloop_pubsub.c and mainloop_pubsub.h for more information and
+implementation details.
+
+## Layers: Dispatch vs publish-subsubscribe vs mainloop.
+
+At the lowest level, messages are sent via the "dispatcher" module in
+@refdir{lib/dispatch}.  For performance, this dispatcher works with a
+untyped messages.  Publishers, subscribers, channels, and messages are
+distinguished by short integers.  Associated data is handled as
+dynamically-typed data pointers, and its types are also stored as short
+integers.
+
+Naturally, this results in a type-unsafe C API, so most other modules
+shouldn't invoke @refdir{lib/dispatch} directly.  At a higher level,
+@refdir{lib/pubsub} defines a set of functions and macros that make
+messages named and type-safe.  This is the one that other modules should
+use when they want to send or receive a message.
+
+The two modules above do not handle message delivery.  Instead, the
+dispatch module takes a callback that it can invoke when a channel
+becomes nonempty, and defines a dispatch_flush() function to deliver all
+the messages queued in a channel.  The work of actually making sure that
+dispatch_flush() is called when appropriate falls to the main loop,
+which needs to integrate the message dispatcher with the rest of our
+events and callbacks.  This work happens in mainloop_pubsub.c.
+
+
+## How to publish and subscribe
+
+This section gives an overview of how to make new messages and how to
+use them.  For full details, see pubsub_macros.h.
+
+Before anybody can publish or subscribe to a message, the message must
+be declared, typically in a header.  This uses DECLARE_MESSAGE() or
+DECLARE_MESSAGE_INT().
+
+Only subsystems can publish or subscribe messages.  For more information
+about the subsystems architecture, see @ref initialization.
+
+To publish a message, you must:
+   - Include the header that declares the message.
+   - Declare a set of helper functions via DECLARE_PUBLISH().  These
+     must be visible wherever you call PUBLISH().
+   - Call PUBLISH() to actually send a message.
+   - Connect your subsystem to the dispatcher by calling
+     DISPATCH_ADD_PUB() from your subsystem's subsys_fns_t.add_pubsub
+     callback.
+
+To subscribe to a message, you must:
+   - Include the header that declares the message.
+   - Declare a callback function to be invoked when the message is delivered.
+   - Use DISPATCH_SUBSCRIBE at file scope to define a set of wrapper
+     functions to call your callback function with the appropriate type.
+   - Connect your subsystem to the dispatcher by calling
+     DISPATCH_ADD_SUB() from your subsystem's subsys_fns_t.add_pubsub
+     callback.
+
+Again, the file-level documentation for pubsub_macros.h describes how to
+declare a message, how to publish it, and how to subscribe to it.
+
+## Designing good messages
+
+**Frequency**:
+The publish-subscribe system uses a few function calls
+and allocations for each message sent. This makes it unsuitable for
+very-high-bandwidth events, like "receiving a single data cell" or "a
+socket has become writable."  It's fine, however, for events that
+ordinarily happen a bit less frequently than that, like a circuit
+getting finished, a new connection getting opened, or so on.
+
+**Semantics**:
+A message should declare that something has happened or is happening,
+not that something in particular should be done.
+
+For example, suppose you want to set up a message so that onion services
+clean up their replay caches whenever we're low on memory.  The event
+should be something like `memory_low`, not `clean_up_replay_caches`.
+The latter name would imply that the publisher knew who was subscribing
+to the message and what they intended to do about it, which would be a
+layering violation.
diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h
new file mode 100644
index 0000000000..d0a4d317f3
--- /dev/null
+++ b/src/lib/pubsub/pubsub.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub.h
+ * @brief Header for OO publish-subscribe functionality.
+ *
+ * This module provides a wrapper around the "dispatch" module,
+ * ensuring type-safety and allowing us to do static analysis on
+ * publication and subscriptions.
+ *
+ * With this module, we enforce:
+ *  <ul>
+ *   <li>that every message has (potential) publishers and subscribers;
+ *   <li>that every message is published and subscribed from the correct
+ *       channels, with the correct type ID, every time it is published.
+ *   <li>that type IDs correspond to a single C type, and that the C types are
+ *       used correctly.
+ *   <li>that when a message is published or subscribed, it is done with
+ *       a correct subsystem identifier
+ * </ul>
+ *
+ * We do this by making "publication requests" and "subscription requests"
+ * into objects, and doing some computation on them before we create
+ * a dispatch_t with them.
+ *
+ * Rather than using the dispatch module directly, a publishing module
+ * receives a "binding" object that it uses to send messages with the right
+ * settings.
+ *
+ * Most users of this module will want to use this header, and the
+ * pubsub_macros.h header for convenience.
+ */
+
+/*
+ *
+ * Overview: Messages are sent over channels.  Before sending a message on a
+ * channel, or receiving a message on a channel, a subsystem needs to register
+ * that it publishes, or subscribes, to that message, on that channel.
+ *
+ * Messages, channels, and subsystems are represented internally as short
+ * integers, though they are associated with human-readable strings for
+ * initialization and debugging.
+ *
+ * When registering for a message, a subsystem must say whether it is an
+ * exclusive publisher/subscriber to that message type, or whether other
+ * subsystems may also publish/subscribe to it.
+ *
+ * All messages and their publishers/subscribers must be registered early in
+ * the initialization process.
+ *
+ * By default, it is an error for a message type to have publishers and no
+ * subscribers on a channel, or subscribers and no publishers on a channel.
+ *
+ * A subsystem may register for a message with a note that delivery or
+ * production is disabled -- for example, because the subsystem is
+ * disabled at compile-time. It is not an error for a message type to
+ * have all of its publishers or subscribers disabled.
+ *
+ * After a message is sent, it is delivered to every recipient.  This
+ * delivery happens from the top level of the event loop; it may be
+ * interleaved with network events, timers, etc.
+ *
+ * Messages may have associated data.  This data is typed, and is owned
+ * by the message.  Strings, byte-arrays, and integers have built-in
+ * support.  Other types may be added.  If objects are to be sent,
+ * they should be identified by handle.  If an object requires cleanup,
+ * it should be declared with an associated free function.
+ *
+ * Semantically, if two subsystems communicate only by this kind of
+ * message passing, neither is considered to depend on the other, though
+ * both are considered to have a dependency on the message and on any
+ * types it contains.
+ *
+ * (Or generational index?)
+ */
+#ifndef TOR_PUBSUB_PUBSUB_H
+#define TOR_PUBSUB_PUBSUB_H
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#endif /* !defined(TOR_PUBSUB_PUBSUB_H) */
diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c
new file mode 100644
index 0000000000..3c134f015c
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.c
@@ -0,0 +1,307 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.c
+ * @brief Construct a dispatch_t in safer, more OO way.
+ **/
+
+#define PUBSUB_PRIVATE
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+ #include <string.h>
+
+/** Construct and return a new empty pubsub_items_t. */
+static pubsub_items_t *
+pubsub_items_new(void)
+{
+  pubsub_items_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->items = smartlist_new();
+  cfg->type_items = smartlist_new();
+  return cfg;
+}
+
+/** Release all storage held in a pubsub_items_t. */
+void
+pubsub_items_free_(pubsub_items_t *cfg)
+{
+  if (! cfg)
+    return;
+  SMARTLIST_FOREACH(cfg->items, pubsub_cfg_t *, item, tor_free(item));
+  SMARTLIST_FOREACH(cfg->type_items,
+                    pubsub_type_cfg_t *, item, tor_free(item));
+  smartlist_free(cfg->items);
+  smartlist_free(cfg->type_items);
+  tor_free(cfg);
+}
+
+/** Construct and return a new pubsub_builder_t. */
+pubsub_builder_t *
+pubsub_builder_new(void)
+{
+  dispatch_naming_init();
+
+  pubsub_builder_t *pb = tor_malloc_zero(sizeof(*pb));
+  pb->cfg = dcfg_new();
+  pb->items = pubsub_items_new();
+  return pb;
+}
+
+/**
+ * Release all storage held by a pubsub_builder_t.
+ *
+ * You'll (mostly) only want to call this function on an error case: if you're
+ * constructing a dispatch_t instead, you should call
+ * pubsub_builder_finalize() to consume the pubsub_builder_t.
+ */
+void
+pubsub_builder_free_(pubsub_builder_t *pb)
+{
+  if (pb == NULL)
+    return;
+  pubsub_items_free(pb->items);
+  dcfg_free(pb->cfg);
+  tor_free(pb);
+}
+
+/**
+ * Create and return a pubsub_connector_t for the subsystem with ID
+ * <b>subsys</b> to use in adding publications, subscriptions, and types to
+ * <b>builder</b>.
+ **/
+pubsub_connector_t *
+pubsub_connector_for_subsystem(pubsub_builder_t *builder,
+                               subsys_id_t subsys)
+{
+  tor_assert(builder);
+  ++builder->n_connectors;
+
+  pubsub_connector_t *con = tor_malloc_zero(sizeof(*con));
+
+  con->builder = builder;
+  con->subsys_id = subsys;
+
+  return con;
+}
+
+/**
+ * Release all storage held by a pubsub_connector_t.
+ **/
+void
+pubsub_connector_free_(pubsub_connector_t *con)
+{
+  if (!con)
+    return;
+
+  if (con->builder) {
+    --con->builder->n_connectors;
+    tor_assert(con->builder->n_connectors >= 0);
+  }
+  tor_free(con);
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>.
+ **/
+int
+pubsub_add_pub_(pubsub_connector_t *con,
+                pub_binding_t *out,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  memset(out, 0, sizeof(*out));
+  cfg->is_publish = true;
+
+  out->msg_template.sender = cfg->subsys = con->subsys_id;
+  out->msg_template.channel = cfg->channel = channel;
+  out->msg_template.msg = cfg->msg = msg;
+  out->msg_template.type = cfg->type = type;
+
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  /* We're grabbing a pointer to the pub_binding_t so we can tell it about
+   * the dispatcher later on.
+   */
+  cfg->pub_binding = out;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>,
+ * passing them to the callback in <b>recv_fn</b>.
+ **/
+int
+pubsub_add_sub_(pubsub_connector_t *con,
+                recv_fn_t recv_fn,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  cfg->is_publish = false;
+  cfg->subsys = con->subsys_id;
+  cfg->channel = channel;
+  cfg->msg = msg;
+  cfg->type = type;
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  cfg->recv_fn = recv_fn;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+  if (! (flags & DISP_FLAG_STUB)) {
+    if (dcfg_add_recv(con->builder->cfg, msg, cfg->subsys, recv_fn) < 0)
+      goto err;
+  }
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to define the functions to use for manipulating the type
+ * <b>type</b>.  Any function pointers left as NULL will be implemented as
+ * no-ops.
+ **/
+int
+pubsub_connector_register_type_(pubsub_connector_t *con,
+                                msg_type_id_t type,
+                                dispatch_typefns_t *fns,
+                                const char *file,
+                                unsigned line)
+{
+  pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->type = type;
+  memcpy(&cfg->fns, fns, sizeof(*fns));
+  cfg->subsys = con->subsys_id;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  smartlist_add(con->builder->items->type_items, cfg);
+
+  if (dcfg_type_set_fns(con->builder->cfg, type, fns) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Initialize the dispatch_ptr field in every relevant publish binding
+ * for <b>d</b>.
+ */
+static void
+pubsub_items_install_bindings(pubsub_items_t *items,
+                              dispatch_t *d)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      // XXXX we could skip this for STUB publishers, and for any publishers
+      // XXXX where all subscribers are STUB.
+      cfg->pub_binding->dispatch_ptr = d;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Remove the dispatch_ptr fields for all the relevant publish bindings
+ * in <b>items</b>.  The prevents subsequent dispatch_pub_() calls from
+ * sending messages to a dispatcher that has been freed.
+ **/
+void
+pubsub_items_clear_bindings(pubsub_items_t *items)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      cfg->pub_binding->dispatch_ptr = NULL;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Create a new dispatcher as configured in a pubsub_builder_t.
+ *
+ * Consumes and frees its input.
+ **/
+dispatch_t *
+pubsub_builder_finalize(pubsub_builder_t *builder,
+                        pubsub_items_t **items_out)
+{
+  dispatch_t *dispatcher = NULL;
+  tor_assert_nonfatal(builder->n_connectors == 0);
+
+  if (pubsub_builder_check(builder) < 0)
+    goto err;
+
+  if (builder->n_errors) {
+    log_warn(LD_GENERAL, "At least one error occurred previously when "
+             "configuring the dispatcher.");
+    goto err;
+  }
+
+  dispatcher = dispatch_new(builder->cfg);
+
+  if (!dispatcher)
+    goto err;
+
+  pubsub_items_install_bindings(builder->items, dispatcher);
+  if (items_out) {
+    *items_out = builder->items;
+    builder->items = NULL; /* Prevent free */
+  }
+
+ err:
+  pubsub_builder_free(builder);
+  return dispatcher;
+}
diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h
new file mode 100644
index 0000000000..2781b8251a
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.h
@@ -0,0 +1,97 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.h
+ * @brief Header used for constructing the OO publish-subscribe facility.
+ *
+ * (See pubsub.h for more general information on this API.)
+ **/
+
+#ifndef TOR_PUBSUB_BUILD_H
+#define TOR_PUBSUB_BUILD_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct dispatch_t;
+struct pubsub_connector_t;
+
+/**
+ * A "dispatch builder" is an incomplete dispatcher, used when
+ * registering messages.  It does not have the same integrity guarantees
+ * as a dispatcher.  It cannot actually handle messages itself: once all
+ * subsystems have registered, it is converted into a dispatch_t.
+ **/
+typedef struct pubsub_builder_t pubsub_builder_t;
+
+/**
+ * A "pubsub items" holds the configuration items used to configure a
+ * pubsub_builder.  After the builder is finalized, this field is extracted,
+ * and used later to tear down pointers that enable publishing.
+ **/
+typedef struct pubsub_items_t pubsub_items_t;
+
+/**
+ * Create a new pubsub_builder. This should only happen in the
+ * main-init code.
+ */
+pubsub_builder_t *pubsub_builder_new(void);
+
+/** DOCDOC */
+int pubsub_builder_check(pubsub_builder_t *);
+
+/**
+ * Free a pubsub builder.  This should only happen on error paths, where
+ * we have decided not to construct a dispatcher for some reason.
+ */
+#define pubsub_builder_free(db) \
+  FREE_AND_NULL(pubsub_builder_t, pubsub_builder_free_, (db))
+
+/** Internal implementation of pubsub_builder_free(). */
+void pubsub_builder_free_(pubsub_builder_t *);
+
+/**
+ * Create a pubsub connector that a single subsystem will use to
+ * register its messages.  The main-init code does this during susbsystem
+ * initialization.
+ */
+struct pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *,
+                                                          subsys_id_t);
+
+/**
+ * The main-init code does this after subsystem initialization.
+ */
+#define pubsub_connector_free(c) \
+  FREE_AND_NULL(struct pubsub_connector_t, pubsub_connector_free_, (c))
+
+void pubsub_connector_free_(struct pubsub_connector_t *);
+
+/**
+ * Constructs a dispatcher from a dispatch_builder, after checking that the
+ * invariances on the messages, channels, and connections have been
+ * respected.
+ *
+ * This should happen after every subsystem has initialized, and before
+ * entering the mainloop.
+ */
+struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *,
+                                           pubsub_items_t **items_out);
+
+/**
+ * Clear all pub_binding_t backpointers in <b>items</b>.
+ **/
+void pubsub_items_clear_bindings(pubsub_items_t *items);
+
+/**
+ * @copydoc pubsub_items_free_
+ *
+ * Additionally, set the pointer <b>cfg</b> to NULL.
+ **/
+#define pubsub_items_free(cfg) \
+  FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg))
+void pubsub_items_free_(pubsub_items_t *cfg);
+
+#endif /* !defined(TOR_PUBSUB_BUILD_H) */
diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h
new file mode 100644
index 0000000000..57de1240ee
--- /dev/null
+++ b/src/lib/pubsub/pubsub_builder_st.h
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_builder_st.h
+ *
+ * @brief private structures used for configuring dispatchers and messages.
+ */
+
+#ifndef TOR_PUBSUB_BUILDER_ST_H
+#define TOR_PUBSUB_BUILDER_ST_H
+
+#ifdef PUBSUB_PRIVATE
+
+#include <stdbool.h>
+#include <stddef.h>
+
+struct dispatch_cfg_t;
+struct smartlist_t;
+struct pub_binding_t;
+
+/**
+ * Configuration for a single publication or subscription request.
+ *
+ * These can be stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ *
+ * There are various fields in this request describing the message; all of
+ * them must match other descriptions of the message, or a bug has occurred.
+ **/
+typedef struct pubsub_cfg_t {
+  /** True if this is a publishing request; false for a subscribing request. */
+  bool is_publish;
+  /** The system making this request. */
+  subsys_id_t subsys;
+  /** The channel on which the message is to be sent. */
+  channel_id_t channel;
+  /** The message ID to be sent or received. */
+  message_id_t msg;
+  /** The C type associated with the message. */
+  msg_type_id_t type;
+  /** One or more DISP_FLAGS_* items, combined with bitwise OR. */
+  unsigned flags;
+
+  /**
+   * Publishing only: a pub_binding object that will receive the binding for
+   * this request.  We will finish filling this in when the dispatcher is
+   * constructed, so that the subsystem can publish then and not before.
+   */
+  struct pub_binding_t *pub_binding;
+
+  /**
+   * Subscribing only: a function to receive message objects for this request.
+   */
+  recv_fn_t recv_fn;
+
+  /** The file from which this message was configured */
+  const char *added_by_file;
+  /** The line at which this message was configured */
+  unsigned added_by_line;
+} pubsub_cfg_t;
+
+/**
+ * Configuration request for a single C type.
+ *
+ * These are stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ **/
+typedef struct pubsub_type_cfg_t {
+  /**
+   * The identifier for this type.
+   */
+  msg_type_id_t type;
+  /**
+   * Functions to use when manipulating the type.
+   */
+  dispatch_typefns_t fns;
+
+  /** The subsystem that configured this type. */
+  subsys_id_t subsys;
+  /** The file from which this type was configured */
+  const char *added_by_file;
+  /** The line at which this type was configured */
+  unsigned added_by_line;
+} pubsub_type_cfg_t;
+
+/**
+ * The set of configuration requests for a dispatcher, as made by various
+ * subsystems.
+ **/
+struct pubsub_items_t {
+  /** List of pubsub_cfg_t. */
+  struct smartlist_t *items;
+  /** List of pubsub_type_cfg_t. */
+  struct smartlist_t *type_items;
+};
+
+/**
+ * Type used to construct a dispatcher.  We use this type to build up the
+ * configuration for a dispatcher, and then pass ownership of that
+ * configuration to the newly constructed dispatcher.
+ **/
+struct pubsub_builder_t {
+  /** Number of outstanding pubsub_connector_t objects pointing to this
+   * pubsub_builder_t. */
+  int n_connectors;
+  /** Number of errors encountered while constructing this object so far. */
+  int n_errors;
+  /** In-progress configuration that we're constructing, as a list of the
+   * requests that have been made. */
+  struct pubsub_items_t *items;
+  /** In-progress configuration that we're constructing, in a form that can
+   * be converted to a dispatch_t. */
+  struct dispatch_cfg_t *cfg;
+};
+
+/**
+ * Type given to a subsystem when adding connections to a pubsub_builder_t.
+ * We use this type to force each subsystem to get blamed for the
+ * publications, subscriptions, and types that it adds.
+ **/
+struct pubsub_connector_t {
+  /** The pubsub_builder that this connector refers to. */
+  struct pubsub_builder_t *builder;
+  /** The subsystem that has been given this connector. */
+  subsys_id_t subsys_id;
+};
+
+/**
+ * Helper structure used when constructing a dispatcher that sorts the
+ * pubsub_cfg_t objects in various ways.
+ **/
+typedef struct pubsub_adjmap_t {
+  /* XXXX The next three fields are currently constructed but not yet
+   * XXXX used. I believe we'll want them in the future, though. -nickm
+   */
+  /** Number of subsystems; length of the *_by_subsys arrays. */
+  size_t n_subsystems;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **pub_by_subsys;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **sub_by_subsys;
+
+  /** Number of message IDs; length of the *_by_msg arrays. */
+  size_t n_msgs;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **pub_by_msg;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **sub_by_msg;
+} pubsub_adjmap_t;
+
+#endif /* defined(PUBSUB_PRIVATE) */
+
+#endif /* !defined(TOR_PUBSUB_BUILDER_ST_H) */
diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c
new file mode 100644
index 0000000000..dbcbb14746
--- /dev/null
+++ b/src/lib/pubsub/pubsub_check.c
@@ -0,0 +1,414 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_check.c
+ * @brief Enforce various requirements on a pubsub_builder.
+ **/
+
+/** @{ */
+#define PUBSUB_PRIVATE
+/** @} */
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_build.h"
+
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_string.h"
+
+#include <string.h>
+
+static void pubsub_adjmap_add(pubsub_adjmap_t *map,
+                                const pubsub_cfg_t *item);
+
+/**
+ * Helper: contruct and return a new pubsub_adjacency_map from <b>cfg</b>.
+ * Return NULL on error.
+ **/
+static pubsub_adjmap_t *
+pubsub_build_adjacency_map(const pubsub_items_t *cfg)
+{
+  pubsub_adjmap_t *map = tor_malloc_zero(sizeof(*map));
+  const size_t n_subsystems = get_num_subsys_ids();
+  const size_t n_msgs = get_num_message_ids();
+
+  map->n_subsystems = n_subsystems;
+  map->n_msgs = n_msgs;
+
+  map->pub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->sub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->pub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+  map->sub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+
+  SMARTLIST_FOREACH_BEGIN(cfg->items, const pubsub_cfg_t *, item) {
+    pubsub_adjmap_add(map, item);
+  } SMARTLIST_FOREACH_END(item);
+
+  return map;
+}
+
+/**
+ * Helper: add a single pubsub_cfg_t to an adjacency map.
+ **/
+static void
+pubsub_adjmap_add(pubsub_adjmap_t *map,
+                  const pubsub_cfg_t *item)
+{
+  smartlist_t **by_subsys;
+  smartlist_t **by_msg;
+
+  tor_assert(item->subsys < map->n_subsystems);
+  tor_assert(item->msg < map->n_msgs);
+
+  if (item->is_publish) {
+    by_subsys = &map->pub_by_subsys[item->subsys];
+    by_msg = &map->pub_by_msg[item->msg];
+  } else {
+    by_subsys = &map->sub_by_subsys[item->subsys];
+    by_msg = &map->sub_by_msg[item->msg];
+  }
+
+  if (! *by_subsys)
+    *by_subsys = smartlist_new();
+  if (! *by_msg)
+    *by_msg = smartlist_new();
+  smartlist_add(*by_subsys, (void*) item);
+  smartlist_add(*by_msg, (void *) item);
+}
+
+/**
+ * Release all storage held by m and set m to NULL.
+ **/
+#define pubsub_adjmap_free(m) \
+  FREE_AND_NULL(pubsub_adjmap_t, pubsub_adjmap_free_, m)
+
+/**
+ * Free every element of an <b>n</b>-element array of smartlists, then
+ * free the array itself.
+ **/
+static void
+pubsub_adjmap_free_helper(smartlist_t **lsts, size_t n)
+{
+  if (!lsts)
+    return;
+
+  for (unsigned i = 0; i < n; ++i) {
+    smartlist_free(lsts[i]);
+  }
+  tor_free(lsts);
+}
+
+/**
+ * Release all storage held by <b>map</b>.
+ **/
+static void
+pubsub_adjmap_free_(pubsub_adjmap_t *map)
+{
+  if (!map)
+    return;
+  pubsub_adjmap_free_helper(map->pub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->sub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->pub_by_msg, map->n_msgs);
+  pubsub_adjmap_free_helper(map->sub_by_msg, map->n_msgs);
+  tor_free(map);
+}
+
+/**
+ * Helper: return the length of <b>sl</b>, or 0 if sl is NULL.
+ **/
+static int
+smartlist_len_opt(const smartlist_t *sl)
+{
+  if (sl)
+    return smartlist_len(sl);
+  else
+    return 0;
+}
+
+/** Return a pointer to a statically allocated string encoding the
+ * dispatcher flags in <b>flags</b>. */
+static const char *
+format_flags(unsigned flags)
+{
+  static char buf[32];
+  buf[0] = 0;
+  if (flags & DISP_FLAG_EXCL) {
+    strlcat(buf, " EXCL", sizeof(buf));
+  }
+  if (flags & DISP_FLAG_STUB) {
+    strlcat(buf, " STUB", sizeof(buf));
+  }
+  return buf[0] ? buf+1 : buf;
+}
+
+/**
+ * Log a message containing a description of <b>cfg</b> at severity, prefixed
+ * by the string <b>prefix</b>.
+ */
+static void
+pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix)
+{
+  tor_assert(prefix);
+
+  tor_log(severity, LD_MESG,
+          "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]",
+          prefix,
+          get_subsys_id_name(cfg->subsys),
+          cfg->is_publish ? "PUB" : "SUB",
+          get_message_id_name(cfg->msg),
+          get_msg_type_id_name(cfg->type),
+          get_channel_id_name(cfg->channel),
+          format_flags(cfg->flags),
+          cfg->subsys, cfg->msg, cfg->type, cfg->channel, cfg->flags,
+          cfg->added_by_file, cfg->added_by_line);
+}
+
+/**
+ * Helper: fill a bitarray <b>out</b> with entries corresponding to the
+ * subsystems listed in <b>items</b>.
+ **/
+static void
+get_message_bitarray(const pubsub_adjmap_t *map,
+                     const smartlist_t *items,
+                     bitarray_t **out)
+{
+  *out = bitarray_init_zero((unsigned)map->n_subsystems);
+  if (! items)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) {
+    bitarray_set(*out, cfg->subsys);
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists obey our local graph topology rules.
+ *
+ * (Right now this is just a matter of "each subsystem only
+ * publishes/subscribes once; no subsystem is a publisher and subscriber for
+ * the same message.")
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+lint_message_graph(const pubsub_adjmap_t *map,
+                   message_id_t msg,
+                   const smartlist_t *pub,
+                   const smartlist_t *sub)
+{
+  bitarray_t *published_by = NULL;
+  bitarray_t *subscribed_by = NULL;
+  bool ok = true;
+
+  get_message_bitarray(map, pub, &published_by);
+  get_message_bitarray(map, sub, &subscribed_by);
+
+  /* Check whether any subsystem is publishing and subscribing the same
+   * message. [??]
+   */
+  for (unsigned i = 0; i < map->n_subsystems; ++i) {
+    if (bitarray_is_set(published_by, i) &&
+        bitarray_is_set(subscribed_by, i)) {
+      log_warn(LD_MESG|LD_BUG,
+               "Message \"%s\" is published and subscribed by the same "
+               "subsystem \"%s\".",
+               get_message_id_name(msg),
+               get_subsys_id_name(i));
+      ok = false;
+    }
+  }
+
+  bitarray_free(published_by);
+  bitarray_free(subscribed_by);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists have compatible flags, channels, and types.
+ **/
+static int
+lint_message_consistency(message_id_t msg,
+                         const smartlist_t *pub,
+                         const smartlist_t *sub)
+{
+  if (!smartlist_len_opt(pub) && !smartlist_len_opt(sub))
+    return 0; // LCOV_EXCL_LINE -- this was already checked.
+
+  /* The 'all' list has the publishers and the subscribers. */
+  smartlist_t *all = smartlist_new();
+  if (pub)
+    smartlist_add_all(all, pub);
+  if (sub)
+    smartlist_add_all(all, sub);
+
+  const pubsub_cfg_t *item0 = smartlist_get(all, 0);
+
+  /* Indicates which subsystems we've found publishing/subscribing here. */
+  bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true;
+
+  /* Simple message consistency properties across messages.
+   */
+  SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) {
+    chan_same &= (cfg->channel == item0->channel);
+    type_same &= (cfg->type == item0->type);
+    if (cfg->is_publish)
+      pub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+    else
+      sub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+  } SMARTLIST_FOREACH_END(cfg);
+
+  bool ok = true;
+
+  if (! chan_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "channels.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (! type_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "message types.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Enforce exclusive-ness for publishers and subscribers that have asked for
+   * it.
+   */
+  if (pub_excl && smartlist_len_opt(pub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple publishers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (sub_excl && smartlist_len_opt(sub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple subscribers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  smartlist_free(all);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check whether there are any errors or inconsistencies for the message
+ * described by <b>msg</b> in <b>map</b>.  If there are problems, log about
+ * them, and return -1.  Otherwise return 0.
+ **/
+static int
+lint_message(const pubsub_adjmap_t *map, message_id_t msg)
+{
+  /* NOTE: Some of the checks in this function are maybe over-zealous, and we
+   * might not want to have them forever.  I've marked them with [?] below.
+   */
+  if (BUG(msg >= map->n_msgs))
+    return 0; // LCOV_EXCL_LINE
+
+  const smartlist_t *pub = map->pub_by_msg[msg];
+  const smartlist_t *sub = map->sub_by_msg[msg];
+
+  const size_t n_pub = smartlist_len_opt(pub);
+  const size_t n_sub = smartlist_len_opt(sub);
+
+  if (n_pub == 0 && n_sub == 0) {
+    log_info(LD_MESG, "Nobody is publishing or subscribing to message "
+             "\"%s\".",
+             get_message_id_name(msg));
+    return 0; // No publishers or subscribers: nothing to do.
+  }
+  /* We'll set this to false if there are any problems. */
+  bool ok = true;
+
+  /* First make sure that if there are publishers, there are subscribers. */
+  if (n_pub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has subscribers, but no publishers.",
+             get_message_id_name(msg));
+    ok = false;
+  } else if (n_sub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has publishers, but no subscribers.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Check the message graph topology. */
+  if (lint_message_graph(map, msg, pub, sub) < 0)
+    ok = false;
+
+  /* Check whether the messages have the same fields set on them. */
+  if (lint_message_consistency(msg, pub, sub) < 0)
+    ok = false;
+
+  if (!ok) {
+    /* There was a problem -- let's log all the publishers and subscribers on
+     * this message */
+    if (pub) {
+      SMARTLIST_FOREACH(pub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+    if (sub) {
+      SMARTLIST_FOREACH(sub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+  }
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check all the messages in <b>map</b> for consistency.  Return 0 on success,
+ * -1 on problems.
+ **/
+static int
+pubsub_adjmap_check(const pubsub_adjmap_t *map)
+{
+  bool all_ok = true;
+  for (unsigned i = 0; i < map->n_msgs; ++i) {
+    if (lint_message(map, i) < 0) {
+      all_ok = false;
+    }
+  }
+  return all_ok ? 0 : -1;
+}
+
+/**
+ * Check builder for consistency and various constraints. Return 0 on success,
+ * -1 on failure.
+ **/
+int
+pubsub_builder_check(pubsub_builder_t *builder)
+{
+  pubsub_adjmap_t *map = pubsub_build_adjacency_map(builder->items);
+  int rv = -1;
+
+  if (!map)
+    goto err; // should be impossible
+
+  if (pubsub_adjmap_check(map) < 0)
+    goto err;
+
+  rv = 0;
+ err:
+  pubsub_adjmap_free(map);
+  return rv;
+}
diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h
new file mode 100644
index 0000000000..b0d6ae7e92
--- /dev/null
+++ b/src/lib/pubsub/pubsub_connect.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_connect.h
+ * @brief Header for functions that add relationships to a pubsub builder.
+ *
+ * These functions are used by modules that need to add publication and
+ * subscription requests.  Most users will want to call these functions
+ * indirectly, via the macros in pubsub_macros.h.
+ **/
+
+#ifndef TOR_PUBSUB_CONNECT_H
+#define TOR_PUBSUB_CONNECT_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct pub_binding_t;
+/**
+ * A "dispatch connector" is a view of the dispatcher that a subsystem
+ * uses while initializing itself.  It is specific to the subsystem, and
+ * ensures that each subsystem doesn't need to identify itself
+ * repeatedly while registering its messages.
+ **/
+typedef struct pubsub_connector_t pubsub_connector_t;
+
+int pubsub_add_pub_(struct pubsub_connector_t *con,
+                    struct pub_binding_t *out,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_add_sub_(struct pubsub_connector_t *con,
+                    recv_fn_t recv_fn,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_connector_register_type_(struct pubsub_connector_t *,
+                                    msg_type_id_t,
+                                    dispatch_typefns_t *,
+                                    const char *file,
+                                    unsigned line);
+
+#endif /* !defined(TOR_PUBSUB_CONNECT_H) */
diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h
new file mode 100644
index 0000000000..9912c1ae89
--- /dev/null
+++ b/src/lib/pubsub/pubsub_flags.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_flags.h
+ * @brief Flags that can be set on publish/subscribe messages.
+ **/
+
+#ifndef TOR_PUBSUB_FLAGS_H
+#define TOR_PUBSUB_FLAGS_H
+
+/**
+ * Flag for registering a message: declare that no other module is allowed to
+ * publish this message if we are publishing it, or subscribe to it if we are
+ * subscribing to it.
+ */
+#define DISP_FLAG_EXCL (1u<<0)
+
+/**
+ * Flag for registering a message: declare that this message is a stub, and we
+ * will not actually publish/subscribe it, but that the dispatcher should
+ * treat us as if we did when typechecking.
+ *
+ * We use this so that messages aren't treated as "dangling" if they are
+ * potentially used by some other build of Tor.
+ */
+#define DISP_FLAG_STUB (1u<<1)
+
+#endif /* !defined(TOR_PUBSUB_FLAGS_H) */
diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h
new file mode 100644
index 0000000000..e5ffbe501a
--- /dev/null
+++ b/src/lib/pubsub/pubsub_macros.h
@@ -0,0 +1,373 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pubsub_macros.h
+ * \brief Macros to help with the publish/subscribe dispatch API.
+ *
+ * The dispatch API allows different subsystems of Tor to communicate with
+ * another asynchronously via a shared "message" system.  Some subsystems
+ * declare that they publish a given message, and others declare that they
+ * subscribe to it.  Both subsystems depend on the message, but not upon one
+ * another.
+ *
+ * To declare a message, use DECLARE_MESSAGE() (for messages that take their
+ * data as a pointer) or DECLARE_MESSAGE_INT() (for messages that take their
+ * data as an integer.  For example, you might say
+ *
+ *     DECLARE_MESSAGE(new_circuit, circ, circuit_handle_t *);
+ * or
+ *     DECLARE_MESSAGE_INT(shutdown_requested, boolean, bool);
+ *
+ * Every message has a unique name, a "type name" that the dispatch system
+ * uses to manage associated data, and a C type name.  You'll want to put
+ * these declarations in a header, to be included by all publishers and all
+ * subscribers.
+ *
+ * When a subsystem wants to publish a message, it uses DECLARE_PUBLISH() at
+ * file scope to create necessary static functions.  Then, in its subsystem
+ * initialization (in the "bind to dispatcher" callback) (TODO: name this
+ * properly!), it calls DISPATCH_ADD_PUB() to tell the dispatcher about its
+ * intent to publish.  When it actually wants to publish, it uses the
+ * PUBLISH() macro.  For example:
+ *
+ *     // At file scope
+ *     DECLARE_PUBLISH(shutdown_requested);
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_PUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ *     // somewhere in a function
+ *        {
+ *            PUBLISH(shutdown_requested, true);
+ *        }
+ *
+ * When a subsystem wants to subscribe to a message, it uses
+ * DECLARE_SUBSCRIBE() at file scope to declare static functions.  It must
+ * declare a hook function that receives the message type.  Then, in its "bind
+ * to dispatcher" function, it calls DISPATCHER_ADD_SUB() to tell the
+ * dispatcher about its intent to subscribe.  When another module publishes
+ * the message, the dispatcher will call the provided hook function.
+ *
+ *     // At file scope.  The first argument is the message that you're
+ *     // subscribing to; the second argument is the hook function to declare.
+ *     DECLARE_SUBSCRIBE(shutdown_requested, on_shutdown_req_cb);
+ *
+ *     // You need to declare this function.
+ *     static void on_shutdown_req_cb(const msg_t *msg,
+ *                                    bool value)
+ *     {
+ *         // (do something here.)
+ *     }
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_SUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ * Where did these types come from?  Somewhere in the code, you need to call
+ * DISPATCH_REGISTER_TYPE() to make sure that the dispatcher can manage the
+ * message auxiliary data.  It associates a vtbl-like structure with the
+ * type name, so that the dispatcher knows how to manipulate the type you're
+ * giving it.
+ *
+ * For example, the "boolean" type we're using above could be defined as:
+ *
+ *    static char *boolean_fmt(msg_aux_data_t d)
+ *    {
+ *        // This is used for debugging and dumping messages.
+ *        if (d.u64)
+ *            return tor_strdup("true");
+ *        else
+ *            return tor_strdup("false");
+ *    }
+ *
+ *    static void boolean_free(msg_aux_data_t d)
+ *    {
+ *        // We don't actually need to do anything to free a boolean.
+ *        // We could use "NULL" instead of this function, but I'm including
+ *        // it as an example.
+ *    }
+ *
+ *    static void bind_to_dispatcher(pubsub_connector_t *con)
+ *    {
+ *        dispatch_typefns_t boolean_fns = {
+ *            .fmt_fn = boolean_fmt,
+ *            .free_fn = boolean_free,
+ *        };
+ *        DISPATCH_REGISTER_TYPE(con, boolean, &boolean_fns);
+ *    }
+ *
+ *
+ *
+ * So, how does this all work?  (You can stop reading here, unless you're
+ * debugging something.)
+ *
+ * When you declare a message in a header with DECLARE_MESSAGE() or
+ * DECLARE_MESSAGE_INT(), it creates five things:
+ *
+ *    * two typedefs for the message argument (constant and non-constant
+ *      variants).
+ *    * a constant string to hold the declared message type name
+ *    * two inline functions, to coerce the message argument type to and from
+ *      a "msg_aux_data_t" union.
+ *
+ * All of these declarations have names based on the message name.
+ *
+ * Later, when you say DECLARE_PUBLISH() or DECLARE_SUBSCRIBE(), we use the
+ * elements defined by DECLARE_MESSAGE() to make sure that the publish
+ * function takes the correct argument type, and that the subscription hook is
+ * declared with the right argument type.
+ **/
+
+#ifndef TOR_DISPATCH_MSG_H
+#define TOR_DISPATCH_MSG_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+/* Implemenation notes:
+ *
+ * For a messagename "foo", the DECLARE_MESSAGE*() macros must declare:
+ *
+ * msg_arg_type__foo -- a typedef for the argument type of the foo message.
+ * msg_arg_consttype__foo -- a typedef for the const argument type of the
+ *     foo message.
+ * msg_arg_name__foo[] -- a static string constant holding the unique
+ *      identifier for the type of the foo message.
+ * msg_arg_get__foo() -- an inline function taking a msg_aux_data_t and
+ *      returning the C data type.
+ * msg_arg_set__foo() -- an inline function taking a msg_aux_data_t and
+ *      the C type, setting the msg_aux_data_t to hold the C type.
+ *
+ * For a messagename "foo", the DECLARE_PUBLISH() macro must declare:
+ *
+ * pub_binding__foo -- A static pub_binding_t object used to send messages
+ *      from this module.
+ * publish_fn__foo -- A function taking an argument of the appropriate
+ *      C type, to be invoked by PUBLISH().
+ *
+ * For a messagename "foo", the DECLARE_SUBSCRIBE() macro must declare:
+ *
+ * hookfn -- A user-provided function name, with the correct signature.
+ * recv_fn__foo -- A wrapper callback that takes a msg_t *, and calls
+ *      hookfn with the appropriate arguments.
+ */
+
+/** Macro to declare common elements shared by DECLARE_MESSAGE and
+ * DECLARE_MESSAGE_INT.  Don't call this directly.
+ *
+ * Note that the "msg_arg_name" string constant is defined in each
+ * translation unit.  This might be undesirable; we can tweak it in the
+ * future if need be.
+ */
+#define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)         \
+  typedef c_type msg_arg_type__ ##messagename;                          \
+  typedef const c_type msg_arg_consttype__ ##messagename;               \
+  ATTR_UNUSED static const char msg_arg_name__ ##messagename[] = # typename;
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking a pointer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.
+ * It needs to be defined somewhere in Tor, using
+ * "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_ptr_type" is a C pointer type (like "char *" or "struct foo *").
+ * The "*" needs to be included.
+ */
+#define DECLARE_MESSAGE(messagename, typename, c_ptr_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_ptr_type)           \
+  ATTR_UNUSED static inline c_ptr_type                                  \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return m.ptr;                                                       \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_ptr_type v)          \
+  {                                                                     \
+    m->ptr = v;                                                         \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking an integer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.  It
+ * needs to be defined somewhere in Tor, using "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_type" is a C integer type, like "int" or "bool".  It needs to fit inside
+ * a uint64_t.
+ */
+#define DECLARE_MESSAGE_INT(messagename, typename, c_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)               \
+  ATTR_UNUSED static inline c_type                                      \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return (c_type)m.u64;                                               \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v)              \
+  {                                                                     \
+    m->u64 = (uint64_t)v;                                               \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C module declare that we'll be publishing a given
+ * message type from within this module.
+ *
+ * It creates necessary functions and wrappers to publish a message whose
+ * unique identifier is "messagename".
+ *
+ * Before you use this, you need to include the header where DECLARE_MESSAGE*()
+ * was used for this message.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_PUBLISH(messagename)                                    \
+  static pub_binding_t pub_binding__ ##messagename;                     \
+  static void                                                           \
+  publish_fn__ ##messagename(msg_arg_type__ ##messagename arg)          \
+  {                                                                     \
+    msg_aux_data_t data;                                                \
+    msg_arg_set__ ##messagename(&data, arg);                            \
+    pubsub_pub_(&pub_binding__ ##messagename, data);                    \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C file to declare that we're subscribing to a
+ * given message and associating it with a given "hook function".  It
+ * declares the hook function static, and helps with strong typing.
+ *
+ * Before you use this, you need to include the header where
+ * DECLARE_MESSAGE*() was used for the message whose unique identifier is
+ * "messagename".
+ *
+ * You will need to define a function with the name that you provide for
+ * "hookfn".  The type of this function will be:
+ *     static void hookfn(const msg_t *, const c_type)
+ * where c_type is the c type that you declared in the header.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_SUBSCRIBE(messagename, hookfn) \
+  static void hookfn(const msg_t *,                             \
+                     const msg_arg_consttype__ ##messagename);  \
+  static void recv_fn__ ## messagename(const msg_t *m)          \
+  {                                                             \
+    msg_arg_type__ ## messagename arg;                          \
+    arg = msg_arg_get__ ##messagename(m->aux_data__);           \
+    hookfn(m, arg);                                             \
+  }                                                             \
+  EAT_SEMICOLON
+
+/**
+ * Add a fake use of the publish function for 'messagename', so that
+ * the compiler does not call it unused.
+ */
+#define DISPATCH__FAKE_USE_OF_PUBFN_(messagename)                       \
+  ( 0 ? (publish_fn__ ##messagename((msg_arg_type__##messagename)0), 1) \
+    : 1)
+
+/**
+ * This macro is for internal use.  It backs DISPATCH_ADD_PUB*()
+ */
+#define DISPATCH_ADD_PUB_(connector, channel, messagename, flags)       \
+  (                                                                     \
+    DISPATCH__FAKE_USE_OF_PUBFN_(messagename),                          \
+    pubsub_add_pub_((connector),                                        \
+                      &pub_binding__ ##messagename,                     \
+                      get_channel_id(# channel),                        \
+                    get_message_id(# messagename),                      \
+                      get_msg_type_id(msg_arg_name__ ## messagename),   \
+                      (flags),                                          \
+                      __FILE__,                                         \
+                      __LINE__)                                         \
+    )
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB(connector, channel, messagename)       \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, 0)
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type, and that no other subsystem is allowed to.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/**
+ * This macro is for internal use. It backs DISPATCH_ADD_SUB*()
+ */
+#define DISPATCH_ADD_SUB_(connector, channel, messagename, flags)       \
+  pubsub_add_sub_((connector),                                          \
+                    recv_fn__ ##messagename,                            \
+                    get_channel_id(#channel),                           \
+                    get_message_id(# messagename),                      \
+                    get_msg_type_id(msg_arg_name__ ##messagename),      \
+                    (flags),                                            \
+                    __FILE__,                                           \
+                    __LINE__)
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB(connector, channel, messagename)       \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, 0)
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type, and that no other subsystem is allowed to do
+ * so.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/**
+ * Publish a given message with a given argument.  (Takes ownership of the
+ * argument if it is a pointer.)
+ */
+#define PUBLISH(messagename, arg)               \
+  publish_fn__ ##messagename(arg)
+
+/**
+ * Use a given connector to declare that the functions to be used to manipuate
+ * a certain C type.
+ **/
+#define DISPATCH_REGISTER_TYPE(con, type, fns)                  \
+  pubsub_connector_register_type_((con),                        \
+                                  get_msg_type_id(#type),       \
+                                  (fns),                        \
+                                  __FILE__,                     \
+                                  __LINE__)
+
+#endif /* !defined(TOR_DISPATCH_MSG_H) */
diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c
new file mode 100644
index 0000000000..84c7dae02c
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.c
@@ -0,0 +1,72 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_publish.c
+ * @brief Header for functions to publish using a pub_binding_t.
+ **/
+
+#define PUBSUB_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Publish a message from the publication binding <b>pub</b> using the
+ * auxiliary data <b>auxdata</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata)
+{
+  dispatch_t *d = pub->dispatch_ptr;
+  if (BUG(! d)) {
+    /* Tried to publish a message before the dispatcher was configured. */
+    /* (Without a dispatcher, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.type >= d->n_types)) {
+    /* The type associated with this message is not known to the dispatcher. */
+    /* (Without a correct type, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.msg >= d->n_msgs) ||
+      BUG(pub->msg_template.channel >= d->n_queues)) {
+    /* The message ID or channel ID was out of bounds. */
+    // LCOV_EXCL_START
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return -1;
+    // LCOV_EXCL_STOP
+  }
+
+  if (! d->table[pub->msg_template.msg]) {
+    /* Fast path: nobody wants this data. */
+
+    // XXXX Faster path: we could store this in the pub_binding_t.
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return 0;
+  }
+
+  /* Construct the message object */
+  msg_t *m = tor_malloc(sizeof(msg_t));
+  memcpy(m, &pub->msg_template, sizeof(msg_t));
+  m->aux_data__ = auxdata;
+
+  return dispatch_send_msg_unchecked(d, m);
+}
diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h
new file mode 100644
index 0000000000..d9d6fa9ba5
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_publish.h
+ * @brief Header for pubsub_publish.c
+ **/
+
+#ifndef TOR_PUBSUB_PUBLISH_H
+#define TOR_PUBSUB_PUBLISH_H
+
+#include "lib/dispatch/msgtypes.h"
+struct pub_binding_t;
+
+int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata);
+
+#endif /* !defined(TOR_PUBSUB_PUBLISH_H) */
diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include
index 84906dfb3d..853dae7880 100644
--- a/src/lib/sandbox/.may_include
+++ b/src/lib/sandbox/.may_include
@@ -5,11 +5,10 @@ lib/container/*.h
 lib/err/*.h
 lib/log/*.h
 lib/malloc/*.h
-lib/net/*.h
 lib/sandbox/*.h
 lib/sandbox/*.inc
 lib/string/*.h
 
-ht.h
-siphash.h
-tor_queue.h
+ext/ht.h
+ext/siphash.h
+ext/tor_queue.h
diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am
index adfda6bde5..e81f14b55f 100644
--- a/src/lib/sandbox/include.am
+++ b/src/lib/sandbox/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_sandbox_a_SOURCES =			\
 	src/lib/sandbox/sandbox.c
 
@@ -13,6 +14,7 @@ src_lib_libtor_sandbox_testing_a_SOURCES = \
 src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/sandbox/linux_syscalls.inc		\
 	src/lib/sandbox/sandbox.h
diff --git a/src/lib/sandbox/lib_sandbox.md b/src/lib/sandbox/lib_sandbox.md
new file mode 100644
index 0000000000..dd168c9b13
--- /dev/null
+++ b/src/lib/sandbox/lib_sandbox.md
@@ -0,0 +1,15 @@
+@dir /lib/sandbox
+@brief lib/sandbox: Linux seccomp2-based sandbox.
+
+This module uses Linux's seccomp2 facility via the
+[`libseccomp` library](https://github.com/seccomp/libseccomp), to restrict
+the set of system calls that Tor is allowed to invoke while it is running.
+
+Because there are many libc versions that invoke different system calls, and
+because handling strings is quite complex, this module is more complex and
+less portable than it needs to be.
+
+A better architecture would put the responsibility for invoking tricky system
+calls (like open()) in another, less restricted process, and give that
+process responsibility for enforcing our sandbox rules.
+
diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c
index 8f577b0660..d4f0da8397 100644
--- a/src/lib/sandbox/sandbox.c
+++ b/src/lib/sandbox/sandbox.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,13 +38,12 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "lib/cc/torint.h"
-#include "lib/net/resolve.h"
 #include "lib/malloc/malloc.h"
 #include "lib/string/scanf.h"
 
-#include "tor_queue.h"
-#include "ht.h"
-#include "siphash.h"
+#include "ext/tor_queue.h"
+#include "ext/ht.h"
+#include "ext/siphash.h"
 
 #define DEBUGGING_CLOSE
 
@@ -83,7 +82,7 @@
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
 #define USE_BACKTRACE
-#define EXPOSE_CLEAN_BACKTRACE
+#define BACKTRACE_PRIVATE
 #include "lib/err/backtrace.h"
 #endif /* defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && ... */
 
@@ -118,6 +117,10 @@
 
 #endif /* defined(__i386__) || ... */
 
+#ifdef M_SYSCALL
+#define SYSCALL_NAME_DEBUGGING
+#endif
+
 /**Determines if at least one sandbox is active.*/
 static int sandbox_active = 0;
 /** Holds the parameter list configuration for the sandbox.*/
@@ -134,6 +137,10 @@ static sandbox_cfg_t *filter_dynamic = NULL;
  * the high bits of the value might get masked out improperly. */
 #define SCMP_CMP_MASKED(a,b,c) \
   SCMP_CMP4((a), SCMP_CMP_MASKED_EQ, ~(scmp_datum_t)(b), (c))
+/* For negative constants, the rule to add depends on the glibc version. */
+#define SCMP_CMP_NEG(a,op,b) (libc_negative_constant_needs_cast() ? \
+                              (SCMP_CMP((a), (op), (unsigned int)(b))) : \
+                              (SCMP_CMP_STR((a), (op), (b))))
 
 /** Variable used for storing all syscall numbers that will be allowed with the
  * stage 1 general Tor sandbox.
@@ -144,6 +151,7 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(clock_gettime),
     SCMP_SYS(close),
     SCMP_SYS(clone),
+    SCMP_SYS(dup),
     SCMP_SYS(epoll_create),
     SCMP_SYS(epoll_wait),
 #ifdef __NR_epoll_pwait
@@ -166,6 +174,7 @@ static int filter_nopar_gen[] = {
 #ifdef __NR_fstat64
     SCMP_SYS(fstat64),
 #endif
+    SCMP_SYS(fsync),
     SCMP_SYS(futex),
     SCMP_SYS(getdents),
     SCMP_SYS(getdents64),
@@ -265,13 +274,27 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(listen),
     SCMP_SYS(connect),
     SCMP_SYS(getsockname),
+#ifdef ENABLE_NSS
+#ifdef __NR_getpeername
+    SCMP_SYS(getpeername),
+#endif
+#endif
     SCMP_SYS(recvmsg),
     SCMP_SYS(recvfrom),
     SCMP_SYS(sendto),
     SCMP_SYS(unlink),
+#ifdef __NR_unlinkat
+    SCMP_SYS(unlinkat),
+#endif
     SCMP_SYS(poll)
 };
 
+/* opendir is not a syscall but it will use either open or openat. We do not
+ * want the decision to allow open/openat to be the callers reponsability, so
+ * we create a phony syscall number for opendir and sb_opendir will choose the
+ * correct syscall. */
+#define PHONY_OPENDIR_SYSCALL -2
+
 /* These macros help avoid the error where the number of filters we add on a
  * single rule don't match the arg_cnt param. */
 #define seccomp_rule_add_0(ctx,act,call) \
@@ -295,6 +318,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   unsigned i;
   int rc;
   int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD,
+                  SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, SIGIO,
 #ifdef SIGXFSZ
       SIGXFSZ
 #endif
@@ -424,31 +448,59 @@ sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 #endif
 #endif
 
-/* Return true if we think we're running with a libc that always uses
- * openat on linux. */
+/* Return true the libc version is greater or equal than
+ * <b>major</b>.<b>minor</b>. Returns false otherwise. */
 static int
-libc_uses_openat_for_everything(void)
+is_libc_at_least(int major, int minor)
 {
 #ifdef CHECK_LIBC_VERSION
   const char *version = gnu_get_libc_version();
   if (version == NULL)
     return 0;
 
-  int major = -1;
-  int minor = -1;
+  int libc_major = -1;
+  int libc_minor = -1;
 
-  tor_sscanf(version, "%d.%d", &major, &minor);
-  if (major >= 3)
+  tor_sscanf(version, "%d.%d", &libc_major, &libc_minor);
+  if (libc_major > major)
     return 1;
-  else if (major == 2 && minor >= 26)
+  else if (libc_major == major && libc_minor >= minor)
     return 1;
   else
     return 0;
-#else /* !(defined(CHECK_LIBC_VERSION)) */
+#else /* !defined(CHECK_LIBC_VERSION) */
+  (void)major;
+  (void)minor;
   return 0;
 #endif /* defined(CHECK_LIBC_VERSION) */
 }
 
+/* Return true if we think we're running with a libc that uses openat for the
+ * open function on linux. */
+static int
+libc_uses_openat_for_open(void)
+{
+  return is_libc_at_least(2, 26);
+}
+
+/* Return true if we think we're running with a libc that uses openat for the
+ * opendir function on linux. */
+static int
+libc_uses_openat_for_opendir(void)
+{
+  // libc 2.27 and above or between 2.15 (inclusive) and 2.22 (exclusive)
+  return is_libc_at_least(2, 27) ||
+         (is_libc_at_least(2, 15) && !is_libc_at_least(2, 22));
+}
+
+/* Return true if we think we're running with a libc that needs to cast
+ * negative arguments like AT_FDCWD for seccomp rules. */
+static int
+libc_negative_constant_needs_cast(void)
+{
+  return is_libc_at_least(2, 27);
+}
+
 /** Allow a single file to be opened.  If <b>use_openat</b> is true,
  * we're using a libc that remaps all the opens into openats. */
 static int
@@ -456,7 +508,7 @@ allow_file_open(scmp_filter_ctx ctx, int use_openat, const char *file)
 {
   if (use_openat) {
     return seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(openat),
-                              SCMP_CMP(0, SCMP_CMP_EQ, (unsigned int)AT_FDCWD),
+                              SCMP_CMP_NEG(0, SCMP_CMP_EQ, AT_FDCWD),
                               SCMP_CMP_STR(1, SCMP_CMP_EQ, file));
   } else {
     return seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(open),
@@ -474,7 +526,7 @@ sb_open(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   int rc;
   sandbox_cfg_t *elem = NULL;
 
-  int use_openat = libc_uses_openat_for_everything();
+  int use_openat = libc_uses_openat_for_open();
 
   // for each dynamic parameter filters
   for (elem = filter; elem != NULL; elem = elem->next) {
@@ -592,7 +644,7 @@ sb_openat(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
     if (param != NULL && param->prot == 1 && param->syscall
         == SCMP_SYS(openat)) {
       rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(openat),
-          SCMP_CMP(0, SCMP_CMP_EQ, AT_FDCWD),
+          SCMP_CMP_NEG(0, SCMP_CMP_EQ, AT_FDCWD),
           SCMP_CMP_STR(1, SCMP_CMP_EQ, param->value),
           SCMP_CMP(2, SCMP_CMP_EQ, O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_DIRECTORY|
               O_CLOEXEC));
@@ -607,6 +659,30 @@ sb_openat(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   return 0;
 }
 
+static int
+sb_opendir(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
+{
+  int rc;
+  sandbox_cfg_t *elem = NULL;
+
+  // for each dynamic parameter filters
+  for (elem = filter; elem != NULL; elem = elem->next) {
+    smp_param_t *param = elem->param;
+
+    if (param != NULL && param->prot == 1 && param->syscall
+        == PHONY_OPENDIR_SYSCALL) {
+      rc = allow_file_open(ctx, libc_uses_openat_for_opendir(), param->value);
+      if (rc != 0) {
+        log_err(LD_BUG,"(Sandbox) failed to add openat syscall, received "
+            "libseccomp error %d", rc);
+        return rc;
+      }
+    }
+  }
+
+  return 0;
+}
+
 /**
  * Function responsible for setting up the socket syscall for
  * the seccomp filter sandbox.
@@ -647,6 +723,15 @@ sb_socket(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
     }
   }
 
+#ifdef ENABLE_NSS
+  rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket),
+    SCMP_CMP(0, SCMP_CMP_EQ, PF_INET),
+    SCMP_CMP(1, SCMP_CMP_EQ, SOCK_STREAM),
+    SCMP_CMP(2, SCMP_CMP_EQ, IPPROTO_IP));
+  if (rc)
+    return rc;
+#endif
+
   rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket),
       SCMP_CMP(0, SCMP_CMP_EQ, PF_UNIX),
       SCMP_CMP_MASKED(1, SOCK_CLOEXEC|SOCK_NONBLOCK, SOCK_STREAM),
@@ -798,6 +883,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   if (rc)
     return rc;
 
+  rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
+      SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
+      SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN));
+  if (rc)
+    return rc;
+
 #ifdef HAVE_SYSTEMD
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
       SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
@@ -1106,6 +1197,7 @@ static sandbox_filter_func_t filter_func[] = {
     sb_chmod,
     sb_open,
     sb_openat,
+    sb_opendir,
     sb_rename,
 #ifdef __NR_fcntl64
     sb_fcntl64,
@@ -1425,6 +1517,19 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   return 0;
 }
 
+int
+sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir)
+{
+  sandbox_cfg_t *elem = NULL;
+
+  elem = new_element(PHONY_OPENDIR_SYSCALL, dir);
+
+  elem->next = *cfg;
+  *cfg = elem;
+
+  return 0;
+}
+
 /**
  * Function responsible for going through the parameter syscall filters and
  * call each function pointer in the list.
@@ -1517,15 +1622,16 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // marking the sandbox as active
   sandbox_active = 1;
-  tor_make_getaddrinfo_cache_active();
 
  end:
   seccomp_release(ctx);
   return (rc < 0 ? -rc : rc);
 }
 
+#ifdef SYSCALL_NAME_DEBUGGING
 #include "lib/sandbox/linux_syscalls.inc"
 
+/** Return a string containing the name of a given syscall (if we know it) */
 static const char *
 get_syscall_name(int syscall_num)
 {
@@ -1543,6 +1649,28 @@ get_syscall_name(int syscall_num)
   }
 }
 
+/** Return the syscall number from a ucontext_t that we got in a signal
+ * handler (if we know how to do that). */
+static int
+get_syscall_from_ucontext(const ucontext_t *ctx)
+{
+  return (int) ctx->uc_mcontext.M_SYSCALL;
+}
+#else
+static const char *
+get_syscall_name(int syscall_num)
+{
+  (void) syscall_num;
+  return "unknown";
+}
+static int
+get_syscall_from_ucontext(const ucontext_t *ctx)
+{
+  (void) ctx;
+  return -1;
+}
+#endif
+
 #ifdef USE_BACKTRACE
 #define MAX_DEPTH 256
 static void *syscall_cb_buf[MAX_DEPTH];
@@ -1558,7 +1686,6 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
 {
   ucontext_t *ctx = (ucontext_t *) (void_context);
   const char *syscall_name;
-  int syscall;
 #ifdef USE_BACKTRACE
   size_t depth;
   int n_fds, i;
@@ -1573,7 +1700,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
   if (!ctx)
     return;
 
-  syscall = (int) ctx->uc_mcontext.M_SYSCALL;
+  int syscall = get_syscall_from_ucontext(ctx);
 
 #ifdef USE_BACKTRACE
   depth = backtrace(syscall_cb_buf, MAX_DEPTH);
@@ -1731,6 +1858,13 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
 }
 
 int
+sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir)
+{
+  (void)cfg; (void)dir;
+  return 0;
+}
+
+int
 sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file)
 {
   (void)cfg; (void)file;
@@ -1764,9 +1898,4 @@ sandbox_is_active(void)
   return 0;
 }
 
-void
-sandbox_disable_getaddrinfo_cache(void)
-{
-}
-
 #endif /* !defined(USE_LIBSECCOMP) */
diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h
index 5bec09a36a..a2b3227b90 100644
--- a/src/lib/sandbox/sandbox.h
+++ b/src/lib/sandbox/sandbox.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,10 +29,10 @@
 #define USE_LIBSECCOMP
 #endif
 
-struct sandbox_cfg_elem;
+struct sandbox_cfg_elem_t;
 
 /** Typedef to structure used to manage a sandbox configuration. */
-typedef struct sandbox_cfg_elem sandbox_cfg_t;
+typedef struct sandbox_cfg_elem_t sandbox_cfg_t;
 
 /**
  * Linux definitions
@@ -58,7 +58,7 @@ typedef enum {
  *  Configuration parameter structure associated with the LIBSECCOMP2
  *  implementation.
  */
-typedef struct smp_param {
+typedef struct smp_param_t {
   /** syscall associated with parameter. */
   int syscall;
 
@@ -77,7 +77,7 @@ typedef struct smp_param {
  * It is implemented as a linked list of parameters. Currently only controls
  * parameters for open, openat, execve, stat64.
  */
-struct sandbox_cfg_elem {
+struct sandbox_cfg_elem_t {
   /** Sandbox implementation which dictates the parameter type. */
   SB_IMPL implem;
 
@@ -85,7 +85,7 @@ struct sandbox_cfg_elem {
   smp_param_t *param;
 
   /** Next element of the configuration*/
-  struct sandbox_cfg_elem *next;
+  struct sandbox_cfg_elem_t *next;
 };
 
 /** Function pointer defining the prototype of a filter function.*/
@@ -108,7 +108,7 @@ typedef struct {
  * it matches the parameter.
  */
 const char* sandbox_intern_string(const char *param);
-#else /* !(defined(USE_LIBSECCOMP)) */
+#else /* !defined(USE_LIBSECCOMP) */
 #define sandbox_intern_string(s) (s)
 #endif /* defined(USE_LIBSECCOMP) */
 
@@ -136,6 +136,13 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2);
 int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file);
 
 /**
+ * Function used to add a opendir allowed filename to a supplied configuration.
+ * The (char*) specifies the path to the allowed dir; we steal the pointer to
+ * that dir.
+ */
+int sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir);
+
+/**
  * Function used to add a stat/stat64 allowed filename to a configuration.
  * The (char*) specifies the path to the allowed file; that pointer is stolen.
  */
diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include
index a8507761a4..2f0c8d341e 100644
--- a/src/lib/smartlist_core/.may_include
+++ b/src/lib/smartlist_core/.may_include
@@ -4,4 +4,4 @@ lib/malloc/*.h
 lib/err/*.h
 lib/string/*.h
 lib/smartlist_core/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am
index 99d65f0b23..548179bc4f 100644
--- a/src/lib/smartlist_core/include.am
+++ b/src/lib/smartlist_core/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_smartlist_core_a_SOURCES =			\
 	src/lib/smartlist_core/smartlist_core.c                 \
 	src/lib/smartlist_core/smartlist_split.c
@@ -15,6 +16,7 @@ src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \
 	$(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/smartlist_core/smartlist_core.h         \
 	src/lib/smartlist_core/smartlist_foreach.h      \
diff --git a/src/lib/smartlist_core/lib_smartlist_core.md b/src/lib/smartlist_core/lib_smartlist_core.md
new file mode 100644
index 0000000000..c031dd6f24
--- /dev/null
+++ b/src/lib/smartlist_core/lib_smartlist_core.md
@@ -0,0 +1,10 @@
+@dir /lib/smartlist_core
+@brief lib/smartlist_core: Minimal dynamic array implementation
+
+A `smartlist_t` is a dynamic array type for holding `void *`.  We use it
+throughout the rest of the codebase.
+
+There are higher-level pieces in \refdir{lib/container} but
+the ones in lib/smartlist_core are used by the logging code, and therefore
+cannot use the logging code.
+
diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c
index ac85a6cc84..571d17aa5d 100644
--- a/src/lib/smartlist_core/smartlist_core.c
+++ b/src/lib/smartlist_core/smartlist_core.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -88,6 +88,30 @@ smartlist_ensure_capacity(smartlist_t *sl, size_t size)
 #undef MAX_CAPACITY
 }
 
+/** Expand <b>sl</b> so that its length is at least <b>new_size</b>,
+ * filling in previously unused entries with NULL>
+ *
+ * Do nothing if <b>sl</b> already had at least <b>new_size</b> elements.
+ */
+void
+smartlist_grow(smartlist_t *sl, size_t new_size)
+{
+  smartlist_ensure_capacity(sl, new_size);
+
+  if (new_size > (size_t)sl->num_used) {
+    /* This memset() should be a no-op: everything else in the smartlist code
+     * tries to make sure that unused entries are always NULL.  Still, that is
+     * meant as a safety mechanism, so let's clear the memory here.
+     */
+    memset(sl->list + sl->num_used, 0,
+           sizeof(void *) * (new_size - sl->num_used));
+
+    /* This cast is safe, since we already asserted that we were below
+     * MAX_CAPACITY in smartlist_ensure_capacity(). */
+    sl->num_used = (int)new_size;
+  }
+}
+
 /** Append element to the end of the list. */
 void
 smartlist_add(smartlist_t *sl, void *element)
@@ -153,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element)
       sl->list[i++] = sl->list[j];
     }
   }
+  memset(sl->list + sl->num_used, 0,
+         sizeof(void *) * (num_used_orig - sl->num_used));
 }
 
 /** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h
index a7fbaa099b..de6fe69d3a 100644
--- a/src/lib/smartlist_core/smartlist_core.h
+++ b/src/lib/smartlist_core/smartlist_core.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -43,6 +43,7 @@ void smartlist_clear(smartlist_t *sl);
 void smartlist_add(smartlist_t *sl, void *element);
 void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
 void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
+void smartlist_grow(smartlist_t *sl, size_t new_size);
 
 void smartlist_remove(smartlist_t *sl, const void *element);
 void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
@@ -76,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
   raw_assert(sl->num_used > idx);
   sl->list[idx] = val;
 }
-#else /* !(defined(DEBUG_SMARTLIST)) */
+#else /* !defined(DEBUG_SMARTLIST) */
 #define smartlist_len(sl) ((sl)->num_used)
 #define smartlist_get(sl, idx) ((sl)->list[idx])
 #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
@@ -97,4 +98,4 @@ void smartlist_del(smartlist_t *sl, int idx);
 void smartlist_del_keeporder(smartlist_t *sl, int idx);
 void smartlist_insert(smartlist_t *sl, int idx, void *val);
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_CORE_H) */
diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h
index 0f6fe30074..03edb80f05 100644
--- a/src/lib/smartlist_core/smartlist_foreach.h
+++ b/src/lib/smartlist_core/smartlist_foreach.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -83,6 +83,19 @@
          ++var ## _sl_idx) {                                    \
       var = (sl)->list[var ## _sl_idx];
 
+/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to
+ *  SMARTLIST_FOREACH_BEGIN
+ *
+ * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT.
+ */
+#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var)  \
+  STMT_BEGIN                                                       \
+    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;             \
+    type var;                                                      \
+    for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0;   \
+         --var ## _sl_idx) {                                       \
+      var = (sl)->list[var ## _sl_idx];
+
 #define SMARTLIST_FOREACH_END(var)              \
     var = NULL;                                 \
     (void) var ## _sl_idx;                      \
diff --git a/src/lib/smartlist_core/smartlist_split.c b/src/lib/smartlist_core/smartlist_split.c
index c9cf59851f..b76b87406d 100644
--- a/src/lib/smartlist_core/smartlist_split.c
+++ b/src/lib/smartlist_core/smartlist_split.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h
index 4f72376125..fc964201e9 100644
--- a/src/lib/smartlist_core/smartlist_split.h
+++ b/src/lib/smartlist_core/smartlist_split.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@
 int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
                            int flags, int max);
 
-#endif
+#endif /* !defined(TOR_SMARTLIST_SPLIT_H) */
diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include
index ec5c769831..1fb9127f19 100644
--- a/src/lib/string/.may_include
+++ b/src/lib/string/.may_include
@@ -6,5 +6,5 @@ lib/malloc/*.h
 lib/ctime/*.h
 lib/string/*.h
 
-strlcat.c
-strlcpy.c
+ext/strlcat.c
+ext/strlcpy.c
diff --git a/src/lib/string/compat_ctype.c b/src/lib/string/compat_ctype.c
index f5d82be3ae..a7668bfbfb 100644
--- a/src/lib/string/compat_ctype.c
+++ b/src/lib/string/compat_ctype.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,7 @@ const uint32_t TOR_ISPRINT_TABLE[8] =
   { 0, 0xffffffff, 0xffffffff, 0x7fffffff, 0, 0, 0, 0x0 };
 const uint32_t TOR_ISUPPER_TABLE[8] = { 0, 0, 0x7fffffe, 0, 0, 0, 0, 0 };
 const uint32_t TOR_ISLOWER_TABLE[8] = { 0, 0, 0, 0x7fffffe, 0, 0, 0, 0 };
+/**@}*/
 
 /** Upper-casing and lowercasing tables to map characters to upper/lowercase
  * equivalents.  Used by tor_toupper() and tor_tolower(). */
diff --git a/src/lib/string/compat_ctype.h b/src/lib/string/compat_ctype.h
index dbddd356c1..53ee6066f8 100644
--- a/src/lib/string/compat_ctype.h
+++ b/src/lib/string/compat_ctype.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c
index e125c921a4..2bd3c2f2b4 100644
--- a/src/lib/string/compat_string.c
+++ b/src/lib/string/compat_string.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,10 +14,10 @@
 
 /* Inline the strl functions if the platform doesn't have them. */
 #ifndef HAVE_STRLCPY
-#include "strlcpy.c"
+#include "ext/strlcpy.c"
 #endif
 #ifndef HAVE_STRLCAT
-#include "strlcat.c"
+#include "ext/strlcat.c"
 #endif
 
 #include <stdlib.h>
diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h
index a0e37bb6dc..f05265bdcc 100644
--- a/src/lib/string/compat_string.h
+++ b/src/lib/string/compat_string.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,20 +25,23 @@ static inline int strncasecmp(const char *a, const char *b, size_t n);
 static inline int strncasecmp(const char *a, const char *b, size_t n) {
   return _strnicmp(a,b,n);
 }
-#endif
+#endif /* !defined(HAVE_STRNCASECMP) */
 #ifndef HAVE_STRCASECMP
 static inline int strcasecmp(const char *a, const char *b);
 static inline int strcasecmp(const char *a, const char *b) {
   return _stricmp(a,b);
 }
-#endif
-#endif
+#endif /* !defined(HAVE_STRCASECMP) */
+#endif /* defined(_WIN32) */
 
 #if defined __APPLE__
 /* On OSX 10.9 and later, the overlap-checking code for strlcat would
  * appear to have a severe bug that can sometimes cause aborts in Tor.
  * Instead, use the non-checking variants.  This is sad.
  *
+ * (If --enable-fragile-hardening is passed to configure, we use the hardened
+ * variants, which do not suffer from this issue.)
+ *
  * See https://trac.torproject.org/projects/tor/ticket/15205
  */
 #undef strlcat
@@ -59,4 +62,4 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
 #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts)
 #endif
 
-#endif
+#endif /* !defined(TOR_COMPAT_STRING_H) */
diff --git a/src/lib/string/include.am b/src/lib/string/include.am
index edd74b8a3e..82d35cc5af 100644
--- a/src/lib/string/include.am
+++ b/src/lib/string/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-string-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_string_a_SOURCES =			\
 	src/lib/string/compat_ctype.c			\
 	src/lib/string/compat_string.c			\
@@ -18,6 +19,7 @@ src_lib_libtor_string_testing_a_SOURCES = \
 src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/string/compat_ctype.h			\
 	src/lib/string/compat_string.h			\
diff --git a/src/lib/string/lib_string.md b/src/lib/string/lib_string.md
new file mode 100644
index 0000000000..98e3e652ed
--- /dev/null
+++ b/src/lib/string/lib_string.md
@@ -0,0 +1,13 @@
+@dir /lib/string
+@brief lib/string: Low-level string manipulation.
+
+We have a number of compatibility functions here: some are for handling
+functionality that is not implemented (or not implemented the same) on every
+platform; some are for providing locale-independent versions of libc
+functions that would otherwise be defined differently for different users.
+
+Other functions here are for common string-manipulation operations that we do
+in the rest of the codebase.
+
+Any string function high-level enough to need logging belongs in a
+higher-level module.
diff --git a/src/lib/string/parse_int.c b/src/lib/string/parse_int.c
index fbdd554a47..11ce0fa415 100644
--- a/src/lib/string/parse_int.c
+++ b/src/lib/string/parse_int.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,6 +9,7 @@
  **/
 
 #include "lib/string/parse_int.h"
+#include "lib/cc/compat_compiler.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -17,6 +18,7 @@
 /* Helper: common code to check whether the result of a strtol or strtoul or
  * strtoll is correct. */
 #define CHECK_STRTOX_RESULT()                           \
+  STMT_BEGIN                                            \
   /* Did an overflow occur? */                          \
   if (errno == ERANGE)                                  \
     goto err;                                           \
@@ -38,7 +40,8 @@
  err:                                                   \
   if (ok) *ok = 0;                                      \
   if (next) *next = endptr;                             \
-  return 0
+  return 0;                                             \
+  STMT_END
 
 /** Extract a long from the start of <b>s</b>, in the given numeric
  * <b>base</b>.  If <b>base</b> is 0, <b>s</b> is parsed as a decimal,
diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h
index 925547942e..27939ade61 100644
--- a/src/lib/string/parse_int.h
+++ b/src/lib/string/parse_int.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,4 @@ double tor_parse_double(const char *s, double min, double max, int *ok,
 uint64_t tor_parse_uint64(const char *s, int base, uint64_t min,
                          uint64_t max, int *ok, char **next);
 
-#endif
+#endif /* !defined(TOR_PARSE_INT_H) */
diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c
index a5cb71ce09..86d860935e 100644
--- a/src/lib/string/printf.c
+++ b/src/lib/string/printf.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -117,8 +117,8 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = NULL;
     return -1;
   }
-  strp_tmp = tor_malloc(len + 1);
-  r = _vsnprintf(strp_tmp, len+1, fmt, args);
+  strp_tmp = tor_malloc((size_t)len + 1);
+  r = _vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
@@ -153,9 +153,9 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = tor_strdup(buf);
     return len;
   }
-  strp_tmp = tor_malloc(len+1);
+  strp_tmp = tor_malloc((size_t)len+1);
   /* use of tor_vsnprintf() will ensure string is null terminated */
-  r = tor_vsnprintf(strp_tmp, len+1, fmt, args);
+  r = tor_vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h
index 2cc13d6bee..5ab751b338 100644
--- a/src/lib/string/printf.h
+++ b/src/lib/string/printf.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,4 +27,4 @@ int tor_asprintf(char **strp, const char *fmt, ...)
 int tor_vasprintf(char **strp, const char *fmt, va_list args)
   CHECK_PRINTF(2,0);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_PRINTF_H) */
diff --git a/src/lib/string/scanf.c b/src/lib/string/scanf.c
index 1bc39b5182..89d1683204 100644
--- a/src/lib/string/scanf.c
+++ b/src/lib/string/scanf.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h
index 6673173de5..67e9c5eb78 100644
--- a/src/lib/string/scanf.h
+++ b/src/lib/string/scanf.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,4 +21,4 @@ int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \
 int tor_sscanf(const char *buf, const char *pattern, ...)
   CHECK_SCANF(2, 3);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_SCANF_H) */
diff --git a/src/lib/string/strings.md b/src/lib/string/strings.md
new file mode 100644
index 0000000000..b22574a05a
--- /dev/null
+++ b/src/lib/string/strings.md
@@ -0,0 +1,102 @@
+
+@page strings String processing in Tor
+
+Since you're reading about a C program, you probably expected this
+section: it's full of functions for manipulating the (notoriously
+dubious) C string abstraction.  I'll describe some often-missed
+highlights here.
+
+### Comparing strings and memory chunks ###
+
+We provide strcmpstart() and strcmpend() to perform a strcmp with the start
+or end of a string.
+
+	tor_assert(!strcmpstart("Hello world","Hello"));
+	tor_assert(!strcmpend("Hello world","world"));
+
+	tor_assert(!strcasecmpstart("HELLO WORLD","Hello"));
+	tor_assert(!strcasecmpend("HELLO WORLD","world"));
+
+To compare two string pointers, either of which might be NULL, use
+strcmp_opt().
+
+To search for a string or a chunk of memory within a non-null
+terminated memory block, use tor_memstr or tor_memmem respectively.
+
+We avoid using memcmp() directly, since it tends to be used in cases
+when having a constant-time operation would be better.  Instead, we
+recommend tor_memeq() and tor_memneq() for when you need a
+constant-time operation.  In cases when you need a fast comparison,
+and timing leaks are not a danger, you can use fast_memeq() and
+fast_memneq().
+
+It's a common pattern to take a string representing one or more lines
+of text, and search within it for some other string, at the start of a
+line.  You could search for "\\ntarget", but that would miss the first
+line.  Instead, use find_str_at_start_of_line.
+
+### Parsing text ###
+
+Over the years, we have accumulated lots of ways to parse text --
+probably too many. Refactoring them to be safer and saner could be a
+good project!  The one that seems most error-resistant is tokenizing
+text with smartlist_split_strings().  This function takes a smartlist,
+a string, and a separator, and splits the string along occurrences of
+the separator, adding new strings for the sub-elements to the given
+smartlist.
+
+To handle time, you can use one of the functions mentioned above in
+"Parsing and encoding time values".
+
+For numbers in general, use the tor_parse_{long,ulong,double,uint64}
+family of functions.  Each of these can be called in a few ways.  The
+most general is as follows:
+
+      const int BASE = 10;
+      const int MINVAL = 10, MAXVAL = 10000;
+      const char *next;
+      int ok;
+      long lng = tor_parse_long("100", BASE, MINVAL, MAXVAL, &ok, &next);
+
+The return value should be ignored if "ok" is set to false.  The input
+string needs to contain an entire number, or it's considered
+invalid... unless the "next" pointer is available, in which case extra
+characters at the end are allowed, and "next" is set to point to the
+first such character.
+
+### Generating blocks of text ###
+
+For not-too-large blocks of text, we provide tor_asprintf(), which
+behaves like other members of the sprintf() family, except that it
+always allocates enough memory on the heap for its output.
+
+For larger blocks: Rather than using strlcat and strlcpy to build
+text, or keeping pointers to the interior of a memory block, we
+recommend that you use the smartlist_* functions to build a smartlist
+full of substrings in order.  Then you can concatenate them into a
+single string with smartlist_join_strings(), which also takes optional
+separator and terminator arguments.
+
+Alternatively, you might find it more convenient (and more
+allocation-efficient) to use the buffer API in buffers.c: Construct a buf_t
+object, add your data to it with buf_add_string(), buf_add_printf(), and so
+on, then call buf_extract() to get the resulting output.
+
+As a convenience, we provide smartlist_add_asprintf(), which combines
+the two methods above together.  Many of the cryptographic digest
+functions also accept a not-yet-concatenated smartlist of strings.
+
+### Logging helpers ###
+
+Often we'd like to log a value that comes from an untrusted source.
+To do this, use escaped() to escape the nonprintable characters and
+other confusing elements in a string, and surround it by quotes.  (Use
+esc_for_log() if you need to allocate a new string.)
+
+It's also handy to put memory chunks into hexadecimal before logging;
+you can use hex_str(memory, length) for that.
+
+The escaped() and hex_str() functions both provide outputs that are
+only valid till they are next invoked; they are not threadsafe.
+
+*/
diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c
index f934f66f02..c8f12d780e 100644
--- a/src/lib/string/util_string.c
+++ b/src/lib/string/util_string.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,7 +71,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle)
 
 /** Return true iff the 'len' bytes at 'mem' are all zero. */
 int
-tor_mem_is_zero(const char *mem, size_t len)
+fast_mem_is_zero(const char *mem, size_t len)
 {
   static const char ZERO[] = {
     0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
@@ -95,17 +95,14 @@ tor_mem_is_zero(const char *mem, size_t len)
 int
 tor_digest_is_zero(const char *digest)
 {
-  static const uint8_t ZERO_DIGEST[] = {
-    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
-  };
-  return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN);
+  return safe_mem_is_zero(digest, DIGEST_LEN);
 }
 
 /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
 int
 tor_digest256_is_zero(const char *digest)
 {
-  return tor_mem_is_zero(digest, DIGEST256_LEN);
+  return safe_mem_is_zero(digest, DIGEST256_LEN);
 }
 
 /** Remove from the string <b>s</b> every character which appears in
@@ -212,21 +209,6 @@ strcmpstart(const char *s1, const char *s2)
   return strncmp(s1, s2, n);
 }
 
-/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>,
- * without depending on a terminating nul in s1.  Sorting order is first by
- * length, then lexically; return values are as for strcmp.
- */
-int
-strcmp_len(const char *s1, const char *s2, size_t s1_len)
-{
-  size_t s2_len = strlen(s2);
-  if (s1_len < s2_len)
-    return -1;
-  if (s1_len > s2_len)
-    return 1;
-  return fast_memcmp(s1, s2, s2_len);
-}
-
 /** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
  * strcasecmp.
  */
@@ -524,6 +506,23 @@ validate_char(const uint8_t *c, uint8_t len)
 int
 string_is_utf8(const char *str, size_t len)
 {
+  // If str is NULL, don't try to read it
+  if (!str) {
+    // We could test for this case, but the low-level logs would produce
+    // confusing test output.
+    // LCOV_EXCL_START
+    if (len) {
+      // Use the low-level logging function, so that the log module can
+      // validate UTF-8 (if needed in future code)
+      tor_log_err_sigsafe(
+        "BUG: string_is_utf8() called with NULL str but non-zero len.");
+      // Since it's a bug, we should probably reject this string
+      return false;
+    }
+    // LCOV_EXCL_STOP
+    return true;
+  }
+
   for (size_t i = 0; i < len;) {
     uint8_t num_bytes = bytes_in_char(str[i]);
     if (num_bytes == 0) // Invalid leading byte found.
@@ -541,3 +540,16 @@ string_is_utf8(const char *str, size_t len)
   }
   return true;
 }
+
+/** As string_is_utf8(), but returns false if the string begins with a UTF-8
+ * byte order mark (BOM).
+ */
+int
+string_is_utf8_no_bom(const char *str, size_t len)
+{
+  if (str && len >= 3 && (!strcmpstart(str, "\uFEFF") ||
+                          !strcmpstart(str, "\uFFFE"))) {
+    return false;
+  }
+  return string_is_utf8(str, len);
+}
diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h
index d9fbf8c61e..e89233df88 100644
--- a/src/lib/string/util_string.h
+++ b/src/lib/string/util_string.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,10 @@ const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
                        size_t nlen);
 const void *tor_memstr(const void *haystack, size_t hlen,
                        const char *needle);
-int tor_mem_is_zero(const char *mem, size_t len);
+int fast_mem_is_zero(const char *mem, size_t len);
+#define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN)
+#define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN)
+
 int tor_digest_is_zero(const char *digest);
 int tor_digest256_is_zero(const char *digest);
 
@@ -33,7 +36,6 @@ int tor_strisnonupper(const char *s);
 int tor_strisspace(const char *s);
 int strcmp_opt(const char *s1, const char *s2);
 int strcmpstart(const char *s1, const char *s2);
-int strcmp_len(const char *s1, const char *s2, size_t len);
 int strcasecmpstart(const char *s1, const char *s2);
 int strcmpend(const char *s1, const char *s2);
 int strcasecmpend(const char *s1, const char *s2);
@@ -53,5 +55,6 @@ const char *find_str_at_start_of_line(const char *haystack,
 int string_is_C_identifier(const char *string);
 
 int string_is_utf8(const char *str, size_t len);
+int string_is_utf8_no_bom(const char *str, size_t len);
 
 #endif /* !defined(TOR_UTIL_STRING_H) */
diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include
new file mode 100644
index 0000000000..2b06e8519c
--- /dev/null
+++ b/src/lib/subsys/.may_include
@@ -0,0 +1 @@
+orconfig.h
diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am
new file mode 100644
index 0000000000..c9ab54ca73
--- /dev/null
+++ b/src/lib/subsys/include.am
@@ -0,0 +1,4 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS += \
+	src/lib/subsys/subsys.h
diff --git a/src/lib/subsys/initialization.md b/src/lib/subsys/initialization.md
new file mode 100644
index 0000000000..012ab7000d
--- /dev/null
+++ b/src/lib/subsys/initialization.md
@@ -0,0 +1,75 @@
+
+@page initialization Initialization and shutdown
+
+@tableofcontents
+
+@section overview Overview
+
+Tor has a single entry point: tor_run_main() in main.c.  All the ways of
+starting a Tor process (ntmain.c, tor_main.c, and tor_api.c) work by invoking tor_run_main().
+
+The tor_run_main() function normally exits (@ref init_exceptwhen "1") by
+returning: not by calling abort() or exit().  Before it returns, it calls
+tor_cleanup() in shutdown.c.
+
+Conceptually, there are several stages in running Tor.
+
+1. First, we initialize those modules that do not depend on the
+   configuration.  This happens in the first half of tor_run_main(), and the
+   first half of tor_init().  (@ref init_pending_refactor "2")
+
+2. Second, we parse the command line and our configuration, and configure
+   systems that depend on our configuration or state.  This configuration
+   happens midway through tor_init(), which invokes
+   options_init_from_torrc().  We then initialize more systems from the
+   second half of tor_init().
+
+3. At this point we may exit early if we have been asked to do something
+   requiring no further initialization, like printing our version number or
+   creating a new signing key.  Otherwise, we proceed to run_tor_main_loop(),
+   which initializes some network-specific parts of Tor, grabs some
+   daemon-only resources (like the data directory lock) and starts Tor itself
+   running.
+
+
+> @anchor init_exceptwhen 1. tor_run_main() _can_ terminate with a call to
+> abort() or exit(), but only when crashing due to a bug, or when forking to
+> run as a daemon.
+
+> @anchor init_pending_refactor 2. The pieces of code that I'm describing as
+> "the first part of tor_init()" and so on deserve to be functions with their
+> own name.  I'd like to refactor them, but before I do so, there is some
+> slight reorganization that needs to happen.  Notably, the
+> nt_service_parse_options() call ought logically to be later in our
+> initialization sequence.  See @ticket{32447} for our refactoring progress.
+
+
+@section subsys Subsystems and initialization
+
+Our current convention is to use the subsystem mechanism to initialize and
+clean up pieces of Tor.  The more recently updated pieces of Tor will use
+this mechanism.  For examples, see e.g. time_sys.c or log_sys.c.
+
+In simplest terms, a **subsytem** is a logically separate part of Tor that
+can be initialized, shut down, managed, and configured somewhat independently
+of the rest of the program.
+
+The subsys_fns_t type describes a subsystem and a set of functions that
+initialize it, desconstruct it, and so on. To define a subsystem, we declare
+a `const` instance of subsys_fns_t.  See the documentation for subsys_fns_t
+for a full list of these functions.
+
+After defining a subsytem, it must be inserted in subsystem_list.c.  At that
+point, table-driven mechanisms in subsysmgr.c will invoke its functions when
+appropriate.
+
+@subsection vsconfig Initialization versus configuration
+
+We note that the initialization phase of Tor occurs before any configuration
+is read from disk -- and therefore before any other files are read from
+disk.  Therefore, any behavior that depends on Tor's configuration or state
+must occur _after_ the initialization process, during configuration.
+
+
+
+
diff --git a/src/lib/subsys/lib_subsys.md b/src/lib/subsys/lib_subsys.md
new file mode 100644
index 0000000000..764d25d1b6
--- /dev/null
+++ b/src/lib/subsys/lib_subsys.md
@@ -0,0 +1,32 @@
+@dir /lib/subsys
+@brief lib/subsys: Types for declaring a "subsystem".
+
+## Subsystems in Tor
+
+A subsystem is a module with support for initialization, shutdown,
+configuration, and so on.
+
+Many parts of Tor can be initialized, cleaned up, and configured somewhat
+independently through a table-driven mechanism.  Each such part is called a
+"subsystem".
+
+To declare a subsystem, make a global `const` instance of the `subsys_fns_t`
+type, filling in the function pointer fields that you require with ones
+corresponding to your subsystem.  Any function pointers left as "NULL" will
+be a no-op.  Each system must have a name and a "level", which corresponds to
+the order in which it is initialized.  (See `app/main/subsystem_list.c` for a
+list of current subsystems and their levels.)
+
+Then, insert your subsystem in the list in `app/main/subsystem_list.c`.  It
+will need to occupy a position corresponding to its level.
+
+At this point, your subsystem will be handled like the others: it will get
+initialized at startup, torn down at exit, and so on.
+
+Historical note: Not all of Tor's code is currently handled as
+subsystems. As you work with older code, you may see some parts of the code
+that are initialized from `tor_init()` or `run_tor_main_loop()` or
+`tor_run_main()`; and torn down from `tor_cleanup()`.  We aim to migrate
+these to subsystems over time; please don't add any new code that follows
+this pattern.
+
diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h
new file mode 100644
index 0000000000..62c0de026d
--- /dev/null
+++ b/src/lib/subsys/subsys.h
@@ -0,0 +1,218 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsys.h
+ * @brief Types used to declare a subsystem.
+ **/
+
+#ifndef TOR_SUBSYS_T
+#define TOR_SUBSYS_T
+
+#include <stdbool.h>
+
+struct pubsub_connector_t;
+struct config_format_t;
+
+/**
+ * A subsystem is a part of Tor that is initialized, shut down, configured,
+ * and connected to other parts of Tor.
+ *
+ * All callbacks are optional -- if a callback is set to NULL, the subsystem
+ * manager will treat it as a no-op.
+ *
+ * You should use c99 named-field initializers with this structure, for
+ * readability and safety. (There are a lot of functions here, all of them
+ * optional, and many of them with similar signatures.)
+ *
+ * See @ref initialization for more information about initialization and
+ * shutdown in Tor.
+ *
+ * To make a new subsystem, you declare a const instance of this type, and
+ * include it on the list in subsystem_list.c.  The code that manages these
+ * subsystems is in subsysmgr.c.
+ **/
+typedef struct subsys_fns_t {
+  /**
+   * The name of this subsystem.  It should be a programmer-readable
+   * identifier.
+   **/
+  const char *name;
+
+  /**
+   * The file in which the subsystem object is declared. Used for debugging.
+   **/
+  const char *location;
+
+  /**
+   * Whether this subsystem is supported -- that is, whether it is compiled
+   * into Tor.  For most subsystems, this should be true.
+   **/
+  bool supported;
+
+  /**
+   * The 'initialization level' for the subsystem.  It should run from -100
+   * through +100.  The subsystems are initialized from lowest level to
+   * highest, and shut down from highest level to lowest.
+   **/
+  int level;
+
+  /**
+   * Initialize any global components of this subsystem.
+   *
+   * This function MAY rely on any lower-level subsystem being initialized.
+   *
+   * This function MUST NOT rely on any runtime configuration information;
+   * it is only for global state or pre-configuration state.
+   *
+   * (If you need to do any setup that depends on configuration, you'll need
+   * to declare a configuration callback instead. (Not yet designed))
+   *
+   * This function MUST NOT have any parts that can fail.
+   **/
+  int (*initialize)(void);
+
+  /**
+   * Connect a subsystem to the message dispatch system.
+   *
+   * This function should use the macros in @refdir{lib/pubsub} to register a
+   * set of messages that this subsystem may publish, and may subscribe to.
+   *
+   * See pubsub_macros.h for more information, and for examples.
+   **/
+  int (*add_pubsub)(struct pubsub_connector_t *);
+
+  /**
+   * Perform any necessary pre-fork cleanup.  This function may not fail.
+   *
+   * On Windows (and any other platforms without fork()), this function will
+   * never be invoked.  Otherwise it is used when we are about to start
+   * running as a background daemon, or when we are about to run a unit test
+   * in a subprocess.  Unlike the subsys_fns_t.postfork callback, it is run
+   * from the parent process.
+   *
+   * Note that we do not invoke this function when the child process's only
+   * purpose is to call exec() and run another program.
+   */
+  void (*prefork)(void);
+
+  /**
+   * Perform any necessary post-fork setup. This function may not fail.
+   *
+   * On Windows (and any other platforms without fork()), this function will
+   * never be invoked.  Otherwise it is used when we are about to start
+   * running as a background daemon, or when we are about to run a unit test
+   * in a subprocess.  Unlike the subsys_fns_t.prefork callback, it is run
+   * from the child process.
+   *
+   * Note that we do not invoke this function when the child process's only
+   * purpose is to call exec() and run another program.
+   */
+  void (*postfork)(void);
+
+  /**
+   * Free any thread-local resources held by this subsystem. Called before
+   * the thread exits.
+   *
+   * This function is not allowed to fail.
+   *
+   * \bug Note that this callback is currently buggy: See \ticket{32103}.
+   */
+  void (*thread_cleanup)(void);
+
+  /**
+   * Free all resources held by this subsystem.
+   *
+   * This function is not allowed to fail.
+   *
+   * Subsystems are shut down when Tor is about to exit or return control to
+   * an embedding program. This callback must return the process to a state
+   * such that subsys_fns_t.init will succeed if invoked again.
+   **/
+  void (*shutdown)(void);
+
+  /**
+   * A config_format_t describing all of the torrc fields owned by this
+   * subsystem.
+   *
+   * This object, if present, is registered in a confmgr_t for Tor's options,
+   * and used to parse option fields from the command line and torrc file.
+   **/
+  const struct config_format_t *options_format;
+
+  /**
+   * A config_format_t describing all of the DataDir/state fields owned by
+   * this subsystem.
+   *
+   * This object, if present, is registered in a confmgr_t for Tor's state,
+   * and used to parse state fields from the DataDir/state file.
+   **/
+  const struct config_format_t *state_format;
+
+  /**
+   * Receive an options object as defined by options_format. Return 0
+   * on success, -1 on failure.
+   *
+   * It is safe to store the pointer to the object until set_options()
+   * is called again.
+   *
+   * This function is only called after all the validation code defined
+   * by subsys_fns_t.options_format has passed.
+   **/
+  int (*set_options)(void *);
+
+  /* XXXX Add an implementation for options_act_reversible() later in this
+   * branch. */
+
+  /**
+   * Receive a state object as defined by state_format. Return 0 on success,
+   * -1 on failure.
+   *
+   * It is safe to store the pointer to the object; set_state() is only
+   * called on startup.
+   *
+   * This function is only called after all the validation code defined
+   * by subsys_fns_t.state_format has passed.
+   *
+   * This function will only be called once per invocation of Tor, since
+   * Tor does not reload its state while it is running.
+   **/
+  int (*set_state)(void *);
+
+  /**
+   * Update any information that needs to be stored in the provided state
+   * object (as defined by state_format).  Return 0 on success, -1 on failure.
+   *
+   * The object provided here will be the same one as provided earlier to
+   * set_state().  This method is called when we are about to save the state
+   * to disk.
+   **/
+  int (*flush_state)(void *);
+} subsys_fns_t;
+
+#ifndef COCCI
+/**
+ * Macro to declare a subsystem's location.
+ **/
+#define SUBSYS_DECLARE_LOCATION() \
+  .location = __FILE__
+#endif /* !defined(COCCI) */
+
+/**
+ * Lowest allowed subsystem level.
+ **/
+#define MIN_SUBSYS_LEVEL -100
+/**
+ * Highest allowed subsystem level.
+ **/
+#define MAX_SUBSYS_LEVEL 100
+
+/**
+ * All tor "libraries" (in src/libs) should have a subsystem level equal to or
+ * less than this value.
+ */
+#define SUBSYS_LEVEL_LIBS -10
+
+#endif /* !defined(TOR_SUBSYS_T) */
diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include
index c93a06e59e..306fa57b7a 100644
--- a/src/lib/term/.may_include
+++ b/src/lib/term/.may_include
@@ -5,5 +5,4 @@ lib/log/*.h
 lib/term/*.h
 lib/malloc/*.h
 
-# From src/ext
-tor_readpassphrase.h
+ext/tor_readpassphrase.h
diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c
index c6459f250f..d2d6cb2b7b 100644
--- a/src/lib/term/getpass.c
+++ b/src/lib/term/getpass.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #elif defined(HAVE_READPASSPHRASE_H)
 #include <readpassphrase.h>
 #else
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #endif /* defined(_WIN32) || ... */
 
 #include <stdlib.h>
diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h
index a9c146ea8f..b080ad2473 100644
--- a/src/lib/term/getpass.h
+++ b/src/lib/term/getpass.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,4 +15,4 @@
 
 ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
 
-#endif
+#endif /* !defined(TOR_GETPASS_H) */
diff --git a/src/lib/term/include.am b/src/lib/term/include.am
index 55fe548ebc..a120bba0cb 100644
--- a/src/lib/term/include.am
+++ b/src/lib/term/include.am
@@ -11,6 +11,7 @@ else
 readpassphrase_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_term_a_SOURCES =			\
 	src/lib/term/getpass.c			\
 	$(readpassphrase_source)
@@ -20,5 +21,6 @@ src_lib_libtor_term_testing_a_SOURCES = \
 src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/term/getpass.h
diff --git a/src/lib/term/lib_term.md b/src/lib/term/lib_term.md
new file mode 100644
index 0000000000..f96d25ffe8
--- /dev/null
+++ b/src/lib/term/lib_term.md
@@ -0,0 +1,2 @@
+@dir /lib/term
+@brief lib/term: Terminal operations (password input).
diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am
index b2aa620985..a5ed46eb67 100644
--- a/src/lib/testsupport/include.am
+++ b/src/lib/testsupport/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/testsupport/testsupport.h
diff --git a/src/lib/testsupport/lib_testsupport.md b/src/lib/testsupport/lib_testsupport.md
new file mode 100644
index 0000000000..7358e6a80f
--- /dev/null
+++ b/src/lib/testsupport/lib_testsupport.md
@@ -0,0 +1,2 @@
+@dir /lib/testsupport
+@brief lib/testsupport: Helpers for test-only code and for function mocking.
diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h
index 9363a9ba66..165c497f71 100644
--- a/src/lib/testsupport/testsupport.h
+++ b/src/lib/testsupport/testsupport.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,16 +15,41 @@
 #ifndef TOR_TESTSUPPORT_H
 #define TOR_TESTSUPPORT_H
 
-#ifdef TOR_UNIT_TESTS
 /** The "STATIC" macro marks a function or variable that is static when
  * building Tor for production, but non-static when building the unit
- * tests. */
+ * tests.
+ *
+ * For example, a function declared as:
+ *
+ *     STATIC int internal_function(void);
+ *
+ * should be only visible for the file on which it is declared, and in the
+ * unit tests.
+ */
+#ifdef TOR_UNIT_TESTS
 #define STATIC
+#else /* !defined(TOR_UNIT_TESTS) */
+#define STATIC static
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** The "EXTERN" macro is used along with "STATIC" for variables declarations:
+ * it expands to an extern declaration when Tor building unit tests, and to
+ * nothing otherwise.
+ *
+ * For example, to declare a variable as visible only visible in one
+ * file and in the unit tests, you would put this in the header:
+ *
+ *     EXTERN(int, local_variable)
+ *
+ * and this in the source:
+ *
+ *     STATIC int local_variable;
+ */
+#ifdef TOR_UNIT_TESTS
 #define EXTERN(type, name) extern type name;
 #else
-#define STATIC static
 #define EXTERN(type, name)
-#endif /* defined(TOR_UNIT_TESTS) */
+#endif
 
 /** Quick and dirty macros to implement test mocking.
  *
@@ -70,32 +95,42 @@
  *
  * @{ */
 #ifdef TOR_UNIT_TESTS
+/** Declare a mocked function. For use in headers. */
 #define MOCK_DECL(rv, funcname, arglist)     \
   rv funcname ##__real arglist;              \
   extern rv(*funcname) arglist
+/** Define the implementation of a mocked function. */
 #define MOCK_IMPL(rv, funcname, arglist)     \
   rv(*funcname) arglist = funcname ##__real; \
   rv funcname ##__real arglist
+/** As MOCK_DECL(), but allow attributes. */
 #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
   rv funcname ##__real arglist attr;                \
   extern rv(*funcname) arglist
-#define MOCK_IMPL(rv, funcname, arglist)     \
-  rv(*funcname) arglist = funcname ##__real; \
-  rv funcname ##__real arglist
+/**
+ * Replace <b>func</b> (a mockable function) with a replacement function.
+ *
+ * Only usable when Tor has been built for unit tests. */
 #define MOCK(func, replacement)                 \
   do {                                          \
     (func) = (replacement);                     \
   } while (0)
+/** Replace <b>func</b> (a mockable function) with its original value.
+ *
+ * Only usable when Tor has been built for unit tests. */
 #define UNMOCK(func)                            \
   do {                                          \
     func = func ##__real;                       \
   } while (0)
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
+/** Declare a mocked function. For use in headers. */
 #define MOCK_DECL(rv, funcname, arglist) \
   rv funcname arglist
-#define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
+/** As MOCK_DECL(), but allow  */
+#define MOCK_DECL_ATTR(rv, funcname, arglist, attr)     \
   rv funcname arglist attr
-#define MOCK_IMPL(rv, funcname, arglist) \
+/** Define the implementation of a mocked function. */
+#define MOCK_IMPL(rv, funcname, arglist)        \
   rv funcname arglist
 #endif /* defined(TOR_UNIT_TESTS) */
 /** @} */
diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include
index fc56f46836..02711348c5 100644
--- a/src/lib/thread/.may_include
+++ b/src/lib/thread/.may_include
@@ -2,6 +2,7 @@ orconfig.h
 lib/cc/*.h
 lib/lock/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/wallclock/*.h
diff --git a/src/lib/thread/compat_pthreads.c b/src/lib/thread/compat_pthreads.c
index 6f7ecd17da..d143b80252 100644
--- a/src/lib/thread/compat_pthreads.c
+++ b/src/lib/thread/compat_pthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c
index 94ab021c52..75ade9c9f2 100644
--- a/src/lib/thread/compat_threads.c
+++ b/src/lib/thread/compat_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,9 +14,11 @@
 #include "orconfig.h"
 #include <stdlib.h>
 #include "lib/thread/threads.h"
+#include "lib/thread/thread_sys.h"
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
+#include "lib/subsys/subsys.h"
 
 #include <string.h>
 
@@ -65,7 +67,15 @@ atomic_counter_init(atomic_counter_t *counter)
   memset(counter, 0, sizeof(*counter));
   tor_mutex_init_nonrecursive(&counter->mutex);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -109,3 +119,18 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return oldval;
 }
 #endif /* !defined(HAVE_WORKING_STDATOMIC) */
+
+static int
+subsys_threads_initialize(void)
+{
+  tor_threads_init();
+  return 0;
+}
+
+const subsys_fns_t sys_threads = {
+  .name = "threads",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -89,
+  .initialize = subsys_threads_initialize,
+};
diff --git a/src/lib/thread/compat_winthreads.c b/src/lib/thread/compat_winthreads.c
index f0b1430e84..2ca5620d23 100644
--- a/src/lib/thread/compat_winthreads.c
+++ b/src/lib/thread/compat_winthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am
index 9ec23d166e..cd8016b5df 100644
--- a/src/lib/thread/include.am
+++ b/src/lib/thread/include.am
@@ -12,6 +12,7 @@ if THREADS_WIN32
 threads_impl_source=src/lib/thread/compat_winthreads.c
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_thread_a_SOURCES =			\
 	src/lib/thread/compat_threads.c			\
 	src/lib/thread/numcpus.c			\
@@ -22,6 +23,8 @@ src_lib_libtor_thread_testing_a_SOURCES = \
 src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/thread/threads.h			\
-	src/lib/thread/numcpus.h
+	src/lib/thread/numcpus.h			\
+	src/lib/thread/thread_sys.h			\
+	src/lib/thread/threads.h
diff --git a/src/lib/thread/lib_thread.md b/src/lib/thread/lib_thread.md
new file mode 100644
index 0000000000..5870ad790f
--- /dev/null
+++ b/src/lib/thread/lib_thread.md
@@ -0,0 +1,7 @@
+@dir /lib/thread
+@brief lib/thread: Mid-level threading.
+
+This module contains compatibility and convenience code for multithreading,
+except for low-level locks (which are in \refdir{lib/lock} and
+workqueue/threadpool code (which belongs in \refdir{lib/evloop}.)
+
diff --git a/src/lib/thread/numcpus.c b/src/lib/thread/numcpus.c
index b293d965d2..18454ce3ad 100644
--- a/src/lib/thread/numcpus.c
+++ b/src/lib/thread/numcpus.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h
index 3f0a29ce7c..65e6c430cf 100644
--- a/src/lib/thread/numcpus.h
+++ b/src/lib/thread/numcpus.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,4 +13,4 @@
 
 int compute_num_cpus(void);
 
-#endif
+#endif /* !defined(TOR_NUMCPUS_H) */
diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h
new file mode 100644
index 0000000000..6206fac9d6
--- /dev/null
+++ b/src/lib/thread/thread_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file thread_sys.h
+ * \brief Declare subsystem object for threads library
+ **/
+
+#ifndef TOR_THREADS_SYS_H
+#define TOR_THREADS_SYS_H
+
+extern const struct subsys_fns_t sys_threads;
+
+#endif /* !defined(TOR_THREADS_SYS_H) */
diff --git a/src/lib/thread/threading.md b/src/lib/thread/threading.md
new file mode 100644
index 0000000000..a1058c97de
--- /dev/null
+++ b/src/lib/thread/threading.md
@@ -0,0 +1,26 @@
+
+@page threading Threading in Tor
+
+Tor is based around a single main thread and one or more worker
+threads.  We aim (with middling success) to use worker threads for
+CPU-intensive activities and the main thread for our networking.
+Fortunately (?) we have enough cryptography that moving what we can
+of the cryptographic processes to the workers should achieve good
+parallelism under most loads.  Unfortunately, we only have a small
+fraction of our cryptography done in our worker threads right now.
+
+Our threads-and-workers abstraction is defined in workqueue.c, which
+combines a work queue with a thread pool, and integrates the
+signalling with libevent.  Tor's main instance of a work queue is
+instantiated in cpuworker.c.  It will probably need some refactoring
+as more types of work are added.
+
+On a lower level, we provide locks with tor_mutex_t in \refdir{lib/lock}, and
+higher-level locking/threading tools in \refdir{lib/thread}, including
+conditions (tor_cond_t), thread-local storage (tor_threadlocal_t), and more.
+
+
+Try to minimize sharing between threads: it is usually best to simply
+make the worker "own" all the data it needs while the work is in
+progress, and to give up ownership when it's complete.
+
diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h
index ecf60641b5..fcc0c23a87 100644
--- a/src/lib/thread/threads.h
+++ b/src/lib/thread/threads.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -63,7 +63,7 @@ int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex,
 void tor_cond_signal_one(tor_cond_t *cond);
 void tor_cond_signal_all(tor_cond_t *cond);
 
-typedef struct tor_threadlocal_s {
+typedef struct tor_threadlocal_t {
 #ifdef _WIN32
   DWORD index;
 #else
@@ -106,8 +106,10 @@ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value);
 typedef struct atomic_counter_t {
   atomic_size_t val;
 } atomic_counter_t;
+#ifndef COCCI
 #define ATOMIC_LINKAGE static
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#endif
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 typedef struct atomic_counter_t {
   tor_mutex_t mutex;
   size_t val;
@@ -131,7 +133,17 @@ atomic_counter_init(atomic_counter_t *counter)
 {
   atomic_init(&counter->val, 0);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * This usage note applies to the compat_threads implementation of
+ * atomic_counter_destroy():
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 static inline void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -162,7 +174,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return atomic_exchange(&counter->val, newval);
 }
 
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 #endif /* defined(HAVE_WORKING_STDATOMIC) */
 
 #endif /* !defined(TOR_COMPAT_THREADS_H) */
diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include
index 2c7e37a836..ae01431b60 100644
--- a/src/lib/time/.may_include
+++ b/src/lib/time/.may_include
@@ -4,8 +4,10 @@ lib/cc/*.h
 lib/err/*.h
 lib/intmath/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/time/*.h
 lib/wallclock/*.h
+lib/defs/time.h
 
 # For load_windows_system_lib.
 lib/fs/winlib.h
\ No newline at end of file
diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c
index 98854bad2c..6bbad4f98a 100644
--- a/src/lib/time/compat_time.c
+++ b/src/lib/time/compat_time.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -164,6 +164,8 @@ static int64_t last_tick_count = 0;
  * to be monotonic; increments them as appropriate so that they actually
  * _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC int64_t
 ratchet_performance_counter(int64_t count_raw)
@@ -202,6 +204,8 @@ static struct timeval timeofday_offset = { 0, 0 };
  * supposed to be monotonic; increments them as appropriate so that they
  * actually _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC void
 ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
@@ -270,7 +274,9 @@ monotime_init_internal(void)
 }
 
 /**
- * Set "out" to the most recent monotonic time value
+ * Set "out" to the most recent monotonic time value.
+ *
+ * The returned time may be the same as the previous returned time.
  */
 void
 monotime_get(monotime_t *out)
@@ -298,10 +304,12 @@ monotime_coarse_get(monotime_coarse_t *out)
 #endif /* defined(TOR_UNIT_TESTS) */
   out->abstime_ = mach_approximate_time();
 }
-#endif
+#endif /* defined(HAVE_MACH_APPROXIMATE_TIME) */
 
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ *
+ * The returned value may be equal to zero.
  */
 int64_t
 monotime_diff_nsec(const monotime_t *start,
@@ -522,7 +530,9 @@ monotime_init_internal(void)
     GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void))
       GetProcAddress(h, "GetTickCount64");
   }
-  // FreeLibrary(h) ?
+  // We can't call FreeLibrary(h) here, because freeing the handle may
+  // unload the library, and cause future calls to GetTickCount64_fn()
+  // to fail. See 29642 for details.
 }
 
 void
@@ -757,7 +767,7 @@ monotime_coarse_zero(monotime_coarse_t *out)
 {
   memset(out, 0, sizeof(*out));
 }
-#endif
+#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 
 int64_t
 monotime_diff_usec(const monotime_t *start,
@@ -787,8 +797,8 @@ monotime_absolute_nsec(void)
   return monotime_diff_nsec(&initialized_at, &now);
 }
 
-uint64_t
-monotime_absolute_usec(void)
+MOCK_IMPL(uint64_t,
+monotime_absolute_usec,(void))
 {
   return monotime_absolute_nsec() / 1000;
 }
@@ -823,7 +833,7 @@ monotime_coarse_absolute_msec(void)
 {
   return monotime_coarse_absolute_nsec() / ONE_MILLION;
 }
-#else
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define initialized_at_coarse initialized_at
 #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 
@@ -855,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
     mach_time_info.numer;
   return abstime_val >> monotime_shift;
 }
-#else
+#else /* !defined(__APPLE__) */
 uint64_t
 monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
 {
@@ -866,4 +876,4 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
 {
   return (msec * STAMP_TICKS_PER_SECOND) / 1000;
 }
-#endif
+#endif /* defined(__APPLE__) */
diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h
index 480d426ac7..5089e16ca5 100644
--- a/src/lib/time/compat_time.h
+++ b/src/lib/time/compat_time.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,6 +15,120 @@
  * of tens of milliseconds.
  */
 
+/* Q: When should I use monotonic time?
+ *
+ * A: If you need a time that never decreases, use monotonic time. If you need
+ * to send a time to a user or another process, or store a time, use the
+ * wall-clock time.
+ *
+ * Q: Should you use monotime or monotime_coarse as your source?
+ *
+ * A: Generally, you get better precision with monotime, but better
+ * performance with monotime_coarse.
+ *
+ * Q: What is a "monotonic" time, exactly?
+ *
+ * A: Monotonic times are strictly non-decreasing. The difference between any
+ * previous monotonic time, and the current monotonic time, is always greater
+ * than *or equal to* zero.
+ * Zero deltas happen more often:
+ *  - on Windows (due to an OS bug),
+ *  - when using monotime_coarse, or on systems with low-resolution timers,
+ *  - on platforms where we emulate monotonic time using wall-clock time, and
+ *  - when using time units that are larger than nanoseconds (due to
+ *    truncation on division).
+ *
+ * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use
+ *    usec? msec? "stamp units?"
+ *
+ * A: Using monotime_t and monotime_coarse_t directly is most time-efficient,
+ * since no conversion needs to happen.  But they can potentially use more
+ * memory than you would need for a usec/msec/"stamp unit" count.
+ *
+ * Converting to usec or msec on some platforms, and working with them in
+ * general, creates a risk of doing a 64-bit division.  64-bit division is
+ * expensive on 32-bit platforms, which still do exist.
+ *
+ * The "stamp unit" type is designed to give a type that is cheap to convert
+ * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a
+ * 32-bit integer.  Its downside is that it does not correspond directly
+ * to a natural unit of time.
+ *
+ * There is not much point in using "coarse usec" or "coarse nsec", since the
+ * current coarse monotime implementations give you on the order of
+ * milliseconds of precision.
+ *
+ * Q: So, what backends is monotime_coarse using?
+ *
+ * A: Generally speaking, it uses "whatever monotonic-ish time implemenation
+ * does not require a context switch."  The various implementations provide
+ * this by having a view of the current time in a read-only memory page that
+ * is updated with a frequency corresponding to the kernel's tick count.
+ *
+ * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on
+ * obsolete systems].  MSDN claims that the resolution is "typically in the
+ * range of 10-16 msec", but it has said that for years.  Storing
+ * monotime_coarse_t uses 8 bytes.
+ *
+ * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where
+ * available, and falls back to regular monotime. The precision is not
+ * documented, but the implementation is open-source: it reads from a page
+ * that the kernel updates. Storing monotime_coarse_t uses 8 bytes.
+ *
+ * On unixy systems, monotime_coarse uses clock_gettime() with
+ * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC.
+ * It typically uses vdso tricks to read from a page that the kernel updates.
+ * Its precision fixed, but you can get it with clock_getres(): on my Linux
+ * desktop, it claims to be 1 msec, but it will depend on the system HZ
+ * setting. Storing monotime_coarse_t uses 16 bytes.
+ *
+ * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.]
+ *
+ * Q: What backends is regular monotonic time using?
+ *
+ * A: In general, regular monotime uses something that requires a system call.
+ * On platforms where system calls are cheap, you win!  Otherwise, you lose.
+ *
+ * On Windows, monotonic time uses QuereyPerformanceCounter.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On OSX/Apple, monotonic time uses mach_absolute_time.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On unixy systems, monotonic time uses CLOCK_MONOTONIC.  Storing
+ * monotime_t costs 16 bytes.
+ *
+ * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec
+ *    count.
+ *
+ * A: Windows, coarse: Cheap, since it's all multiplication.
+ *
+ * Windows, precise: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Apple, all: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Unixy, all: Fairly cheap, since the only division required is dividing
+ * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value.
+ *
+ * All, "timestamp units": Cheap everywhere: it never divides.
+ *
+ * Q: This is only somewhat related, but how much precision could I hope for
+ *    from a libevent time?
+ *
+ * A: Actually, it's _very_ related if you're timing in order to have a
+ * timeout happen.
+ *
+ * On Windows, it uses select: you could in theory have a microsecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On Linux, you have epoll: It has a millisecond resolution.  Some recent
+ * Libevents can also use timerfd for higher resolution if
+ * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag.
+ */
+
 #ifndef TOR_COMPAT_TIME_H
 #define TOR_COMPAT_TIME_H
 
@@ -86,26 +200,36 @@ void monotime_init(void);
 void monotime_get(monotime_t *out);
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
  */
 int64_t monotime_diff_nsec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of microseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of milliseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_msec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of nanoseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
  */
 uint64_t monotime_absolute_nsec(void);
 /**
  * Return the number of microseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
-uint64_t monotime_absolute_usec(void);
+MOCK_DECL(uint64_t, monotime_absolute_usec,(void));
 /**
  * Return the number of milliseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_absolute_msec(void);
 
@@ -129,10 +253,13 @@ void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec);
  * Set <b>out</b> to the current coarse time.
  */
 void monotime_coarse_get(monotime_coarse_t *out);
+/**
+ * Like monotime_absolute_*(), but faster on some platforms.
+ */
 uint64_t monotime_coarse_absolute_nsec(void);
 uint64_t monotime_coarse_absolute_usec(void);
 uint64_t monotime_coarse_absolute_msec(void);
-#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define monotime_coarse_get monotime_get
 #define monotime_coarse_absolute_nsec monotime_absolute_nsec
 #define monotime_coarse_absolute_usec monotime_absolute_usec
@@ -152,23 +279,32 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t);
 /**
  * Convert a difference, expressed in the units of monotime_coarse_to_stamp,
  * into an approximate number of milliseconds.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units);
 uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec);
 uint32_t monotime_coarse_get_stamp(void);
 
 #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)
+/**
+ * Like monotime_diff_*(), but faster on some platforms.
+ */
 int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
+/**
+ * Like monotime_*(), but faster on some platforms.
+ */
 void monotime_coarse_zero(monotime_coarse_t *out);
 int monotime_coarse_is_zero(const monotime_coarse_t *val);
 void monotime_coarse_add_msec(monotime_coarse_t *out,
                               const monotime_coarse_t *val, uint32_t msec);
-#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 #define monotime_coarse_diff_nsec monotime_diff_nsec
 #define monotime_coarse_diff_usec monotime_diff_usec
 #define monotime_coarse_diff_msec monotime_diff_msec
@@ -182,6 +318,9 @@ void monotime_coarse_add_msec(monotime_coarse_t *out,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
                                      const monotime_coarse_t *end);
@@ -191,6 +330,9 @@ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 static inline int32_t
 monotime_coarse_diff_msec32(const monotime_coarse_t *start,
@@ -202,7 +344,7 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start,
 #else
 #define USING_32BIT_MSEC_HACK
   return monotime_coarse_diff_msec32_(start, end);
-#endif
+#endif /* SIZEOF_VOID_P == 8 */
 }
 
 #ifdef TOR_UNIT_TESTS
diff --git a/src/lib/time/include.am b/src/lib/time/include.am
index a3f93a3744..dcb199b142 100644
--- a/src/lib/time/include.am
+++ b/src/lib/time/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-time-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_time_a_SOURCES =	\
 		src/lib/time/compat_time.c	\
+		src/lib/time/time_sys.c		\
 		src/lib/time/tvdiff.c
 
 src_lib_libtor_time_testing_a_SOURCES = \
@@ -14,6 +16,8 @@ src_lib_libtor_time_testing_a_SOURCES = \
 src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/time/compat_time.h	\
+		src/lib/time/time_sys.h		\
 		src/lib/time/tvdiff.h
diff --git a/src/lib/time/lib_time.md b/src/lib/time/lib_time.md
new file mode 100644
index 0000000000..8e58aafcd8
--- /dev/null
+++ b/src/lib/time/lib_time.md
@@ -0,0 +1,9 @@
+@dir /lib/time
+@brief lib/time: Higher-level time functions
+
+This includes both fine-grained timers and monotonic timers, along with
+wrappers for them to try to improve efficiency.
+
+For "what time is it" in UTC, see \refdir{lib/wallclock}.  For parsing and
+encoding times and dates, see \refdir{lib/encoding}.
+
diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c
new file mode 100644
index 0000000000..1c1bc4cd18
--- /dev/null
+++ b/src/lib/time/time_sys.c
@@ -0,0 +1,29 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.c
+ * \brief Subsystem object for monotime setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/time/time_sys.h"
+#include "lib/time/compat_time.h"
+
+static int
+subsys_time_initialize(void)
+{
+  monotime_init();
+  return 0;
+}
+
+const subsys_fns_t sys_time = {
+  .name = "time",
+  SUBSYS_DECLARE_LOCATION(),
+  /* Monotonic time depends on logging, and a lot of other modules depend on
+   * monotonic time. */
+  .level = -80,
+  .supported = true,
+  .initialize = subsys_time_initialize,
+};
diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h
new file mode 100644
index 0000000000..f2401e1911
--- /dev/null
+++ b/src/lib/time/time_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.h
+ * \brief Declare subsystem object for the time module.
+ **/
+
+#ifndef TOR_TIME_SYS_H
+#define TOR_TIME_SYS_H
+
+extern const struct subsys_fns_t sys_time;
+
+#endif /* !defined(TOR_TIME_SYS_H) */
diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c
index a87d0d96dc..cbad5a48b8 100644
--- a/src/lib/time/tvdiff.c
+++ b/src/lib/time/tvdiff.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,7 @@
 #include "lib/time/tvdiff.h"
 
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/time.h"
 #include "lib/log/log.h"
 
 #ifdef _WIN32
@@ -20,8 +21,6 @@
 #include <sys/time.h>
 #endif
 
-#define TOR_USEC_PER_SEC 1000000
-
 /** Return the difference between start->tv_sec and end->tv_sec.
  * Returns INT64_MAX on overflow and underflow.
  */
diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h
index 724af1528a..e779e758f1 100644
--- a/src/lib/time/tvdiff.h
+++ b/src/lib/time/tvdiff.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ int64_t tv_to_msec(const struct timeval *tv);
 
 time_t time_diff(const time_t from, const time_t to);
 
-#endif
+#endif /* !defined(TOR_TVDIFF_H) */
diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include
index 2840e590b8..c550bde024 100644
--- a/src/lib/tls/.may_include
+++ b/src/lib/tls/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/crypt_ops/*.h
@@ -11,7 +12,7 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/tls/*.h
-
-ciphers.inc
+lib/tls/*.inc
diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c
index b570216df0..b92a14d6a1 100644
--- a/src/lib/tls/buffers_tls.c
+++ b/src/lib/tls/buffers_tls.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
@@ -68,10 +68,10 @@ buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
 
   check_no_tls_errors();
 
-  IF_BUG_ONCE(buf->datalen >= INT_MAX)
-    return -1;
-  IF_BUG_ONCE(buf->datalen >= INT_MAX - at_most)
-    return -1;
+  IF_BUG_ONCE(buf->datalen > BUF_MAX_LEN)
+    return TOR_TLS_ERROR_MISC;
+  IF_BUG_ONCE(buf->datalen > BUF_MAX_LEN - at_most)
+    return TOR_TLS_ERROR_MISC;
 
   while (at_most > total_read) {
     size_t readlen = at_most - total_read;
@@ -90,7 +90,7 @@ buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
     r = read_to_chunk_tls(buf, chunk, tls, readlen);
     if (r < 0)
       return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
+    tor_assert(total_read+r <= BUF_MAX_LEN);
     total_read += r;
     if ((size_t)r < readlen) /* eof, block, or no more to read. */
       break;
@@ -177,6 +177,6 @@ buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
     if (r == 0) /* Can't flush any more now. */
       break;
   } while (sz > 0);
-  tor_assert(flushed < INT_MAX);
+  tor_assert(flushed <= BUF_MAX_LEN);
   return (int)flushed;
 }
diff --git a/src/lib/tls/buffers_tls.h b/src/lib/tls/buffers_tls.h
index 65788c3f34..587426801d 100644
--- a/src/lib/tls/buffers_tls.h
+++ b/src/lib/tls/buffers_tls.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am
index a664b29fb2..7e05ef4f8c 100644
--- a/src/lib/tls/include.am
+++ b/src/lib/tls/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-tls-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_tls_a_SOURCES =			\
 	src/lib/tls/buffers_tls.c		\
 	src/lib/tls/tortls.c			\
@@ -29,6 +30,7 @@ src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_tls_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/tls/ciphers.inc			\
 	src/lib/tls/buffers_tls.h		\
@@ -36,5 +38,6 @@ noinst_HEADERS +=				\
 	src/lib/tls/tortls.h			\
 	src/lib/tls/tortls_internal.h		\
 	src/lib/tls/tortls_st.h			\
+	src/lib/tls/tortls_sys.h		\
 	src/lib/tls/x509.h			\
 	src/lib/tls/x509_internal.h
diff --git a/src/lib/tls/lib_tls.md b/src/lib/tls/lib_tls.md
new file mode 100644
index 0000000000..26fea723f9
--- /dev/null
+++ b/src/lib/tls/lib_tls.md
@@ -0,0 +1,11 @@
+@dir /lib/tls
+@brief lib/tls: TLS library wrappers
+
+This module has compatibility wrappers around the library (NSS or OpenSSL,
+depending on configuration) that Tor uses to implement the TLS link security
+protocol.
+
+It also implements the logic for some legacy TLS protocol usage we used to
+support in old versions of Tor, involving conditional delivery of certificate
+chains (v1 link protocol) and conditional renegotiation (v2 link protocol).
+
diff --git a/src/lib/tls/nss_countbytes.c b/src/lib/tls/nss_countbytes.c
index 7761727acd..4b98df80ec 100644
--- a/src/lib/tls/nss_countbytes.c
+++ b/src/lib/tls/nss_countbytes.c
@@ -1,4 +1,4 @@
-/* Copyright 2018-2019, The Tor Project Inc. */
+/* Copyright 2018-2020, The Tor Project Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h
index 8b31603923..36ed55e10d 100644
--- a/src/lib/tls/nss_countbytes.h
+++ b/src/lib/tls/nss_countbytes.h
@@ -1,4 +1,4 @@
-/* Copyright 2018-2019, The Tor Project, Inc. */
+/* Copyright 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,4 @@ int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd,
                                    uint64_t *n_read_out,
                                    uint64_t *n_written_out);
 
-#endif
+#endif /* !defined(TOR_NSS_COUNTBYTES_H) */
diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c
index 4ca7c7d9d3..9e70e54725 100644
--- a/src/lib/tls/tortls.c
+++ b/src/lib/tls/tortls.c
@@ -1,12 +1,18 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tortls.c
+ * @brief Shared functionality for our TLS backends.
+ **/
+
 #define TORTLS_PRIVATE
 #define TOR_X509_PRIVATE
 #include "lib/tls/x509.h"
 #include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls_sys.h"
 #include "lib/tls/tortls.h"
 #include "lib/tls/tortls_st.h"
 #include "lib/tls/tortls_internal.h"
@@ -15,6 +21,7 @@
 #include "lib/crypt_ops/crypto_rsa.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/net/socket.h"
+#include "lib/subsys/subsys.h"
 
 #ifdef _WIN32
   #include <winsock2.h>
@@ -440,3 +447,16 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity)
 
   return rv;
 }
+
+static void
+subsys_tortls_shutdown(void)
+{
+  tor_tls_free_all();
+}
+
+const subsys_fns_t sys_tortls = {
+  .name = "tortls",
+  SUBSYS_DECLARE_LOCATION(),
+  .level = -50,
+  .shutdown = subsys_tortls_shutdown
+};
diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h
index 9398318035..3703ecad56 100644
--- a/src/lib/tls/tortls.h
+++ b/src/lib/tls/tortls.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TORTLS_H
@@ -25,12 +25,12 @@ struct ssl_ctx_st;
 struct ssl_session_st;
 typedef struct ssl_ctx_st tor_tls_context_impl_t;
 typedef struct ssl_st tor_tls_impl_t;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 struct PRFileDesc;
 typedef struct PRFileDesc tor_tls_context_impl_t;
 typedef struct PRFileDesc tor_tls_impl_t;
-#endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+#endif /* defined(TORTLS_PRIVATE) */
 
 struct tor_x509_cert_t;
 
@@ -81,6 +81,7 @@ void tor_tls_free_all(void);
 void tor_tls_init(void);
 void tls_log_errors(tor_tls_t *tls, int severity, int domain,
                     const char *doing);
+const char *tor_tls_get_last_error_msg(const tor_tls_t *tls);
 int tor_tls_context_init(unsigned flags,
                          crypto_pk_t *client_identity,
                          crypto_pk_t *server_identity,
@@ -143,9 +144,9 @@ void check_no_tls_errors_(const char *fname, int line);
 
 void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
                            int severity, int domain, const char *doing);
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 #define check_no_tls_errors() STMT_NIL
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 int tor_tls_get_my_certs(int server,
                          const struct tor_x509_cert_t **link_cert_out,
diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h
index 071c506561..3f56f181ee 100644
--- a/src/lib/tls/tortls_internal.h
+++ b/src/lib/tls/tortls_internal.h
@@ -1,11 +1,18 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tortls_internal.h
+ * @brief Declare internal functions for lib/tls
+ **/
+
 #ifndef TORTLS_INTERNAL_H
 #define TORTLS_INTERNAL_H
 
+#include "lib/tls/x509.h"
+
 int tor_errno_to_tls_error(int e);
 #ifdef ENABLE_OPENSSL
 int tor_tls_get_error(tor_tls_t *tls, int r, int extra,
@@ -61,8 +68,8 @@ STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret,
                             void *arg);
 STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
                              uint16_t cipher);
-#endif
-#endif
+#endif /* defined(TORTLS_OPENSSL_PRIVATE) */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef TOR_UNIT_TESTS
 extern int tor_tls_object_ex_data_index;
@@ -73,4 +80,4 @@ extern uint64_t total_bytes_written_over_tls;
 extern uint64_t total_bytes_written_by_tls;
 #endif /* defined(TOR_UNIT_TESTS) */
 
-#endif /* defined(TORTLS_INTERNAL_H) */
+#endif /* !defined(TORTLS_INTERNAL_H) */
diff --git a/src/lib/tls/tortls_nss.c b/src/lib/tls/tortls_nss.c
index b7f3513150..25446a81af 100644
--- a/src/lib/tls/tortls_nss.c
+++ b/src/lib/tls/tortls_nss.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,7 +34,7 @@
 #include "lib/tls/nss_countbytes.h"
 #include "lib/log/util_bug.h"
 
-DISABLE_GCC_WARNING(strict-prototypes)
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
 #include <prio.h>
 // For access to rar sockets.
 #include <private/pprio.h>
@@ -42,7 +42,7 @@ DISABLE_GCC_WARNING(strict-prototypes)
 #include <sslt.h>
 #include <sslproto.h>
 #include <certt.h>
-ENABLE_GCC_WARNING(strict-prototypes)
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
 
 static SECStatus always_accept_cert_cb(void *, PRFileDesc *, PRBool, PRBool);
 
@@ -369,6 +369,8 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain,
 
   (void)tls;
   PRErrorCode code = PORT_GetError();
+  if (tls)
+    tls->last_error = code;
 
   const char *addr = tls ? tls->address : NULL;
   const char *string = PORT_ErrorToString(code);
@@ -391,6 +393,17 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain,
            with, addr);
   }
 }
+const char *
+tor_tls_get_last_error_msg(const tor_tls_t *tls)
+{
+  IF_BUG_ONCE(!tls) {
+    return NULL;
+  }
+  if (tls->last_error == 0) {
+    return NULL;
+  }
+  return PORT_ErrorToString((PRErrorCode)tls->last_error);
+}
 
 tor_tls_t *
 tor_tls_new(tor_socket_t sock, int is_server)
diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c
index c5031a00aa..5c92df1259 100644
--- a/src/lib/tls/tortls_openssl.c
+++ b/src/lib/tls/tortls_openssl.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
    * <winsock.h> and mess things up, in at least some openssl versions. */
   #include <winsock2.h>
   #include <ws2tcpip.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rand.h"
@@ -37,7 +37,7 @@
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
  * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/opensslv.h>
 
@@ -54,7 +54,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "lib/tls/tortls.h"
 #include "lib/tls/tortls_st.h"
@@ -245,10 +245,30 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing)
   unsigned long err;
 
   while ((err = ERR_get_error()) != 0) {
+    if (tls)
+      tls->last_error = err;
     tor_tls_log_one_error(tls, err, severity, domain, doing);
   }
 }
 
+/**
+ * Return a string representing more detail about the last error received
+ * on TLS.
+ *
+ * May return null if no error was found.
+ **/
+const char *
+tor_tls_get_last_error_msg(const tor_tls_t *tls)
+{
+  IF_BUG_ONCE(!tls) {
+    return NULL;
+  }
+  if (tls->last_error == 0) {
+    return NULL;
+  }
+  return (const char*)ERR_reason_error_string(tls->last_error);
+}
+
 #define CATCH_SYSCALL 1
 #define CATCH_ZERO    2
 
@@ -318,7 +338,7 @@ tor_tls_init(void)
 #else
     SSL_library_init();
     SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))
@@ -383,7 +403,7 @@ static const char SERVER_CIPHER_LIST[] =
    * conclude that it has no valid ciphers if it's running with TLS1.3.
    */
   TLS1_3_TXT_AES_128_GCM_SHA256 ":"
-#endif
+#endif /* defined(TLS1_3_TXT_AES_128_GCM_SHA256) */
   TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":"
   TLS1_TXT_DHE_RSA_WITH_AES_128_SHA;
 
@@ -464,7 +484,9 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 /** List of ciphers that clients should advertise, omitting items that
  * our OpenSSL doesn't know about. */
 static const char CLIENT_CIPHER_LIST[] =
-#include "ciphers.inc"
+#ifndef COCCI
+#include "lib/tls/ciphers.inc"
+#endif
   /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version
    * of any cipher we say. */
   "!SSLv2"
@@ -657,7 +679,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     if (r < 0)
       goto error;
   }
-#else
+#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */
   if (! is_client) {
     int nid;
     EC_KEY *ec_key;
@@ -673,7 +695,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
       SSL_CTX_set_tmp_ecdh(result->ctx, ec_key);
     EC_KEY_free(ec_key);
   }
-#endif
+#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */
   SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER,
                      always_accept_verify_cb);
   /* let us realloc bufs that we're writing from */
@@ -764,7 +786,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher);
     return c != NULL;
   }
-#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */
+#else /* !defined(HAVE_SSL_CIPHER_FIND) */
 
 # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR)
   if (m && m->get_cipher_by_char) {
@@ -1062,7 +1084,7 @@ tor_tls_new(tor_socket_t sock, int isServer)
     /* We can't actually use TLS 1.3 until this bug is fixed. */
     SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION);
   }
-#endif
+#endif /* defined(SSL_CTRL_SET_MAX_PROTO_VERSION) */
 
   if (!SSL_set_cipher_list(result->ssl,
                      isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) {
@@ -1715,7 +1737,7 @@ tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
     else
       return -1;
   }
-#endif
+#endif /* defined(TLS1_3_VERSION) */
 
   return (r == 1) ? 0 : -1;
 }
diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h
index 3f7ea8ac6a..34abe52ee3 100644
--- a/src/lib/tls/tortls_st.h
+++ b/src/lib/tls/tortls_st.h
@@ -1,11 +1,19 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TORTLS_ST_H
 #define TOR_TORTLS_ST_H
 
+/**
+ * @file tortls_st.h
+ * @brief Structure declarations for internal TLS types.
+ *
+ * These should generally be treated as opaque outside of the
+ * lib/tls module.
+ **/
+
 #include "lib/net/socket.h"
 
 #define TOR_TLS_MAGIC 0x71571571
@@ -59,17 +67,20 @@ struct tor_tls_t {
    */
   unsigned long last_write_count;
   unsigned long last_read_count;
+  /** Most recent error value from ERR_get_error(). */
+  unsigned long last_error;
   /** If set, a callback to invoke whenever the client tries to renegotiate
    * the handshake. */
   void (*negotiated_callback)(tor_tls_t *tls, void *arg);
   /** Argument to pass to negotiated_callback. */
   void *callback_arg;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
   /** Last values retried from tor_get_prfiledesc_byte_counts(). */
   uint64_t last_write_count;
   uint64_t last_read_count;
+  long last_error;
 #endif
 };
 
-#endif
+#endif /* !defined(TOR_TORTLS_ST_H) */
diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h
new file mode 100644
index 0000000000..177c198f71
--- /dev/null
+++ b/src/lib/tls/tortls_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tortls_sys.h
+ * \brief Declare subsystem object for the tortls module
+ **/
+
+#ifndef TOR_TORTLS_SYS_H
+#define TOR_TORTLS_SYS_H
+
+extern const struct subsys_fns_t sys_tortls;
+
+#endif /* !defined(TOR_TORTLS_SYS_H) */
diff --git a/src/lib/tls/x509.c b/src/lib/tls/x509.c
index b4a0f8dabf..2515499298 100644
--- a/src/lib/tls/x509.c
+++ b/src/lib/tls/x509.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h
index 5e6660de5c..5919b9089d 100644
--- a/src/lib/tls/x509.h
+++ b/src/lib/tls/x509.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_X509_H
@@ -35,7 +35,7 @@ struct tor_x509_cert_t {
   common_digests_t cert_digests;
   common_digests_t pkey_digests;
 };
-#endif
+#endif /* defined(TOR_X509_PRIVATE) */
 
 void tor_tls_pick_certificate_lifetime(time_t now,
                                        unsigned cert_lifetime,
@@ -47,7 +47,7 @@ tor_x509_cert_t *tor_x509_cert_replace_expiration(
                                                const tor_x509_cert_t *inp,
                                                time_t new_expiration_time,
                                                crypto_pk_t *signing_key);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 
@@ -72,4 +72,4 @@ int tor_tls_cert_is_valid(int severity,
                           time_t now,
                           int check_rsa_1024);
 
-#endif
+#endif /* !defined(TOR_X509_H) */
diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h
index bf2bec9689..145be7e71c 100644
--- a/src/lib/tls/x509_internal.h
+++ b/src/lib/tls/x509_internal.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_X509_INTERNAL_H
@@ -50,4 +50,4 @@ int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert);
 #define tor_x509_cert_set_cached_der_encoding(cert) (0)
 #endif
 
-#endif
+#endif /* !defined(TOR_X509_INTERNAL_H) */
diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c
index fb4af54c52..341bb57104 100644
--- a/src/lib/tls/x509_nss.c
+++ b/src/lib/tls/x509_nss.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -120,13 +120,13 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                      der.data, der.len,
                                      (SECKEYPrivateKey *)signing_key,//const
                                      &cert->signature);
-#else
+#else /* !(0) */
   s = SEC_DerSignData(cert->arena,
                       &signed_der,
                       der.data, der.len,
                       (SECKEYPrivateKey *)signing_key,//const
                       SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION);
-#endif
+#endif /* 0 */
 
   if (s != SECSuccess)
     goto err;
@@ -145,7 +145,7 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                &result_cert->signatureWrap, issuer_pk, NULL);
     tor_assert(cert_ok == SECSuccess);
   }
-#endif
+#endif /* 1 */
 
  err:
   if (subject_spki)
@@ -455,4 +455,4 @@ tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
 
   return newcert ? tor_x509_cert_new(newcert) : NULL;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c
index a344279c22..2abf02851d 100644
--- a/src/lib/tls/x509_openssl.c
+++ b/src/lib/tls/x509_openssl.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,7 @@
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
  * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/opensslv.h>
 
@@ -36,7 +36,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/rsa.h>
 #include <openssl/x509.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -59,12 +59,12 @@ ENABLE_GCC_WARNING(redundant-decls)
 #define X509_get_notAfter(cert) \
     X509_getm_notAfter(cert)
 #endif
-#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
+#else /* !defined(OPENSSL_1_1_API) */
 #define X509_get_notBefore_const(cert) \
   ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
 #define X509_get_notAfter_const(cert) \
   ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 /** Return a newly allocated X509 name with commonName <b>cname</b>. */
 static X509_NAME *
diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h
index e35616cf50..87b3074e0b 100644
--- a/src/lib/trace/debug.h
+++ b/src/lib/trace/debug.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,4 +27,4 @@
                         "\"" XSTR(subsystem) "\" hit. " \
                         "(line "XSTR(__LINE__) ")")
 
-#endif /* TOR_TRACE_LOG_DEBUG_H */
+#endif /* !defined(TOR_TRACE_LOG_DEBUG_H) */
diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h
index 1e1e7b9d16..368f85dd02 100644
--- a/src/lib/trace/events.h
+++ b/src/lib/trace/events.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,12 +34,12 @@
 #include "lib/trace/debug.h"
 #endif
 
-#else /* TOR_EVENT_TRACING_ENABLED */
+#else /* !defined(TOR_EVENT_TRACING_ENABLED) */
 
 /* Reaching this point, we NOP every event declaration because event tracing
  * is not been enabled at compile time. */
 #define tor_trace(subsystem, name, args...)
 
-#endif /* TOR_EVENT_TRACING_ENABLED */
+#endif /* defined(TOR_EVENT_TRACING_ENABLED) */
 
-#endif /* TOR_TRACE_EVENTS_H */
+#endif /* !defined(TOR_TRACE_EVENTS_H) */
diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am
index 6f10c98744..98098c87f4 100644
--- a/src/lib/trace/include.am
+++ b/src/lib/trace/include.am
@@ -2,6 +2,7 @@
 noinst_LIBRARIES += \
 	src/lib/libtor-trace.a
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 TRACEHEADERS = \
 	src/lib/trace/trace.h \
 	src/lib/trace/events.h
@@ -11,7 +12,7 @@ TRACEHEADERS += \
 	src/lib/trace/debug.h
 endif
 
-# Library source files.
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_trace_a_SOURCES = \
 	src/lib/trace/trace.c
 
diff --git a/src/lib/trace/lib_trace.md b/src/lib/trace/lib_trace.md
new file mode 100644
index 0000000000..a7a32529b0
--- /dev/null
+++ b/src/lib/trace/lib_trace.md
@@ -0,0 +1,6 @@
+@dir /lib/trace
+@brief lib/trace: Function-tracing functionality API.
+
+This module is used for adding "trace" support (low-granularity function
+logging) to Tor.  Right now it doesn't have many users.
+
diff --git a/src/lib/trace/trace.c b/src/lib/trace/trace.c
index 18be63c5a8..4e5c66b4c6 100644
--- a/src/lib/trace/trace.c
+++ b/src/lib/trace/trace.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h
index 606d435568..5e24678c3c 100644
--- a/src/lib/trace/trace.h
+++ b/src/lib/trace/trace.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,4 +11,4 @@
 
 void tor_trace_init(void);
 
-#endif // TOR_TRACE_TRACE_H
+#endif /* !defined(TOR_TRACE_TRACE_H) */
diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include
new file mode 100644
index 0000000000..d159ceb41f
--- /dev/null
+++ b/src/lib/version/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+micro-revision.i
+lib/version/*.h
\ No newline at end of file
diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c
index 7d27549cad..09f11aa316 100644
--- a/src/lib/log/git_revision.c
+++ b/src/lib/version/git_revision.c
@@ -1,24 +1,39 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "lib/log/git_revision.h"
+#include "lib/version/git_revision.h"
+
+/**
+ * @file git_revision.c
+ * @brief Strings to describe the current Git commit.
+ **/
 
 /** String describing which Tor Git repository version the source was
  * built from.  This string is generated by a bit of shell kludging in
  * src/core/include.am, and is usually right.
  */
 const char tor_git_revision[] =
+#ifndef COCCI
 #ifndef _MSC_VER
 #include "micro-revision.i"
 #endif
+#endif
   "";
 
+/**
+ * String appended to Tor bug messages describing the Tor version.
+ *
+ * It has the form "(on Tor 0.4.3.1-alpha)" or
+ * "(on Tor 0.4.3.1-alpha git-b994397f1af193f8)"
+ **/
 const char tor_bug_suffix[] = " (on Tor " VERSION
+#ifndef COCCI
 #ifndef _MSC_VER
   " "
 #include "micro-revision.i"
 #endif
+#endif /* !defined(COCCI) */
   ")";
diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h
index 79e3c6684b..80b6c4734e 100644
--- a/src/lib/log/git_revision.h
+++ b/src/lib/version/git_revision.h
@@ -1,11 +1,16 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_GIT_REVISION_H
 #define TOR_GIT_REVISION_H
 
+/**
+ * @file git_revision.h
+ * @brief Header for git_revision.c
+ **/
+
 extern const char tor_git_revision[];
 extern const char tor_bug_suffix[];
 
diff --git a/src/lib/version/include.am b/src/lib/version/include.am
new file mode 100644
index 0000000000..0ae31be1b2
--- /dev/null
+++ b/src/lib/version/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-version.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-version-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_version_a_SOURCES =			\
+	src/lib/version/git_revision.c			\
+	src/lib/version/version.c
+
+src_lib_libtor_version_testing_a_SOURCES = \
+	$(src_lib_libtor_version_a_SOURCES)
+src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# Declare that these object files depend on micro-revision.i.  Without this
+# rule, we could try to build them before micro-revision.i was created.
+src/lib/version/git_revision.$(OBJEXT) \
+  src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \
+	micro-revision.i
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/version/git_revision.h			\
+	src/lib/version/torversion.h
diff --git a/src/lib/version/lib_version.md b/src/lib/version/lib_version.md
new file mode 100644
index 0000000000..ccc45920f9
--- /dev/null
+++ b/src/lib/version/lib_version.md
@@ -0,0 +1,2 @@
+@dir /lib/version
+@brief lib/version: holds the current version of Tor.
diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h
new file mode 100644
index 0000000000..679df74381
--- /dev/null
+++ b/src/lib/version/torversion.h
@@ -0,0 +1,17 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_VERSION_H
+#define TOR_VERSION_H
+
+/**
+ * @file torversion.h
+ * @brief Header for version.c.
+ **/
+
+const char *get_version(void);
+const char *get_short_version(void);
+
+#endif /* !defined(TOR_VERSION_H) */
diff --git a/src/lib/version/version.c b/src/lib/version/version.c
new file mode 100644
index 0000000000..ec1d0bea2f
--- /dev/null
+++ b/src/lib/version/version.c
@@ -0,0 +1,59 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/version/torversion.h"
+#include "lib/version/git_revision.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * @file version.c
+ * @brief Functions to get the version of Tor.
+ **/
+
+/** A shorter version of this Tor process's version, for export in our router
+ *  descriptor.  (Does not include the git version, if any.) */
+static const char the_short_tor_version[] =
+  VERSION
+#ifdef TOR_BUILD_TAG
+  " ("TOR_BUILD_TAG")"
+#endif
+  "";
+
+/**
+ * Longest possible version length. We make this a constant so that we
+ * can statically allocate the_tor_version.
+ **/
+#define MAX_VERSION_LEN 128
+
+/** The version of this Tor process, possibly including git version */
+static char the_tor_version[MAX_VERSION_LEN] = "";
+
+/** Return the current Tor version. */
+const char *
+get_version(void)
+{
+  if (the_tor_version[0] == 0) {
+    if (strlen(tor_git_revision)) {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s (git-%s)", the_short_tor_version, tor_git_revision);
+    } else {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s", the_short_tor_version);
+    }
+    the_tor_version[sizeof(the_tor_version)-1] = 0;
+  }
+
+  return the_tor_version;
+}
+
+/** Return the current Tor version, without any git tag. */
+const char *
+get_short_version(void)
+{
+  return the_short_tor_version;
+}
diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include
index dc010da063..ce7a26472b 100644
--- a/src/lib/wallclock/.may_include
+++ b/src/lib/wallclock/.may_include
@@ -3,4 +3,5 @@ lib/cc/*.h
 lib/err/*.h
 lib/wallclock/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c
index ee498702d5..c815f20e51 100644
--- a/src/lib/wallclock/approx_time.c
+++ b/src/lib/wallclock/approx_time.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,7 +9,9 @@
  **/
 
 #include "orconfig.h"
+#include "lib/subsys/subsys.h"
 #include "lib/wallclock/approx_time.h"
+#include "lib/wallclock/wallclock_sys.h"
 
 /* =====
  * Cached time
@@ -41,3 +43,26 @@ update_approx_time(time_t now)
   cached_approx_time = now;
 }
 #endif /* !defined(TIME_IS_FAST) */
+
+/**
+ * Initialize the "wallclock" subsystem by setting the current cached time.
+ **/
+static int
+subsys_wallclock_initialize(void)
+{
+  update_approx_time(time(NULL));
+  return 0;
+}
+
+/**
+ * Subsystem function table describing the "wallclock" subsystem.
+ **/
+const subsys_fns_t sys_wallclock = {
+  .name = "wallclock",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  /* Approximate time is a diagnostic feature, we want it to init right after
+   * low-level error handling. */
+  .level = -98,
+  .initialize = subsys_wallclock_initialize,
+};
diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h
index e6b53f2c27..42040a1f52 100644
--- a/src/lib/wallclock/approx_time.h
+++ b/src/lib/wallclock/approx_time.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,4 @@ time_t approx_time(void);
 void update_approx_time(time_t now);
 #endif /* defined(TIME_IS_FAST) */
 
-#endif
+#endif /* !defined(TOR_APPROX_TIME_H) */
diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am
index 1961639bd7..2b50d6ccbb 100644
--- a/src/lib/wallclock/include.am
+++ b/src/lib/wallclock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_wallclock_a_SOURCES =			\
 	src/lib/wallclock/approx_time.c			\
 	src/lib/wallclock/time_to_tm.c			\
@@ -15,8 +16,10 @@ src_lib_libtor_wallclock_testing_a_SOURCES = \
 src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/wallclock/approx_time.h			\
 	src/lib/wallclock/timeval.h			\
 	src/lib/wallclock/time_to_tm.h			\
-	src/lib/wallclock/tor_gettimeofday.h
+	src/lib/wallclock/tor_gettimeofday.h		\
+	src/lib/wallclock/wallclock_sys.h
diff --git a/src/lib/wallclock/lib_wallclock.md b/src/lib/wallclock/lib_wallclock.md
new file mode 100644
index 0000000000..f21721f6f6
--- /dev/null
+++ b/src/lib/wallclock/lib_wallclock.md
@@ -0,0 +1,11 @@
+@dir /lib/wallclock
+@brief lib/wallclock: Inspect and manipulate the current time.
+
+This module handles our concept of "what time is it" or "what time does the
+world agree it is?"  Generally, if you want something derived from UTC, this
+is the module for you.
+
+For versions of the time that are more local, more monotonic, or more
+accurate, see \refdir{lib/time}.  For parsing and encoding times and dates,
+see \refdir{lib/encoding}.
+
diff --git a/src/lib/wallclock/time_to_tm.c b/src/lib/wallclock/time_to_tm.c
index f7cb21827b..8c747b4c7b 100644
--- a/src/lib/wallclock/time_to_tm.c
+++ b/src/lib/wallclock/time_to_tm.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -198,3 +198,4 @@ tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
   return correct_tm(0, timep, result, r, err_out);
 }
 #endif /* defined(HAVE_GMTIME_R) || ... */
+/**@}*/
diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h
index abe78c0efe..bfa8fa3689 100644
--- a/src/lib/wallclock/time_to_tm.h
+++ b/src/lib/wallclock/time_to_tm.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result,
 struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result,
                             char **err_out);
 
-#endif
+#endif /* !defined(TOR_WALLCLOCK_TIME_TO_TM_H) */
diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h
index 4967e939bf..d7d5bda99f 100644
--- a/src/lib/wallclock/timeval.h
+++ b/src/lib/wallclock/timeval.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,6 +20,27 @@
 #include <sys/time.h>
 #endif
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we use a slower definition of these macros without
+ * branches, to make coverage consistent. */
+#undef timeradd
+#undef timersub
+#define timeradd(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec;    \
+    (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;      \
+    (tvout)->tv_usec %= 1000000;                        \
+  } while (0)
+#define timersub(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec - 1;            \
+    (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec + 1000000;   \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;                  \
+    (tvout)->tv_usec %= 1000000;                                    \
+  } while (0)
+#endif /* defined(TOR_COVERAGE) */
+
 #ifndef timeradd
 /** Replacement for timeradd on platforms that do not have it: sets tvout to
  * the sum of tv1 and tv2. */
@@ -48,6 +69,7 @@
   } while (0)
 #endif /* !defined(timersub) */
 
+#ifndef COCCI
 #ifndef timercmp
 /** Replacement for timercmp on platforms that do not have it: returns true
  * iff the relational operator "op" makes the expression tv1 op tv2 true.
@@ -61,5 +83,6 @@
    ((tv1)->tv_usec op (tv2)->tv_usec) :         \
    ((tv1)->tv_sec op (tv2)->tv_sec))
 #endif /* !defined(timercmp) */
+#endif /* !defined(COCCI) */
 
-#endif
+#endif /* !defined(TOR_TIMEVAL_H) */
diff --git a/src/lib/wallclock/tor_gettimeofday.c b/src/lib/wallclock/tor_gettimeofday.c
index 63538f3b81..a07f83220d 100644
--- a/src/lib/wallclock/tor_gettimeofday.c
+++ b/src/lib/wallclock/tor_gettimeofday.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h
index c7fff9747a..c1a8afca3a 100644
--- a/src/lib/wallclock/tor_gettimeofday.h
+++ b/src/lib/wallclock/tor_gettimeofday.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@ struct timeval;
 
 MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
 
-#endif
+#endif /* !defined(TOR_GETTIMEOFDAY_H) */
diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h
new file mode 100644
index 0000000000..3997d11e7a
--- /dev/null
+++ b/src/lib/wallclock/wallclock_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file wallclock_sys.h
+ * \brief Declare subsystem object for the wallclock module.
+ **/
+
+#ifndef TOR_WALLCLOCK_SYS_H
+#define TOR_WALLCLOCK_SYS_H
+
+extern const struct subsys_fns_t sys_wallclock;
+
+#endif /* !defined(TOR_WALLCLOCK_SYS_H) */
diff --git a/src/mainpage.md b/src/mainpage.md
new file mode 100644
index 0000000000..2c4c494354
--- /dev/null
+++ b/src/mainpage.md
@@ -0,0 +1,150 @@
+@mainpage Tor source reference
+
+@tableofcontents
+
+@section welcome Welcome to Tor
+
+(For an up-to-date rendered copy of this documentation, see
+https://src-ref.docs.torproject.org/tor/index.html .)
+
+This documentation describes the general structure of the Tor codebase, how
+it fits together, what functionality is available for extending Tor, and
+gives some notes on how Tor got that way.  It also includes a reference for
+nearly every function, type, file, and module in the Tor source code.  The
+high-level documentation is a work in progress.
+
+Tor itself remains a work in progress too: We've been working on it for
+nearly two decades, and we've learned a lot about good coding since we first
+started.  This means, however, that some of the older pieces of Tor will have
+some "code smell" in them that could stand a brisk refactoring.  So when we
+describe a piece of code, we'll sometimes give a note on how it got that way,
+and whether we still think that's a good idea.
+
+This document is not an overview of the Tor protocol.  For that, see the
+design paper and the specifications at https://spec.torproject.org/ .
+
+For more information about Tor's coding standards and some helpful
+development tools, see
+[doc/HACKING](https://gitweb.torproject.org/tor.git/tree/doc/HACKING) in the
+Tor repository.
+
+@section topics Topic-related documentation
+
+@subpage intro
+
+@subpage arch_goals
+
+@subpage initialization
+
+@subpage dataflow
+
+@subpage certificates
+
+@subpage threading
+
+@subpage strings
+
+@subpage time_periodic
+
+@subpage configuration
+
+@subpage publish_subscribe
+
+@page intro A high-level overview
+
+@tableofcontents
+
+@section highlevel The very high level
+
+Ultimately, Tor runs as an event-driven network daemon: it responds to
+network events, signals, and timers by sending and receiving things over
+the network.  Clients, relays, and directory authorities all use the
+same codebase: the Tor process will run as a client, relay, or authority
+depending on its configuration.
+
+Tor has a few major dependencies, including Libevent (used to tell which
+sockets are readable and writable), OpenSSL or NSS (used for many encryption
+functions, and to implement the TLS protocol), and zlib (used to
+compress and uncompress directory information).
+
+Most of Tor's work today is done in a single event-driven main thread.
+Tor also spawns one or more worker threads to handle CPU-intensive
+tasks.  (Right now, this only includes circuit encryption and the more
+expensive compression algorithms.)
+
+On startup, Tor initializes its libraries, reads and responds to its
+configuration files, and launches a main event loop.  At first, the only
+events that Tor listens for are a few signals (like TERM and HUP), and
+one or more listener sockets (for different kinds of incoming
+connections).  Tor also configures several timers to handle periodic
+events.  As Tor runs over time, other events will open, and new events
+will be scheduled.
+
+The codebase is divided into a few top-level subdirectories, each of
+which contains several sub-modules.
+
+   - `ext` -- Code maintained elsewhere that we include in the Tor
+     source distribution.
+
+   - \refdir{lib} -- Lower-level utility code, not necessarily
+     tor-specific.
+
+   - `trunnel` -- Automatically generated code (from the Trunnel
+     tool): used to parse and encode binary formats.
+
+   - \refdir{core} -- Networking code that is implements the central
+     parts of the Tor protocol and main loop.
+
+   - \refdir{feature} -- Aspects of Tor (like directory management,
+     running a relay, running a directory authorities, managing a list of
+     nodes, running and using onion services) that are built on top of the
+     mainloop code.
+
+   - \refdir{app} -- Highest-level functionality; responsible for setting
+     up and configuring the Tor daemon, making sure all the lower-level
+     modules start up when required, and so on.
+
+   - \refdir{tools} -- Binaries other than Tor that we produce.
+      Currently this is tor-resolve, tor-gencert, and the tor_runner.o helper
+      module.
+
+   - `test` -- unit tests, regression tests, and a few integration
+     tests.
+
+In theory, the above parts of the codebase are sorted from highest-level to
+lowest-level, where high-level code is only allowed to invoke lower-level
+code, and lower-level code never includes or depends on code of a higher
+level.  In practice, this refactoring is incomplete: The modules in
+\refdir{lib} are well-factored, but there are many layer violations ("upward
+dependencies") in \refdir{core} and \refdir{feature}.
+We aim to eliminate those over time.
+
+@section keyabstractions Some key high-level abstractions
+
+The most important abstractions at Tor's high-level are Connections,
+Channels, Circuits, and Nodes.
+
+A 'Connection' (connection_t) represents a stream-based information flow.
+Most connections are TCP connections to remote Tor servers and clients. (But
+as a shortcut, a relay will sometimes make a connection to itself without
+actually using a TCP connection.  More details later on.)  Connections exist
+in different varieties, depending on what functionality they provide.  The
+principle types of connection are edge_connection_t (eg a socks connection or
+a connection from an exit relay to a destination), or_connection_t (a TLS
+stream connecting to a relay), dir_connection_t (an HTTP connection to learn
+about the network), and control_connection_t (a connection from a
+controller).
+
+A 'Circuit' (circuit_t) is persistent tunnel through the Tor network,
+established with public-key cryptography, and used to send cells one or more
+hops.  Clients keep track of multi-hop circuits (origin_circuit_t), and the
+cryptography associated with each hop.  Relays, on the other hand, keep track
+only of their hop of each circuit (or_circuit_t).
+
+A 'Channel' (channel_t) is an abstract view of sending cells to and from a
+Tor relay.  Currently, all channels are implemented using OR connections
+(channel_tls_t).  If we switch to other strategies in the future, we'll have
+more connection types.
+
+A 'Node' (node_t) is a view of a Tor instance's current knowledge and opinions
+about a Tor relay or bridge.
diff --git a/src/rust/build.rs b/src/rust/build.rs
index 123d5c0682..5626b35f75 100644
--- a/src/rust/build.rs
+++ b/src/rust/build.rs
@@ -149,8 +149,9 @@ pub fn main() {
             cfg.component("tor-sandbox-testing");
             cfg.component("tor-encoding-testing");
             cfg.component("tor-fs-testing");
-            cfg.component("tor-time-testing");
             cfg.component("tor-net-testing");
+            cfg.component("tor-buf-testing");
+            cfg.component("tor-time-testing");
             cfg.component("tor-thread-testing");
             cfg.component("tor-memarea-testing");
             cfg.component("tor-log-testing");
@@ -162,6 +163,7 @@ pub fn main() {
             cfg.component("tor-malloc");
             cfg.component("tor-wallclock");
             cfg.component("tor-err-testing");
+            cfg.component("tor-version-testing");
             cfg.component("tor-intmath-testing");
             cfg.component("tor-ctime-testing");
             cfg.component("curve25519_donna");
diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs
index 6ee63adb10..14170d0353 100644
--- a/src/rust/protover/ffi.rs
+++ b/src/rust/protover/ffi.rs
@@ -30,6 +30,8 @@ fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> {
         7 => Ok(Protocol::Desc),
         8 => Ok(Protocol::Microdesc),
         9 => Ok(Protocol::Cons),
+        10 => Ok(Protocol::Padding),
+        11 => Ok(Protocol::FlowCtrl),
         _ => Err(ProtoverError::UnknownProtocol),
     }
 }
diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs
index 536667f61b..06c4dd2398 100644
--- a/src/rust/protover/protover.rs
+++ b/src/rust/protover/protover.rs
@@ -46,6 +46,8 @@ pub enum Protocol {
     LinkAuth,
     Microdesc,
     Relay,
+    Padding,
+    FlowCtrl,
 }
 
 impl fmt::Display for Protocol {
@@ -73,6 +75,8 @@ impl FromStr for Protocol {
             "LinkAuth" => Ok(Protocol::LinkAuth),
             "Microdesc" => Ok(Protocol::Microdesc),
             "Relay" => Ok(Protocol::Relay),
+            "Padding" => Ok(Protocol::Padding),
+            "FlowCtrl" => Ok(Protocol::FlowCtrl),
             _ => Err(ProtoverError::UnknownProtocol),
         }
     }
@@ -157,12 +161,14 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
             "Cons=1-2 \
              Desc=1-2 \
              DirCache=1-2 \
+             FlowCtrl=1 \
              HSDir=1-2 \
              HSIntro=3-4 \
              HSRend=1-2 \
              Link=1-5 \
              LinkAuth=3 \
              Microdesc=1-2 \
+             Padding=2 \
              Relay=1-2"
         )
     } else {
@@ -170,12 +176,14 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
             "Cons=1-2 \
              Desc=1-2 \
              DirCache=1-2 \
+             FlowCtrl=1 \
              HSDir=1-2 \
              HSIntro=3-4 \
              HSRend=1-2 \
              Link=1-5 \
              LinkAuth=1,3 \
              Microdesc=1-2 \
+             Padding=2 \
              Relay=1-2"
         )
     }
diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs
index 98fccba5a9..bbaf97129c 100644
--- a/src/rust/tor_log/tor_log.rs
+++ b/src/rust/tor_log/tor_log.rs
@@ -99,14 +99,14 @@ pub mod log {
     /// Domain log types. These mirror definitions in src/lib/log/log.h
     /// C_RUST_COUPLED: src/lib/log/log.c, log severity types
     extern "C" {
-        static LD_NET_: u32;
-        static LD_GENERAL_: u32;
+        static LD_NET_: u64;
+        static LD_GENERAL_: u64;
     }
 
     /// Translate Rust defintions of log domain levels to C. This exposes a 1:1
     /// mapping between types.
     #[inline]
-    pub unsafe fn translate_domain(domain: LogDomain) -> u32 {
+    pub unsafe fn translate_domain(domain: LogDomain) -> u64 {
         match domain {
             LogDomain::Net => LD_NET_,
             LogDomain::General => LD_GENERAL_,
@@ -128,7 +128,7 @@ pub mod log {
     extern "C" {
         pub fn tor_log_string(
             severity: c_int,
-            domain: u32,
+            domain: u64,
             function: *const c_char,
             string: *const c_char,
         );
diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs
index 2a19458f46..ede42c6ea8 100644
--- a/src/rust/tor_util/strings.rs
+++ b/src/rust/tor_util/strings.rs
@@ -105,11 +105,7 @@ macro_rules! cstr {
     ($($bytes:expr),*) => (
         ::std::ffi::CStr::from_bytes_with_nul(
             concat!($($bytes),*, "\0").as_bytes()
-        ).unwrap_or(
-            unsafe{
-                ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0")
-            }
-        )
+        ).unwrap_or_default()
     )
 }
 
diff --git a/src/test/.may_include b/src/test/.may_include
new file mode 100644
index 0000000000..11c5ffbb14
--- /dev/null
+++ b/src/test/.may_include
@@ -0,0 +1,2 @@
+*.h
+*.inc
diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake
index cfbe281b94..ca6a84cf8a 100644
--- a/src/test/Makefile.nmake
+++ b/src/test/Makefile.nmake
@@ -1,4 +1,4 @@
-all: test.exe test-child.exe bench.exe
+all: test.exe bench.exe
 
 CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \
     /I ..\ext
@@ -19,6 +19,7 @@ TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \
 	test_cell_formats.obj test_relay.obj test_replay.obj \
 	test_channelpadding.obj \
 	test_circuitstats.obj \
+	test_circuitpadding.obj \
 	test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj
 
 tinytest.obj: ..\ext\tinytest.c
@@ -30,8 +31,5 @@ test.exe: $(TEST_OBJECTS)
 bench.exe: bench.obj
 	$(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@
 
-test-child.exe: test-child.obj
-	$(CC) $(CFLAGS) test-child.obj /Fe$@
-
 clean:
-	del *.obj *.lib test.exe bench.exe test-child.exe
+	del *.obj *.lib test.exe bench.exe
diff --git a/src/test/bench.c b/src/test/bench.c
index 06c616c3b0..7a8c04e802 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,16 +14,19 @@
 #include "core/crypto/onion_tap.h"
 #include "core/crypto/relay_crypto.h"
 
+#include "lib/intmath/weakrng.h"
+
 #ifdef ENABLE_OPENSSL
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #include <openssl/ec.h>
 #include <openssl/ecdh.h>
 #include <openssl/obj_mac.h>
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "core/or/circuitlist.h"
 #include "app/config/config.h"
+#include "app/main/subsysmgr.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "core/crypto/onion_ntor.h"
@@ -38,6 +41,9 @@
 #include "lib/crypt_ops/digestset.h"
 #include "lib/crypt_ops/crypto_init.h"
 
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/nodelist/microdesc.h"
+
 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
 static uint64_t nanostart;
 static inline uint64_t
@@ -332,6 +338,65 @@ bench_ed25519(void)
 }
 
 static void
+bench_rand_len(int len)
+{
+  const int N = 100000;
+  int i;
+  char *buf = tor_malloc(len);
+  uint64_t start,end;
+
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_rand(buf, len);
+  }
+  end = perftime();
+  printf("crypto_rand(%d): %f nsec.\n", len, NANOCOUNT(start,end,N));
+
+  crypto_fast_rng_t *fr = crypto_fast_rng_new();
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_fast_rng_getbytes(fr,(uint8_t*)buf,len);
+  }
+  end = perftime();
+  printf("crypto_fast_rng_getbytes(%d): %f nsec.\n", len,
+         NANOCOUNT(start,end,N));
+  crypto_fast_rng_free(fr);
+
+  if (len <= 32) {
+    start = perftime();
+    for (i = 0; i < N; ++i) {
+      crypto_strongest_rand((uint8_t*)buf, len);
+    }
+    end = perftime();
+    printf("crypto_strongest_rand(%d): %f nsec.\n", len,
+           NANOCOUNT(start,end,N));
+  }
+
+  if (len == 4) {
+    tor_weak_rng_t weak;
+    tor_init_weak_random(&weak, 1337);
+
+    start = perftime();
+    uint32_t t=0;
+    for (i = 0; i < N; ++i) {
+      t += tor_weak_random(&weak);
+    }
+    end = perftime();
+    printf("weak_rand(4): %f nsec.\n", NANOCOUNT(start,end,N));
+  }
+
+  tor_free(buf);
+}
+
+static void
+bench_rand(void)
+{
+  bench_rand_len(4);
+  bench_rand_len(16);
+  bench_rand_len(128);
+}
+
+static void
 bench_cell_aes(void)
 {
   uint64_t start, end;
@@ -636,7 +701,42 @@ bench_ecdh_p224(void)
 {
   bench_ecdh_impl(NID_secp224r1, "P-224");
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+
+static void
+bench_md_parse(void)
+{
+  uint64_t start, end;
+  const int N = 100000;
+  // selected arbitrarily
+  const char md_text[] =
+    "@last-listed 2018-12-14 18:14:14\n"
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n"
+    "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n"
+    "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n"
+    "family $00E9A86E7733240E60D8435A7BBD634A23894098 "
+    "$329BD7545DEEEBBDC8C4285F243916F248972102 "
+    "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D "
+    "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n"
+    "p accept 53,80,443,5222-5223,25565\n"
+    "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n";
+
+  reset_perftime();
+  start = perftime();
+  for (int i = 0; i < N; ++i) {
+    smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1,
+                                                  SAVED_IN_CACHE, NULL);
+    SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md));
+    smartlist_free(s);
+  }
+
+  end = perftime();
+  printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N));
+}
 
 typedef void (*bench_fn)(void);
 
@@ -656,6 +756,7 @@ static struct benchmark_t benchmarks[] = {
   ENT(onion_TAP),
   ENT(onion_ntor),
   ENT(ed25519),
+  ENT(rand),
 
   ENT(cell_aes),
   ENT(cell_ops),
@@ -665,6 +766,8 @@ static struct benchmark_t benchmarks[] = {
   ENT(ecdh_p256),
   ENT(ecdh_p224),
 #endif
+
+  ENT(md_parse),
   {NULL,NULL,0}
 };
 
@@ -690,9 +793,10 @@ main(int argc, const char **argv)
   char *errmsg;
   or_options_t *options;
 
-  tor_threads_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
+
   tor_compress_init();
-  init_logging(1);
 
   if (argc == 4 && !strcmp(argv[1], "diff")) {
     const int N = 200;
@@ -702,11 +806,13 @@ main(int argc, const char **argv)
       perror("X");
       return 1;
     }
+    size_t f1len = strlen(f1);
+    size_t f2len = strlen(f2);
     for (i = 0; i < N; ++i) {
-      char *diff = consensus_diff_generate(f1, f2);
+      char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
       tor_free(diff);
     }
-    char *diff = consensus_diff_generate(f1, f2);
+    char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
     printf("%s", diff);
     tor_free(f1);
     tor_free(f2);
@@ -737,7 +843,6 @@ main(int argc, const char **argv)
 
   init_protocol_warning_severity_level();
   options = options_new();
-  init_logging(1);
   options->command = CMD_RUN_UNITTESTS;
   options->DataDirectory = tor_strdup("");
   options->KeyDirectory = tor_strdup("");
diff --git a/src/test/bt_test.py b/src/test/bt_test.py
index f9ca79efde..d728f13596 100755
--- a/src/test/bt_test.py
+++ b/src/test/bt_test.py
@@ -15,7 +15,11 @@ OK
 
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 
 
diff --git a/src/test/conf_examples/badnick_1/error b/src/test/conf_examples/badnick_1/error
new file mode 100644
index 0000000000..3e92ddc832
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/error
@@ -0,0 +1 @@
+nicknames must be between 1 and 19 characters inclusive
diff --git a/src/test/conf_examples/badnick_1/expected_log_no_dirauth_relay b/src/test/conf_examples/badnick_1/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..9190a3326b
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*badnick_1[./]*torrc
diff --git a/src/test/conf_examples/badnick_1/expected_no_dirauth_relay b/src/test/conf_examples/badnick_1/expected_no_dirauth_relay
new file mode 100644
index 0000000000..b00be15c2e
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/expected_no_dirauth_relay
@@ -0,0 +1 @@
+Nickname TooManyCharactersInThisNickname
diff --git a/src/test/conf_examples/badnick_1/torrc b/src/test/conf_examples/badnick_1/torrc
new file mode 100644
index 0000000000..087e3f2ff1
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/torrc
@@ -0,0 +1,4 @@
+# This nickname is too long; we won't accept it.
+# (Unless the relay module is disabled, because Nickname is a
+# relay-only option. We'll ignore all relay-only options in #32395.)
+Nickname TooManyCharactersInThisNickname
diff --git a/src/test/conf_examples/badnick_2/error b/src/test/conf_examples/badnick_2/error
new file mode 100644
index 0000000000..ceac99f012
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/error
@@ -0,0 +1 @@
+must contain only the characters \[a-zA-Z0-9\]
diff --git a/src/test/conf_examples/badnick_2/expected_log_no_dirauth_relay b/src/test/conf_examples/badnick_2/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..a15c7b02cb
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*badnick_2[./]*torrc
diff --git a/src/test/conf_examples/badnick_2/expected_no_dirauth_relay b/src/test/conf_examples/badnick_2/expected_no_dirauth_relay
new file mode 100644
index 0000000000..08dcdc33a9
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/expected_no_dirauth_relay
@@ -0,0 +1 @@
+Nickname has a space
diff --git a/src/test/conf_examples/badnick_2/torrc b/src/test/conf_examples/badnick_2/torrc
new file mode 100644
index 0000000000..51a5f96c00
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/torrc
@@ -0,0 +1,4 @@
+# this nickname has spaces in it and won't work.
+# (Unless the relay module is disabled, because Nickname is a
+# relay-only option. We'll ignore all relay-only options in #32395.)
+Nickname has a space
diff --git a/src/test/conf_examples/bridgeauth_1/error_no_dirauth b/src/test/conf_examples/bridgeauth_1/error_no_dirauth
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/error_no_dirauth
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/bridgeauth_1/error_no_dirauth_relay b/src/test/conf_examples/bridgeauth_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/bridgeauth_1/expected b/src/test/conf_examples/bridgeauth_1/expected
new file mode 100644
index 0000000000..d43aaf2c8b
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/expected
@@ -0,0 +1,7 @@
+Address 198.51.100.123
+AuthoritativeDirectory 1
+BridgeAuthoritativeDir 1
+ContactInfo tor_parse_test@example.com
+DirPort 80
+Nickname Unnamed
+ORPort 443
diff --git a/src/test/conf_examples/bridgeauth_1/expected_log b/src/test/conf_examples/bridgeauth_1/expected_log
new file mode 100644
index 0000000000..cabe9d3f89
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*bridgeauth_1[./]*torrc
diff --git a/src/test/conf_examples/bridgeauth_1/torrc b/src/test/conf_examples/bridgeauth_1/torrc
new file mode 100644
index 0000000000..740bc6c2eb
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/torrc
@@ -0,0 +1,8 @@
+AuthoritativeDirectory 1
+BridgeAuthoritativeDir 1
+
+ContactInfo tor_parse_test@example.com
+
+Address 198.51.100.123
+ORPort 443
+DirPort 80
diff --git a/src/test/conf_examples/contactinfo_notutf8/error b/src/test/conf_examples/contactinfo_notutf8/error
new file mode 100644
index 0000000000..6d165152ce
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/error
@@ -0,0 +1 @@
+ContactInfo config option must be UTF-8
diff --git a/src/test/conf_examples/contactinfo_notutf8/expected_log_no_dirauth_relay b/src/test/conf_examples/contactinfo_notutf8/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..caa07aca40
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*contactinfo_notutf8[./]*torrc
diff --git a/src/test/conf_examples/contactinfo_notutf8/expected_no_dirauth_relay b/src/test/conf_examples/contactinfo_notutf8/expected_no_dirauth_relay
new file mode 100644
index 0000000000..cc8bd7b8e3
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/expected_no_dirauth_relay
@@ -0,0 +1 @@
+ContactInfo "\304\353\304\353\304\353@example.com"
diff --git a/src/test/conf_examples/contactinfo_notutf8/torrc b/src/test/conf_examples/contactinfo_notutf8/torrc
new file mode 100644
index 0000000000..0176a56a97
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/torrc
@@ -0,0 +1,5 @@
+# We only accept ContactInfo in UTF-8 (or 7-bit ASCII)
+# (Unless the relay module is disabled, because ContactInfo is a
+# relay-only option. We'll ignore all relay-only options in #32395.
+# But at the moment, tor canonicalises and quotes the string.)
+ContactInfo ÄëÄëÄë@example.com
diff --git a/src/test/conf_examples/controlsock/error b/src/test/conf_examples/controlsock/error
new file mode 100644
index 0000000000..8fbea37894
--- /dev/null
+++ b/src/test/conf_examples/controlsock/error
@@ -0,0 +1 @@
+not supported on this OS\|without setting a ControlSocket
diff --git a/src/test/conf_examples/controlsock/torrc b/src/test/conf_examples/controlsock/torrc
new file mode 100644
index 0000000000..dd3cb7ede5
--- /dev/null
+++ b/src/test/conf_examples/controlsock/torrc
@@ -0,0 +1 @@
+ControlSocketsGroupWritable 1
diff --git a/src/test/conf_examples/crypto_accel/expected b/src/test/conf_examples/crypto_accel/expected
new file mode 100644
index 0000000000..ea80ca19dc
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected
@@ -0,0 +1,2 @@
+AccelName nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel/expected_log b/src/test/conf_examples/crypto_accel/expected_log
new file mode 100644
index 0000000000..7fab0c8dad
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected_log
@@ -0,0 +1 @@
+Unable to load dynamic OpenSSL engine "nonexistent_chartreuse_accelerator"
diff --git a/src/test/conf_examples/crypto_accel/expected_log_nss b/src/test/conf_examples/crypto_accel/expected_log_nss
new file mode 100644
index 0000000000..c0fe7b003c
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected_log_nss
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, NSS .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/crypto_accel/expected_nss b/src/test/conf_examples/crypto_accel/expected_nss
new file mode 100644
index 0000000000..ea80ca19dc
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected_nss
@@ -0,0 +1,2 @@
+AccelName nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel/torrc b/src/test/conf_examples/crypto_accel/torrc
new file mode 100644
index 0000000000..9ca18903b7
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/torrc
@@ -0,0 +1,3 @@
+
+AccelName nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel_req/error b/src/test/conf_examples/crypto_accel_req/error
new file mode 100644
index 0000000000..e12e002915
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/error
@@ -0,0 +1 @@
+Unable to load required dynamic OpenSSL engine "nonexistent_chartreuse_accelerator"
diff --git a/src/test/conf_examples/crypto_accel_req/expected_log_nss b/src/test/conf_examples/crypto_accel_req/expected_log_nss
new file mode 100644
index 0000000000..c0fe7b003c
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/expected_log_nss
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, NSS .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/crypto_accel_req/expected_nss b/src/test/conf_examples/crypto_accel_req/expected_nss
new file mode 100644
index 0000000000..f3e172f640
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/expected_nss
@@ -0,0 +1,2 @@
+AccelName !nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel_req/torrc b/src/test/conf_examples/crypto_accel_req/torrc
new file mode 100644
index 0000000000..981d9116fc
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/torrc
@@ -0,0 +1,3 @@
+
+AccelName !nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/dirauth_1/error_no_dirauth b/src/test/conf_examples/dirauth_1/error_no_dirauth
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/error_no_dirauth
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/dirauth_1/error_no_dirauth_relay b/src/test/conf_examples/dirauth_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/dirauth_1/expected b/src/test/conf_examples/dirauth_1/expected
new file mode 100644
index 0000000000..901f7d947f
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/expected
@@ -0,0 +1,8 @@
+Address 192.0.2.1
+AuthoritativeDirectory 1
+ContactInfo tor_parse_test@example.net
+DirPort 9030
+DownloadExtraInfo 1
+Nickname Unnamed
+ORPort 9001
+V3AuthoritativeDirectory 1
diff --git a/src/test/conf_examples/dirauth_1/expected_log b/src/test/conf_examples/dirauth_1/expected_log
new file mode 100644
index 0000000000..b788be2e33
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*dirauth_1[./]*torrc
diff --git a/src/test/conf_examples/dirauth_1/torrc b/src/test/conf_examples/dirauth_1/torrc
new file mode 100644
index 0000000000..b870e6e8e0
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/torrc
@@ -0,0 +1,8 @@
+AuthoritativeDirectory 1
+V3AuthoritativeDirectory 1
+
+ContactInfo tor_parse_test@example.net
+
+Address 192.0.2.1
+ORPort 9001
+DirPort 9030
diff --git a/src/test/conf_examples/dirauth_2/expected b/src/test/conf_examples/dirauth_2/expected
new file mode 100644
index 0000000000..19ab024ed3
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected
@@ -0,0 +1 @@
+AuthDirMaxServersPerAddr 8
diff --git a/src/test/conf_examples/dirauth_2/expected_log b/src/test/conf_examples/dirauth_2/expected_log
new file mode 100644
index 0000000000..88611fee9d
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_log
@@ -0,0 +1 @@
+Read configuration file
diff --git a/src/test/conf_examples/dirauth_2/expected_log_no_dirauth b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth
new file mode 100644
index 0000000000..01110c5d8c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth
@@ -0,0 +1 @@
+This copy of Tor was built without support for the option "AuthDirMaxServersPerAddr". Skipping.
\ No newline at end of file
diff --git a/src/test/conf_examples/dirauth_2/expected_log_no_dirauth_relay b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..01110c5d8c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+This copy of Tor was built without support for the option "AuthDirMaxServersPerAddr". Skipping.
\ No newline at end of file
diff --git a/src/test/conf_examples/dirauth_2/expected_no_dirauth b/src/test/conf_examples/dirauth_2/expected_no_dirauth
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_no_dirauth
diff --git a/src/test/conf_examples/dirauth_2/expected_no_dirauth_relay b/src/test/conf_examples/dirauth_2/expected_no_dirauth_relay
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_no_dirauth_relay
diff --git a/src/test/conf_examples/dirauth_2/torrc b/src/test/conf_examples/dirauth_2/torrc
new file mode 100644
index 0000000000..bd1cdbc8b9
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/torrc
@@ -0,0 +1,5 @@
+#
+# This will get accepted if the module is enabled, and ignored if the module
+# is disabled.
+#
+AuthDirMaxServersPerAddr 8
diff --git a/src/test/conf_examples/empty_1/expected b/src/test/conf_examples/empty_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_1/expected
diff --git a/src/test/conf_examples/empty_1/expected_log b/src/test/conf_examples/empty_1/expected_log
new file mode 100644
index 0000000000..4c6b00069f
--- /dev/null
+++ b/src/test/conf_examples/empty_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*empty_1[./]*torrc
diff --git a/src/test/conf_examples/empty_1/torrc b/src/test/conf_examples/empty_1/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_1/torrc
diff --git a/src/test/conf_examples/empty_2/cmdline b/src/test/conf_examples/empty_2/cmdline
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/cmdline
diff --git a/src/test/conf_examples/empty_2/expected b/src/test/conf_examples/empty_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/expected
diff --git a/src/test/conf_examples/empty_2/expected_log b/src/test/conf_examples/empty_2/expected_log
new file mode 100644
index 0000000000..9c846a03f3
--- /dev/null
+++ b/src/test/conf_examples/empty_2/expected_log
@@ -0,0 +1 @@
+Read configuration file .*empty_2[./]*torrc\.defaults
diff --git a/src/test/conf_examples/empty_2/torrc b/src/test/conf_examples/empty_2/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/torrc
diff --git a/src/test/conf_examples/empty_2/torrc.defaults b/src/test/conf_examples/empty_2/torrc.defaults
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/torrc.defaults
diff --git a/src/test/conf_examples/empty_3/expected b/src/test/conf_examples/empty_3/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_3/expected
diff --git a/src/test/conf_examples/empty_3/expected_log b/src/test/conf_examples/empty_3/expected_log
new file mode 100644
index 0000000000..e3f2365893
--- /dev/null
+++ b/src/test/conf_examples/empty_3/expected_log
@@ -0,0 +1 @@
+Processing configuration path \".*included\" at recursion level 1\.
diff --git a/src/test/conf_examples/empty_3/included/empty b/src/test/conf_examples/empty_3/included/empty
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_3/included/empty
diff --git a/src/test/conf_examples/empty_3/torrc b/src/test/conf_examples/empty_3/torrc
new file mode 100644
index 0000000000..049b438903
--- /dev/null
+++ b/src/test/conf_examples/empty_3/torrc
@@ -0,0 +1 @@
+%include "included"
diff --git a/src/test/conf_examples/empty_4/error b/src/test/conf_examples/empty_4/error
new file mode 100644
index 0000000000..e6c2f7d885
--- /dev/null
+++ b/src/test/conf_examples/empty_4/error
@@ -0,0 +1 @@
+Unable to open configuration file
\ No newline at end of file
diff --git a/src/test/conf_examples/example_1/expected b/src/test/conf_examples/example_1/expected
new file mode 100644
index 0000000000..9d6688a565
--- /dev/null
+++ b/src/test/conf_examples/example_1/expected
@@ -0,0 +1,2 @@
+ContactInfo tor_tellini@example.com
+SocksPort 80
diff --git a/src/test/conf_examples/example_1/expected_log b/src/test/conf_examples/example_1/expected_log
new file mode 100644
index 0000000000..8f83eec988
--- /dev/null
+++ b/src/test/conf_examples/example_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*example_1[./]*torrc
diff --git a/src/test/conf_examples/example_1/torrc b/src/test/conf_examples/example_1/torrc
new file mode 100644
index 0000000000..bff7fa0aa2
--- /dev/null
+++ b/src/test/conf_examples/example_1/torrc
@@ -0,0 +1,5 @@
+
+# Here is a simple example torrc.
+  SocksPort  80
+
+ContactInfo "tor_tellini@example.com"
diff --git a/src/test/conf_examples/example_2/error b/src/test/conf_examples/example_2/error
new file mode 100644
index 0000000000..ce18b68db4
--- /dev/null
+++ b/src/test/conf_examples/example_2/error
@@ -0,0 +1 @@
+Unknown option 'JumpingJellyjars'
diff --git a/src/test/conf_examples/example_2/torrc b/src/test/conf_examples/example_2/torrc
new file mode 100644
index 0000000000..8ec8133b24
--- /dev/null
+++ b/src/test/conf_examples/example_2/torrc
@@ -0,0 +1 @@
+JumpingJellyjars 1
diff --git a/src/test/conf_examples/example_3/cmdline b/src/test/conf_examples/example_3/cmdline
new file mode 100644
index 0000000000..5b2fadcebb
--- /dev/null
+++ b/src/test/conf_examples/example_3/cmdline
@@ -0,0 +1 @@
+--socksport 99
diff --git a/src/test/conf_examples/example_3/expected b/src/test/conf_examples/example_3/expected
new file mode 100644
index 0000000000..867fb8bcc8
--- /dev/null
+++ b/src/test/conf_examples/example_3/expected
@@ -0,0 +1 @@
+SocksPort 99
diff --git a/src/test/conf_examples/example_3/expected_log b/src/test/conf_examples/example_3/expected_log
new file mode 100644
index 0000000000..807f9c2fc8
--- /dev/null
+++ b/src/test/conf_examples/example_3/expected_log
@@ -0,0 +1 @@
+Read configuration file .*example_3[./]*torrc
diff --git a/src/test/conf_examples/example_3/torrc b/src/test/conf_examples/example_3/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/example_3/torrc
diff --git a/src/test/conf_examples/include_1/error_no_dirauth_relay b/src/test/conf_examples/include_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/include_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/include_1/expected b/src/test/conf_examples/include_1/expected
new file mode 100644
index 0000000000..4bbf52ce9f
--- /dev/null
+++ b/src/test/conf_examples/include_1/expected
@@ -0,0 +1,3 @@
+ContactInfo includefile@example.com
+Nickname nested
+ORPort 8008
diff --git a/src/test/conf_examples/include_1/expected_log b/src/test/conf_examples/include_1/expected_log
new file mode 100644
index 0000000000..0791a494d2
--- /dev/null
+++ b/src/test/conf_examples/include_1/expected_log
@@ -0,0 +1 @@
+Processing configuration path \".*nested\.inc\" at recursion level 2\.
diff --git a/src/test/conf_examples/include_1/included.inc b/src/test/conf_examples/include_1/included.inc
new file mode 100644
index 0000000000..8d1834345d
--- /dev/null
+++ b/src/test/conf_examples/include_1/included.inc
@@ -0,0 +1,4 @@
+
+ContactInfo includefile@example.com
+
+%include "nested.inc"
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/nested.inc b/src/test/conf_examples/include_1/nested.inc
new file mode 100644
index 0000000000..789b044a2b
--- /dev/null
+++ b/src/test/conf_examples/include_1/nested.inc
@@ -0,0 +1,2 @@
+
+Nickname nested
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/torrc b/src/test/conf_examples/include_1/torrc
new file mode 100644
index 0000000000..2ed4074f6e
--- /dev/null
+++ b/src/test/conf_examples/include_1/torrc
@@ -0,0 +1,4 @@
+
+%include "included.inc"
+
+ORPort 8008
diff --git a/src/test/conf_examples/include_bug_31408/error_no_dirauth_relay b/src/test/conf_examples/include_bug_31408/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/include_bug_31408/expected b/src/test/conf_examples/include_bug_31408/expected
new file mode 100644
index 0000000000..2e822f1a78
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/expected
@@ -0,0 +1,2 @@
+Nickname test31408
+ORPort 31408
diff --git a/src/test/conf_examples/include_bug_31408/expected_log b/src/test/conf_examples/include_bug_31408/expected_log
new file mode 100644
index 0000000000..e3f2365893
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/expected_log
@@ -0,0 +1 @@
+Processing configuration path \".*included\" at recursion level 1\.
diff --git a/src/test/conf_examples/include_bug_31408/included/01_nickname.inc b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
new file mode 100644
index 0000000000..508dd89a35
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
@@ -0,0 +1 @@
+Nickname test31408
diff --git a/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
new file mode 100644
index 0000000000..140e927f19
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
@@ -0,0 +1,3 @@
+# Bug 31048 is triggered when the last file in a config directory:
+#   * contains no configuration options,
+#   * but is non-empty: that is, it contains comments or whitespace.
diff --git a/src/test/conf_examples/include_bug_31408/torrc b/src/test/conf_examples/include_bug_31408/torrc
new file mode 100644
index 0000000000..a42685e93c
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/torrc
@@ -0,0 +1,2 @@
+%include "included"
+ORPort 31408
diff --git a/src/test/conf_examples/large_1/error_no_dirauth_relay b/src/test/conf_examples/large_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/large_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/large_1/expected b/src/test/conf_examples/large_1/expected
new file mode 100644
index 0000000000..99a12ffc84
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected
@@ -0,0 +1,158 @@
+AccountingMax 10737418240
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2147483647
+BandwidthRate 1610612736
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirPortFrontPage /dirport/frontpage
+DirPort 99
+DirReqStatistics 0
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 1260
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 300
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPortCookieAuthFile /foobar
+ExtORPort 99
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 691200
+HeartbeatPeriod 2700
+IPv6Exit 1
+KeepalivePeriod 540
+KeyDirectory /keyz
+KISTSchedRunInterval 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 60000
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 3600
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 2629728
+MaxMemInQueues 314572800
+MaxOnionQueueDelay 60000
+MaxUnparseableDescSizeToLog 1048576
+MiddleNodes grommit,truffle,parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 7200
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PerConnBWBurst 10485760
+PerConnBWRate 102400
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 600
+RephistTrackTime 600
+SafeLogging 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10
+SigningKeyLifetime 4838400
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 600
+SSLKeyLifetime 86400
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1000
+TrackHostExits www.example.com
+TrackHostExitsExpire 3600
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/large_1/expected_log b/src/test/conf_examples/large_1/expected_log
new file mode 100644
index 0000000000..21248bb5e4
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected_log
@@ -0,0 +1 @@
+Your log may contain sensitive information
diff --git a/src/test/conf_examples/large_1/expected_log_no_dirauth b/src/test/conf_examples/large_1/expected_log_no_dirauth
new file mode 100644
index 0000000000..0b74de4e40
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected_log_no_dirauth
@@ -0,0 +1 @@
+This copy of Tor was built without support for the option "ConsensusParams". Skipping.
diff --git a/src/test/conf_examples/large_1/expected_no_dirauth b/src/test/conf_examples/large_1/expected_no_dirauth
new file mode 100644
index 0000000000..26a33bdc7c
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected_no_dirauth
@@ -0,0 +1,157 @@
+AccountingMax 10737418240
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2147483647
+BandwidthRate 1610612736
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirPortFrontPage /dirport/frontpage
+DirPort 99
+DirReqStatistics 0
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 1260
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 300
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPortCookieAuthFile /foobar
+ExtORPort 99
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 691200
+HeartbeatPeriod 2700
+IPv6Exit 1
+KeepalivePeriod 540
+KeyDirectory /keyz
+KISTSchedRunInterval 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 60000
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 3600
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 2629728
+MaxMemInQueues 314572800
+MaxOnionQueueDelay 60000
+MaxUnparseableDescSizeToLog 1048576
+MiddleNodes grommit,truffle,parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 7200
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PerConnBWBurst 10485760
+PerConnBWRate 102400
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 600
+RephistTrackTime 600
+SafeLogging 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10
+SigningKeyLifetime 4838400
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 600
+SSLKeyLifetime 86400
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1000
+TrackHostExits www.example.com
+TrackHostExitsExpire 3600
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/large_1/torrc b/src/test/conf_examples/large_1/torrc
new file mode 100644
index 0000000000..20ddf00e16
--- /dev/null
+++ b/src/test/conf_examples/large_1/torrc
@@ -0,0 +1,166 @@
+AccountingMax 10 GB
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2 GB
+BandwidthRate 1.5 GB
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitPadding 1
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirReqStatistics 0
+DirPort 99
+DirPortFrontPage /dirport/frontpage
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 21 minutes
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 5 minutes
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPort 99
+ExtORPortCookieAuthFile /foobar
+ExtraInfoStatistics 1
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchHidServDescriptors 1
+FetchServerDescriptors 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 8 days
+HeartbeatPeriod 45 minutes
+IPv6Exit 1
+KeepalivePeriod 9 minutes
+KeyDirectory /keyz
+KISTSchedRunInterval 1 msec
+LearnCircuitBuildTimeout 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 1 minute
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 1 hour
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 1 month
+MaxMemInQueues 300 MB
+MaxOnionQueueDelay 60 seconds
+MaxUnparseableDescSizeToLog 1 MB
+MiddleNodes grommit, truffle, parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 2 hours
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PaddingStatistics 1
+PerConnBWBurst 10 MB
+PerConnBWRate 100 kb
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 10 minutes
+RephistTrackTime 10 minutes
+SafeLogging 0
+SafeSocks 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10 seconds
+SigningKeyLifetime 8 weeks
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 10 minutes
+SSLKeyLifetime 1 day
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1 second
+TrackHostExits www.example.com
+TrackHostExitsExpire 1 hour
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseEntryGuards 1
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/lzma_zstd_1/expected b/src/test/conf_examples/lzma_zstd_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log b/src/test/conf_examples/lzma_zstd_1/expected_log
new file mode 100644
index 0000000000..a5531ca21e
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma N/A, and Libzstd N/A
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log_lzma b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma
new file mode 100644
index 0000000000..2947e5991b
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma .*, and Libzstd N/A
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log_lzma_zstd b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma_zstd
new file mode 100644
index 0000000000..e76e4357f8
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma_zstd
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma .*, and Libzstd .*
\ No newline at end of file
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log_zstd b/src/test/conf_examples/lzma_zstd_1/expected_log_zstd
new file mode 100644
index 0000000000..c8b174423b
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log_zstd
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma N/A, and Libzstd .*
\ No newline at end of file
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_lzma b/src/test/conf_examples/lzma_zstd_1/expected_lzma
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_lzma
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_lzma_zstd b/src/test/conf_examples/lzma_zstd_1/expected_lzma_zstd
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_lzma_zstd
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_zstd b/src/test/conf_examples/lzma_zstd_1/expected_zstd
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_zstd
diff --git a/src/test/conf_examples/lzma_zstd_1/torrc b/src/test/conf_examples/lzma_zstd_1/torrc
new file mode 100644
index 0000000000..f873d79028
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/torrc
@@ -0,0 +1 @@
+# This test checks for the optional library list in tor's logs
diff --git a/src/test/conf_examples/missing_cl_arg/cmdline b/src/test/conf_examples/missing_cl_arg/cmdline
new file mode 100644
index 0000000000..7fc4d0a54d
--- /dev/null
+++ b/src/test/conf_examples/missing_cl_arg/cmdline
@@ -0,0 +1 @@
+--hash-password
diff --git a/src/test/conf_examples/missing_cl_arg/error b/src/test/conf_examples/missing_cl_arg/error
new file mode 100644
index 0000000000..61dbeac8aa
--- /dev/null
+++ b/src/test/conf_examples/missing_cl_arg/error
@@ -0,0 +1 @@
+Command-line option '--hash-password' with no value.
diff --git a/src/test/conf_examples/missing_cl_arg/torrc b/src/test/conf_examples/missing_cl_arg/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/missing_cl_arg/torrc
diff --git a/src/test/conf_examples/nss_1/expected b/src/test/conf_examples/nss_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected
diff --git a/src/test/conf_examples/nss_1/expected_log b/src/test/conf_examples/nss_1/expected_log
new file mode 100644
index 0000000000..32e8cfc2f8
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected_log
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, OpenSSL .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/nss_1/expected_log_nss b/src/test/conf_examples/nss_1/expected_log_nss
new file mode 100644
index 0000000000..c0fe7b003c
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected_log_nss
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, NSS .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/nss_1/expected_nss b/src/test/conf_examples/nss_1/expected_nss
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected_nss
diff --git a/src/test/conf_examples/nss_1/torrc b/src/test/conf_examples/nss_1/torrc
new file mode 100644
index 0000000000..f873d79028
--- /dev/null
+++ b/src/test/conf_examples/nss_1/torrc
@@ -0,0 +1 @@
+# This test checks for the optional library list in tor's logs
diff --git a/src/test/conf_examples/obsolete_1/expected b/src/test/conf_examples/obsolete_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/expected
diff --git a/src/test/conf_examples/obsolete_1/expected_log b/src/test/conf_examples/obsolete_1/expected_log
new file mode 100644
index 0000000000..52f6f70a2e
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/expected_log
@@ -0,0 +1 @@
+Skipping obsolete configuration option
diff --git a/src/test/conf_examples/obsolete_1/torrc b/src/test/conf_examples/obsolete_1/torrc
new file mode 100644
index 0000000000..e711fe4065
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/torrc
@@ -0,0 +1,70 @@
+# These options are obsolete as of 0.4.2
+# Obsolete options without arguments, or with an empty argument,
+# are silently ignored. So we give each one of these options an argument.
+AllowDotExit 1
+AllowInvalidNodes 1
+AllowSingleHopCircuits 1
+AllowSingleHopExits 1
+AlternateHSAuthority 1
+AuthDirBadDir 1
+AuthDirBadDirCCs 1
+AuthDirRejectUnlisted 1
+AuthDirListBadDirs 1
+AuthDirMaxServersPerAuthAddr 1
+CircuitIdleTimeout 1
+ControlListenAddress 1
+DirListenAddress 1
+DisableIOCP 1
+DisableV2DirectoryInfo_ 1
+DynamicDHGroups 1
+DNSListenAddress 1
+TestingEnableTbEmptyEvent 1
+ExcludeSingleHopRelays 1
+FallbackNetworkstatusFile 1
+FastFirstHopPK 1
+FetchV2Networkstatus 1
+Group 1
+HidServDirectoryV2 1
+CloseHSClientCircuitsImmediatelyOnTimeout 1
+CloseHSServiceRendCircuitsImmediatelyOnTimeout 1
+MaxOnionsPending 1
+NamingAuthoritativeDirectory 1
+NATDListenAddress 1
+PredictedPortsRelevanceTime 1
+WarnUnsafeSocks 1
+ORListenAddress 1
+PathBiasDisableRate 1
+PathBiasScaleFactor 1
+PathBiasMultFactor 1
+PathBiasUseCloseCounts 1
+PortForwarding 1
+PortForwardingHelper 1
+PreferTunneledDirConns 1
+RecommendedPackages 1
+RunTesting 1
+SchedulerLowWaterMark__ 1
+SchedulerHighWaterMark__ 1
+SchedulerMaxFlushCells__ 1
+SocksListenAddress 1
+StrictEntryNodes 1
+StrictExitNodes 1
+Support022HiddenServices 1
+Tor2webMode 1
+Tor2webRendezvousPoints 1
+TLSECGroup 1
+TransListenAddress 1
+TunnelDirConns 1
+UseEntryGuardsAsDirGuards 1
+UseNTorHandshake 1
+UserspaceIOCPBuffers 1
+V1AuthoritativeDirectory 1
+V2AuthoritativeDirectory 1
+VoteOnHidServDirectoriesV2 1
+UseFilteringSSLBufferevents 1
+__UseFilteringSSLBufferevents 1
+TestingConsensusMaxDownloadTries 1
+ClientBootstrapConsensusMaxDownloadTries 1
+ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries 1
+TestingDescriptorMaxDownloadTries 1
+TestingMicrodescMaxDownloadTries 1
+TestingCertMaxDownloadTries 1
diff --git a/src/test/conf_examples/obsolete_2/expected b/src/test/conf_examples/obsolete_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/expected
diff --git a/src/test/conf_examples/obsolete_2/expected_log b/src/test/conf_examples/obsolete_2/expected_log
new file mode 100644
index 0000000000..2160355ee9
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/expected_log
@@ -0,0 +1 @@
+Read configuration file .*obsolete_2[./]*torrc
diff --git a/src/test/conf_examples/obsolete_2/torrc b/src/test/conf_examples/obsolete_2/torrc
new file mode 100644
index 0000000000..b83e7a7369
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/torrc
@@ -0,0 +1,5 @@
+# This option has been obsolete for some time
+# Obsolete options without arguments, or with an empty argument,
+# are silently ignored.
+AllowDotExit
+AllowInvalidNodes ""
diff --git a/src/test/conf_examples/obsolete_3/expected b/src/test/conf_examples/obsolete_3/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_3/expected
diff --git a/src/test/conf_examples/obsolete_3/expected_log b/src/test/conf_examples/obsolete_3/expected_log
new file mode 100644
index 0000000000..82d76ec818
--- /dev/null
+++ b/src/test/conf_examples/obsolete_3/expected_log
@@ -0,0 +1 @@
+Skipping obsolete configuration option "AllowDotExit"
diff --git a/src/test/conf_examples/obsolete_3/torrc b/src/test/conf_examples/obsolete_3/torrc
new file mode 100644
index 0000000000..e0efe752bf
--- /dev/null
+++ b/src/test/conf_examples/obsolete_3/torrc
@@ -0,0 +1,4 @@
+# This option has been obsolete for some time
+# Obsolete options without arguments, or with an empty argument,
+# are silently ignored. So we give this option an argument.
+AllowDotExit 1
diff --git a/src/test/conf_examples/ops_1/cmdline b/src/test/conf_examples/ops_1/cmdline
new file mode 100644
index 0000000000..2bb9bfa132
--- /dev/null
+++ b/src/test/conf_examples/ops_1/cmdline
@@ -0,0 +1 @@
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/error_no_dirauth_relay b/src/test/conf_examples/ops_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_1/expected b/src/test/conf_examples/ops_1/expected
new file mode 100644
index 0000000000..84be6a70e2
--- /dev/null
+++ b/src/test/conf_examples/ops_1/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/expected_log b/src/test/conf_examples/ops_1/expected_log
new file mode 100644
index 0000000000..b785d7fb52
--- /dev/null
+++ b/src/test/conf_examples/ops_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_1[./]*torrc
diff --git a/src/test/conf_examples/ops_1/torrc b/src/test/conf_examples/ops_1/torrc
new file mode 100644
index 0000000000..daf8ae60fe
--- /dev/null
+++ b/src/test/conf_examples/ops_1/torrc
@@ -0,0 +1,3 @@
+# We'll replace this option on the command line.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_2/cmdline b/src/test/conf_examples/ops_2/cmdline
new file mode 100644
index 0000000000..fdd48a045c
--- /dev/null
+++ b/src/test/conf_examples/ops_2/cmdline
@@ -0,0 +1 @@
+/ORPort
diff --git a/src/test/conf_examples/ops_2/expected b/src/test/conf_examples/ops_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_2/expected
diff --git a/src/test/conf_examples/ops_2/expected_log b/src/test/conf_examples/ops_2/expected_log
new file mode 100644
index 0000000000..17fbc3ffbf
--- /dev/null
+++ b/src/test/conf_examples/ops_2/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_2[./]*torrc
diff --git a/src/test/conf_examples/ops_2/torrc b/src/test/conf_examples/ops_2/torrc
new file mode 100644
index 0000000000..21fcc93f9a
--- /dev/null
+++ b/src/test/conf_examples/ops_2/torrc
@@ -0,0 +1,3 @@
+# We'll remove this option on the command line, and not replace it.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_3/cmdline b/src/test/conf_examples/ops_3/cmdline
new file mode 100644
index 0000000000..e4965d26f8
--- /dev/null
+++ b/src/test/conf_examples/ops_3/cmdline
@@ -0,0 +1 @@
++ORPort 1000
diff --git a/src/test/conf_examples/ops_3/error_no_dirauth_relay b/src/test/conf_examples/ops_3/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_3/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_3/expected b/src/test/conf_examples/ops_3/expected
new file mode 100644
index 0000000000..569d26b577
--- /dev/null
+++ b/src/test/conf_examples/ops_3/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9999
+ORPort 1000
diff --git a/src/test/conf_examples/ops_3/expected_log b/src/test/conf_examples/ops_3/expected_log
new file mode 100644
index 0000000000..151498f0df
--- /dev/null
+++ b/src/test/conf_examples/ops_3/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_3[./]*torrc
diff --git a/src/test/conf_examples/ops_3/torrc b/src/test/conf_examples/ops_3/torrc
new file mode 100644
index 0000000000..14adf87d7f
--- /dev/null
+++ b/src/test/conf_examples/ops_3/torrc
@@ -0,0 +1,3 @@
+# We will extend this option on the command line
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_4/error_no_dirauth_relay b/src/test/conf_examples/ops_4/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_4/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_4/expected b/src/test/conf_examples/ops_4/expected
new file mode 100644
index 0000000000..bf52f6a330
--- /dev/null
+++ b/src/test/conf_examples/ops_4/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/expected_log b/src/test/conf_examples/ops_4/expected_log
new file mode 100644
index 0000000000..7632b2290c
--- /dev/null
+++ b/src/test/conf_examples/ops_4/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_4[./]*torrc\.defaults
diff --git a/src/test/conf_examples/ops_4/torrc b/src/test/conf_examples/ops_4/torrc
new file mode 100644
index 0000000000..dcec2aa95d
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc
@@ -0,0 +1,3 @@
+# This value is unadorned, so replaces the one from defaults.torrc.
+
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/torrc.defaults b/src/test/conf_examples/ops_4/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_5/error_no_dirauth_relay b/src/test/conf_examples/ops_5/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_5/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_5/expected b/src/test/conf_examples/ops_5/expected
new file mode 100644
index 0000000000..288721da53
--- /dev/null
+++ b/src/test/conf_examples/ops_5/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9000
+ORPort 9099
diff --git a/src/test/conf_examples/ops_5/expected_log b/src/test/conf_examples/ops_5/expected_log
new file mode 100644
index 0000000000..ec63cb0638
--- /dev/null
+++ b/src/test/conf_examples/ops_5/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_5[./]*torrc\.defaults
diff --git a/src/test/conf_examples/ops_5/torrc b/src/test/conf_examples/ops_5/torrc
new file mode 100644
index 0000000000..3284fc1c55
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc
@@ -0,0 +1,3 @@
+# This value has a plus, and so extends the one from defaults.torrc.
+
++ORPort 9099
diff --git a/src/test/conf_examples/ops_5/torrc.defaults b/src/test/conf_examples/ops_5/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_6/expected b/src/test/conf_examples/ops_6/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_6/expected
diff --git a/src/test/conf_examples/ops_6/expected_log b/src/test/conf_examples/ops_6/expected_log
new file mode 100644
index 0000000000..f9b1ca0412
--- /dev/null
+++ b/src/test/conf_examples/ops_6/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_6[./]*torrc\.defaults
diff --git a/src/test/conf_examples/ops_6/torrc b/src/test/conf_examples/ops_6/torrc
new file mode 100644
index 0000000000..4d51caaff7
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc
@@ -0,0 +1,3 @@
+# This value has a slash, and so clears the one from defaults.torrc.
+
+/ORPort
diff --git a/src/test/conf_examples/ops_6/torrc.defaults b/src/test/conf_examples/ops_6/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/pt_01/expected b/src/test/conf_examples/pt_01/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/pt_01/expected
diff --git a/src/test/conf_examples/pt_01/expected_log b/src/test/conf_examples/pt_01/expected_log
new file mode 100644
index 0000000000..65bfa7a7b7
--- /dev/null
+++ b/src/test/conf_examples/pt_01/expected_log
@@ -0,0 +1 @@
+Linelist option 'ExtORPort' has no value\. Skipping
diff --git a/src/test/conf_examples/pt_01/torrc b/src/test/conf_examples/pt_01/torrc
new file mode 100644
index 0000000000..574bb32a0d
--- /dev/null
+++ b/src/test/conf_examples/pt_01/torrc
@@ -0,0 +1,7 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Empty linelist values are ignored with a warning
+ExtORPort
+ServerTransportPlugin
+ServerTransportListenAddr
+ServerTransportOptions
diff --git a/src/test/conf_examples/pt_02/error b/src/test/conf_examples/pt_02/error
new file mode 100644
index 0000000000..ce28eab729
--- /dev/null
+++ b/src/test/conf_examples/pt_02/error
@@ -0,0 +1 @@
+Invalid ExtORPort configuration
diff --git a/src/test/conf_examples/pt_02/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_02/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..0e48dca7fd
--- /dev/null
+++ b/src/test/conf_examples/pt_02/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_02[./]*torrc
diff --git a/src/test/conf_examples/pt_02/expected_no_dirauth_relay b/src/test/conf_examples/pt_02/expected_no_dirauth_relay
new file mode 100644
index 0000000000..f5cd26e6b7
--- /dev/null
+++ b/src/test/conf_examples/pt_02/expected_no_dirauth_relay
@@ -0,0 +1,8 @@
+ExtORPort illegal_hostname_chars$()^*%(%
+ServerTransportListenAddr bad
+ServerTransportListenAddr bad2 illegal_hostname_chars$()^*%(%
+ServerTransportOptions bad
+ServerTransportOptions bad2 not_kv
+ServerTransportPlugin bad
+ServerTransportPlugin bad2 exec
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_02/torrc b/src/test/conf_examples/pt_02/torrc
new file mode 100644
index 0000000000..825f2c4be4
--- /dev/null
+++ b/src/test/conf_examples/pt_02/torrc
@@ -0,0 +1,13 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Bad options are also ignored
+# (Unless the relay module is disabled, because they are relay-only
+# options. We'll ignore all relay-only options in #32395.)
+ExtORPort illegal_hostname_chars$()^*%(%#%)#(%*
+ServerTransportPlugin bad
+ServerTransportPlugin bad2 exec
+ServerTransportPlugin bad3 exec /
+ServerTransportListenAddr bad
+ServerTransportListenAddr bad2 illegal_hostname_chars$()^*%(%#%)#(%*
+ServerTransportOptions bad
+ServerTransportOptions bad2 not_kv
diff --git a/src/test/conf_examples/pt_03/expected b/src/test/conf_examples/pt_03/expected
new file mode 100644
index 0000000000..f849f2a78f
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected
@@ -0,0 +1 @@
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_03/expected_log b/src/test/conf_examples/pt_03/expected_log
new file mode 100644
index 0000000000..285a189c28
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected_log
@@ -0,0 +1 @@
+We use pluggable transports but the Extended ORPort is disabled
diff --git a/src/test/conf_examples/pt_03/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_03/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..88f4e5bdfb
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_03[./]*torrc
diff --git a/src/test/conf_examples/pt_03/expected_no_dirauth_relay b/src/test/conf_examples/pt_03/expected_no_dirauth_relay
new file mode 100644
index 0000000000..f849f2a78f
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected_no_dirauth_relay
@@ -0,0 +1 @@
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_03/torrc b/src/test/conf_examples/pt_03/torrc
new file mode 100644
index 0000000000..9868c39b26
--- /dev/null
+++ b/src/test/conf_examples/pt_03/torrc
@@ -0,0 +1,4 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Plugin, but no ExtORPort
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_04/expected b/src/test/conf_examples/pt_04/expected
new file mode 100644
index 0000000000..9087f600e0
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected
@@ -0,0 +1,3 @@
+ExtORPortCookieAuthFile /
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_04/expected_log b/src/test/conf_examples/pt_04/expected_log
new file mode 100644
index 0000000000..5b3ab51d25
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected_log
@@ -0,0 +1 @@
+Tor is not configured as a relay but you specified a ServerTransportPlugin line.*The ServerTransportPlugin line will be ignored
diff --git a/src/test/conf_examples/pt_04/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_04/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..2b989bf320
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_04[./]*torrc
diff --git a/src/test/conf_examples/pt_04/expected_no_dirauth_relay b/src/test/conf_examples/pt_04/expected_no_dirauth_relay
new file mode 100644
index 0000000000..9087f600e0
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected_no_dirauth_relay
@@ -0,0 +1,3 @@
+ExtORPortCookieAuthFile /
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_04/torrc b/src/test/conf_examples/pt_04/torrc
new file mode 100644
index 0000000000..18bb28f9cf
--- /dev/null
+++ b/src/test/conf_examples/pt_04/torrc
@@ -0,0 +1,6 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a bad cookie auth file
+ExtORPort 1
+ExtORPortCookieAuthFile /
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_05/error_no_dirauth_relay b/src/test/conf_examples/pt_05/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/pt_05/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/pt_05/expected b/src/test/conf_examples/pt_05/expected
new file mode 100644
index 0000000000..61568bb9ac
--- /dev/null
+++ b/src/test/conf_examples/pt_05/expected
@@ -0,0 +1,4 @@
+ExtORPort 1
+Nickname Unnamed
+ORPort 2
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_05/expected_log b/src/test/conf_examples/pt_05/expected_log
new file mode 100644
index 0000000000..c05a0931d6
--- /dev/null
+++ b/src/test/conf_examples/pt_05/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
diff --git a/src/test/conf_examples/pt_05/torrc b/src/test/conf_examples/pt_05/torrc
new file mode 100644
index 0000000000..55c569bb1b
--- /dev/null
+++ b/src/test/conf_examples/pt_05/torrc
@@ -0,0 +1,6 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a valid minimal config
+ORPort 2
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_06/expected b/src/test/conf_examples/pt_06/expected
new file mode 100644
index 0000000000..d5788b92c9
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected
@@ -0,0 +1,6 @@
+ExtORPortCookieAuthFile /
+ExtORPortCookieAuthFileGroupReadable 1
+ExtORPort 1
+ServerTransportListenAddr bad3 127.0.0.1:2
+ServerTransportOptions bad3 a=b
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_06/expected_log b/src/test/conf_examples/pt_06/expected_log
new file mode 100644
index 0000000000..5b3ab51d25
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected_log
@@ -0,0 +1 @@
+Tor is not configured as a relay but you specified a ServerTransportPlugin line.*The ServerTransportPlugin line will be ignored
diff --git a/src/test/conf_examples/pt_06/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_06/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..f35a380c9f
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_06[./]*torrc
diff --git a/src/test/conf_examples/pt_06/expected_no_dirauth_relay b/src/test/conf_examples/pt_06/expected_no_dirauth_relay
new file mode 100644
index 0000000000..d5788b92c9
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected_no_dirauth_relay
@@ -0,0 +1,6 @@
+ExtORPortCookieAuthFile /
+ExtORPortCookieAuthFileGroupReadable 1
+ExtORPort 1
+ServerTransportListenAddr bad3 127.0.0.1:2
+ServerTransportOptions bad3 a=b
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_06/torrc b/src/test/conf_examples/pt_06/torrc
new file mode 100644
index 0000000000..20cfc329a7
--- /dev/null
+++ b/src/test/conf_examples/pt_06/torrc
@@ -0,0 +1,9 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a config with all the options
+ExtORPort 1
+ExtORPortCookieAuthFile /
+ExtORPortCookieAuthFileGroupReadable 1
+ServerTransportPlugin bad3 exec /
+ServerTransportListenAddr bad3 127.0.0.1:2
+ServerTransportOptions bad3 a=b
diff --git a/src/test/conf_examples/pt_07/error_no_dirauth_relay b/src/test/conf_examples/pt_07/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/pt_07/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/pt_07/expected b/src/test/conf_examples/pt_07/expected
new file mode 100644
index 0000000000..c3a75dc407
--- /dev/null
+++ b/src/test/conf_examples/pt_07/expected
@@ -0,0 +1,4 @@
+ExtORPort 2.2.2.2:1
+Nickname Unnamed
+ORPort 2
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_07/expected_log b/src/test/conf_examples/pt_07/expected_log
new file mode 100644
index 0000000000..5afaf02ba9
--- /dev/null
+++ b/src/test/conf_examples/pt_07/expected_log
@@ -0,0 +1 @@
+You specified a public address .* for ExtORPort
diff --git a/src/test/conf_examples/pt_07/torrc b/src/test/conf_examples/pt_07/torrc
new file mode 100644
index 0000000000..40eaf50e64
--- /dev/null
+++ b/src/test/conf_examples/pt_07/torrc
@@ -0,0 +1,6 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a valid config with a risky ExtORPort address
+ORPort 2
+ExtORPort 2.2.2.2:1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_08/error b/src/test/conf_examples/pt_08/error
new file mode 100644
index 0000000000..7931bbb4b9
--- /dev/null
+++ b/src/test/conf_examples/pt_08/error
@@ -0,0 +1 @@
+ExtORPort does not support unix sockets
\ No newline at end of file
diff --git a/src/test/conf_examples/pt_08/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_08/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..79dcbc10ca
--- /dev/null
+++ b/src/test/conf_examples/pt_08/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_08[./]*torrc
diff --git a/src/test/conf_examples/pt_08/expected_no_dirauth_relay b/src/test/conf_examples/pt_08/expected_no_dirauth_relay
new file mode 100644
index 0000000000..9a1a9bd1e4
--- /dev/null
+++ b/src/test/conf_examples/pt_08/expected_no_dirauth_relay
@@ -0,0 +1,2 @@
+ExtORPort unix:/
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_08/torrc b/src/test/conf_examples/pt_08/torrc
new file mode 100644
index 0000000000..6f1d79d706
--- /dev/null
+++ b/src/test/conf_examples/pt_08/torrc
@@ -0,0 +1,7 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try an invalid config with a unix socket for ExtORPort
+# (Unless the relay module is disabled, because they are relay-only
+# options. We'll ignore all relay-only options in #32395.)
+ExtORPort unix:/
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_09/error b/src/test/conf_examples/pt_09/error
new file mode 100644
index 0000000000..882b50a7bc
--- /dev/null
+++ b/src/test/conf_examples/pt_09/error
@@ -0,0 +1 @@
+Error parsing ServerTransportListenAddr address
\ No newline at end of file
diff --git a/src/test/conf_examples/pt_09/error_no_dirauth_relay b/src/test/conf_examples/pt_09/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/pt_09/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/pt_09/torrc b/src/test/conf_examples/pt_09/torrc
new file mode 100644
index 0000000000..50a8e95b95
--- /dev/null
+++ b/src/test/conf_examples/pt_09/torrc
@@ -0,0 +1,7 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a valid minimal config, with a bad ServerTransportListenAddr
+ORPort 2
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
+ServerTransportListenAddr bad3 [aaaa::bbbb:ccccc]
diff --git a/src/test/conf_examples/relay_01/expected b/src/test/conf_examples/relay_01/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/relay_01/expected
diff --git a/src/test/conf_examples/relay_01/expected_log b/src/test/conf_examples/relay_01/expected_log
new file mode 100644
index 0000000000..32e8c99d27
--- /dev/null
+++ b/src/test/conf_examples/relay_01/expected_log
@@ -0,0 +1 @@
+Linelist option 'ORPort' has no value\. Skipping
diff --git a/src/test/conf_examples/relay_01/torrc b/src/test/conf_examples/relay_01/torrc
new file mode 100644
index 0000000000..da3e85b427
--- /dev/null
+++ b/src/test/conf_examples/relay_01/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Empty linelist values are ignored with a warning
+ORPort
+DirPort
diff --git a/src/test/conf_examples/relay_02/error b/src/test/conf_examples/relay_02/error
new file mode 100644
index 0000000000..dd87d9f7e2
--- /dev/null
+++ b/src/test/conf_examples/relay_02/error
@@ -0,0 +1 @@
+Unrecognized value bad
diff --git a/src/test/conf_examples/relay_02/error_no_dirauth_relay b/src/test/conf_examples/relay_02/error_no_dirauth_relay
new file mode 100644
index 0000000000..dd87d9f7e2
--- /dev/null
+++ b/src/test/conf_examples/relay_02/error_no_dirauth_relay
@@ -0,0 +1 @@
+Unrecognized value bad
diff --git a/src/test/conf_examples/relay_02/torrc b/src/test/conf_examples/relay_02/torrc
new file mode 100644
index 0000000000..3eaa4403a9
--- /dev/null
+++ b/src/test/conf_examples/relay_02/torrc
@@ -0,0 +1,7 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Bad options are also ignored
+ORPort illegal_hostname_chars$()^*%(%#%)#(%*
+DirPort illegal_hostname_chars$()^*%(%#%)#(%*
+DirCache bad
+BridgeRelay bad
diff --git a/src/test/conf_examples/relay_03/error_no_dirauth_relay b/src/test/conf_examples/relay_03/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_03/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_03/expected b/src/test/conf_examples/relay_03/expected
new file mode 100644
index 0000000000..15056a8d1f
--- /dev/null
+++ b/src/test/conf_examples/relay_03/expected
@@ -0,0 +1,2 @@
+DirPort 1
+ORPort 0
diff --git a/src/test/conf_examples/relay_03/expected_log b/src/test/conf_examples/relay_03/expected_log
new file mode 100644
index 0000000000..46ab723e4a
--- /dev/null
+++ b/src/test/conf_examples/relay_03/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_03[./]*torrc
diff --git a/src/test/conf_examples/relay_03/torrc b/src/test/conf_examples/relay_03/torrc
new file mode 100644
index 0000000000..fd7da7bb95
--- /dev/null
+++ b/src/test/conf_examples/relay_03/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# DirPort, but no ORPort
+ORPort 0
+DirPort 1
diff --git a/src/test/conf_examples/relay_04/error_no_dirauth_relay b/src/test/conf_examples/relay_04/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_04/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_04/expected b/src/test/conf_examples/relay_04/expected
new file mode 100644
index 0000000000..1d25374ed1
--- /dev/null
+++ b/src/test/conf_examples/relay_04/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_04/expected_log b/src/test/conf_examples/relay_04/expected_log
new file mode 100644
index 0000000000..c05a0931d6
--- /dev/null
+++ b/src/test/conf_examples/relay_04/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
diff --git a/src/test/conf_examples/relay_04/torrc b/src/test/conf_examples/relay_04/torrc
new file mode 100644
index 0000000000..ff08b2376b
--- /dev/null
+++ b/src/test/conf_examples/relay_04/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal config
+ORPort 1
diff --git a/src/test/conf_examples/relay_05/error_no_dirauth_relay b/src/test/conf_examples/relay_05/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_05/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_05/expected b/src/test/conf_examples/relay_05/expected
new file mode 100644
index 0000000000..ae58cee1af
--- /dev/null
+++ b/src/test/conf_examples/relay_05/expected
@@ -0,0 +1,3 @@
+DirPort 2
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_05/expected_log b/src/test/conf_examples/relay_05/expected_log
new file mode 100644
index 0000000000..483c2e2aae
--- /dev/null
+++ b/src/test/conf_examples/relay_05/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_05[./]*torrc
diff --git a/src/test/conf_examples/relay_05/torrc b/src/test/conf_examples/relay_05/torrc
new file mode 100644
index 0000000000..faeaad32a0
--- /dev/null
+++ b/src/test/conf_examples/relay_05/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal directory mirror config
+ORPort 1
+DirPort 2
diff --git a/src/test/conf_examples/relay_06/error_no_dirauth_relay b/src/test/conf_examples/relay_06/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_06/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_06/expected b/src/test/conf_examples/relay_06/expected
new file mode 100644
index 0000000000..904c7339e0
--- /dev/null
+++ b/src/test/conf_examples/relay_06/expected
@@ -0,0 +1,3 @@
+BridgeRelay 1
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_06/expected_log b/src/test/conf_examples/relay_06/expected_log
new file mode 100644
index 0000000000..70eb18df19
--- /dev/null
+++ b/src/test/conf_examples/relay_06/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_06[./]*torrc
diff --git a/src/test/conf_examples/relay_06/torrc b/src/test/conf_examples/relay_06/torrc
new file mode 100644
index 0000000000..baeae8df5d
--- /dev/null
+++ b/src/test/conf_examples/relay_06/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal bridge config
+ORPort 1
+BridgeRelay 1
diff --git a/src/test/conf_examples/relay_07/error_no_dirauth_relay b/src/test/conf_examples/relay_07/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_07/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_07/expected b/src/test/conf_examples/relay_07/expected
new file mode 100644
index 0000000000..79fa3e5a47
--- /dev/null
+++ b/src/test/conf_examples/relay_07/expected
@@ -0,0 +1,3 @@
+DirCache 0
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_07/expected_log b/src/test/conf_examples/relay_07/expected_log
new file mode 100644
index 0000000000..14729a7ab1
--- /dev/null
+++ b/src/test/conf_examples/relay_07/expected_log
@@ -0,0 +1 @@
+DirCache is disabled and we are configured as a relay
diff --git a/src/test/conf_examples/relay_07/torrc b/src/test/conf_examples/relay_07/torrc
new file mode 100644
index 0000000000..01ac138597
--- /dev/null
+++ b/src/test/conf_examples/relay_07/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal non-directory cache config
+ORPort 1
+DirCache 0
diff --git a/src/test/conf_examples/relay_08/error_no_dirauth_relay b/src/test/conf_examples/relay_08/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_08/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_08/expected b/src/test/conf_examples/relay_08/expected
new file mode 100644
index 0000000000..904c7339e0
--- /dev/null
+++ b/src/test/conf_examples/relay_08/expected
@@ -0,0 +1,3 @@
+BridgeRelay 1
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_08/expected_log b/src/test/conf_examples/relay_08/expected_log
new file mode 100644
index 0000000000..b0168c803d
--- /dev/null
+++ b/src/test/conf_examples/relay_08/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_08[./]*torrc
diff --git a/src/test/conf_examples/relay_08/torrc b/src/test/conf_examples/relay_08/torrc
new file mode 100644
index 0000000000..9e2ff9465c
--- /dev/null
+++ b/src/test/conf_examples/relay_08/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid config with all the bridge options
+ORPort 1
+BridgeRelay 1
+DirCache 1
diff --git a/src/test/conf_examples/relay_09/error_no_dirauth_relay b/src/test/conf_examples/relay_09/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_09/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_09/expected b/src/test/conf_examples/relay_09/expected
new file mode 100644
index 0000000000..ae58cee1af
--- /dev/null
+++ b/src/test/conf_examples/relay_09/expected
@@ -0,0 +1,3 @@
+DirPort 2
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_09/expected_log b/src/test/conf_examples/relay_09/expected_log
new file mode 100644
index 0000000000..d3ab4f6593
--- /dev/null
+++ b/src/test/conf_examples/relay_09/expected_log
@@ -0,0 +1 @@
+By default, Tor does not run as an exit relay
diff --git a/src/test/conf_examples/relay_09/torrc b/src/test/conf_examples/relay_09/torrc
new file mode 100644
index 0000000000..014eeca34b
--- /dev/null
+++ b/src/test/conf_examples/relay_09/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid config with all the non-bridge options
+ORPort 1
+DirPort 2
+DirCache 1
diff --git a/src/test/conf_examples/relay_10/error_no_dirauth_relay b/src/test/conf_examples/relay_10/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_10/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_10/expected b/src/test/conf_examples/relay_10/expected
new file mode 100644
index 0000000000..904c7339e0
--- /dev/null
+++ b/src/test/conf_examples/relay_10/expected
@@ -0,0 +1,3 @@
+BridgeRelay 1
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_10/expected_log b/src/test/conf_examples/relay_10/expected_log
new file mode 100644
index 0000000000..5b81a904e5
--- /dev/null
+++ b/src/test/conf_examples/relay_10/expected_log
@@ -0,0 +1 @@
+Can't set a DirPort on a bridge relay
diff --git a/src/test/conf_examples/relay_10/torrc b/src/test/conf_examples/relay_10/torrc
new file mode 100644
index 0000000000..4318ebb45b
--- /dev/null
+++ b/src/test/conf_examples/relay_10/torrc
@@ -0,0 +1,7 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid config, that has a warning: Bridge, warn and disable DirPort
+ORPort 1
+DirPort 2
+DirCache 1
+BridgeRelay 1
diff --git a/src/test/conf_examples/relay_11/error b/src/test/conf_examples/relay_11/error
new file mode 100644
index 0000000000..8ed5c31bc7
--- /dev/null
+++ b/src/test/conf_examples/relay_11/error
@@ -0,0 +1 @@
+We are advertising an ORPort, but not actually listening on one
diff --git a/src/test/conf_examples/relay_11/error_no_dirauth_relay b/src/test/conf_examples/relay_11/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_11/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_11/torrc b/src/test/conf_examples/relay_11/torrc
new file mode 100644
index 0000000000..a1e13eb3ce
--- /dev/null
+++ b/src/test/conf_examples/relay_11/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Advertising but not listening: ORPort
+ORPort 1 NoListen
diff --git a/src/test/conf_examples/relay_12/error b/src/test/conf_examples/relay_12/error
new file mode 100644
index 0000000000..57706d6a7a
--- /dev/null
+++ b/src/test/conf_examples/relay_12/error
@@ -0,0 +1 @@
+We are advertising a DirPort, but not actually listening on one
diff --git a/src/test/conf_examples/relay_12/error_no_dirauth_relay b/src/test/conf_examples/relay_12/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_12/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_12/torrc b/src/test/conf_examples/relay_12/torrc
new file mode 100644
index 0000000000..4a7d398112
--- /dev/null
+++ b/src/test/conf_examples/relay_12/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Advertising but not listening: DirPort
+DirPort 1 NoListen
diff --git a/src/test/conf_examples/relay_13/error b/src/test/conf_examples/relay_13/error
new file mode 100644
index 0000000000..cd74247ea8
--- /dev/null
+++ b/src/test/conf_examples/relay_13/error
@@ -0,0 +1 @@
+We are listening on an ORPort, but not advertising any ORPorts
diff --git a/src/test/conf_examples/relay_13/error_no_dirauth_relay b/src/test/conf_examples/relay_13/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_13/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_13/torrc b/src/test/conf_examples/relay_13/torrc
new file mode 100644
index 0000000000..b76b72c0cc
--- /dev/null
+++ b/src/test/conf_examples/relay_13/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Listening but not advertising: ORPort
+ORPort 1 NoAdvertise
diff --git a/src/test/conf_examples/relay_14/error_no_dirauth_relay b/src/test/conf_examples/relay_14/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_14/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_14/expected b/src/test/conf_examples/relay_14/expected
new file mode 100644
index 0000000000..31bb1c2507
--- /dev/null
+++ b/src/test/conf_examples/relay_14/expected
@@ -0,0 +1 @@
+DirPort 1 NoAdvertise
diff --git a/src/test/conf_examples/relay_14/expected_log b/src/test/conf_examples/relay_14/expected_log
new file mode 100644
index 0000000000..9b0c820211
--- /dev/null
+++ b/src/test/conf_examples/relay_14/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_14[./]*torrc
diff --git a/src/test/conf_examples/relay_14/torrc b/src/test/conf_examples/relay_14/torrc
new file mode 100644
index 0000000000..15c6496c7e
--- /dev/null
+++ b/src/test/conf_examples/relay_14/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Listening but not advertising: DirPort
+DirPort 1 NoAdvertise
diff --git a/src/test/conf_examples/relay_15/error b/src/test/conf_examples/relay_15/error
new file mode 100644
index 0000000000..da30f0cd14
--- /dev/null
+++ b/src/test/conf_examples/relay_15/error
@@ -0,0 +1 @@
+Can't advertise more than one DirPort
diff --git a/src/test/conf_examples/relay_15/error_no_dirauth_relay b/src/test/conf_examples/relay_15/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_15/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_15/torrc b/src/test/conf_examples/relay_15/torrc
new file mode 100644
index 0000000000..e1f78ee6a0
--- /dev/null
+++ b/src/test/conf_examples/relay_15/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Advertising more than one DirPort
+DirPort 1
+DirPort 2
diff --git a/src/test/conf_examples/relay_16/error b/src/test/conf_examples/relay_16/error
new file mode 100644
index 0000000000..37b89ee572
--- /dev/null
+++ b/src/test/conf_examples/relay_16/error
@@ -0,0 +1 @@
+Configured public relay to listen only on an IPv6 address. Tor needs to listen on an IPv4 address
diff --git a/src/test/conf_examples/relay_16/error_no_dirauth_relay b/src/test/conf_examples/relay_16/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_16/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_16/torrc b/src/test/conf_examples/relay_16/torrc
new file mode 100644
index 0000000000..e544cd87a4
--- /dev/null
+++ b/src/test/conf_examples/relay_16/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# IPv6 ORPort only
+ORPort [::1]:2
diff --git a/src/test/conf_examples/relay_17/error_no_dirauth_relay b/src/test/conf_examples/relay_17/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_17/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_17/expected b/src/test/conf_examples/relay_17/expected
new file mode 100644
index 0000000000..3fb0c9db92
--- /dev/null
+++ b/src/test/conf_examples/relay_17/expected
@@ -0,0 +1,4 @@
+AccountingMax 1
+KeepBindCapabilities 0
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_17/expected_log b/src/test/conf_examples/relay_17/expected_log
new file mode 100644
index 0000000000..7711f90178
--- /dev/null
+++ b/src/test/conf_examples/relay_17/expected_log
@@ -0,0 +1 @@
+You have set AccountingMax to use hibernation
diff --git a/src/test/conf_examples/relay_17/torrc b/src/test/conf_examples/relay_17/torrc
new file mode 100644
index 0000000000..f63f36815e
--- /dev/null
+++ b/src/test/conf_examples/relay_17/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Rebind warning
+ORPort 1
+AccountingMax 1
+KeepBindCapabilities 0
diff --git a/src/test/conf_examples/relay_18/error b/src/test/conf_examples/relay_18/error
new file mode 100644
index 0000000000..5b28d311b0
--- /dev/null
+++ b/src/test/conf_examples/relay_18/error
@@ -0,0 +1 @@
+Invalid DirPort configuration
diff --git a/src/test/conf_examples/relay_18/error_no_dirauth_relay b/src/test/conf_examples/relay_18/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_18/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_18/torrc b/src/test/conf_examples/relay_18/torrc
new file mode 100644
index 0000000000..67a0fd0dfb
--- /dev/null
+++ b/src/test/conf_examples/relay_18/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Bad DirPort
+DirPort illegal_hostname_chars$()^*%(%#%)#(%*
diff --git a/src/test/conf_examples/relay_19/error_no_dirauth_relay b/src/test/conf_examples/relay_19/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_19/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_19/expected b/src/test/conf_examples/relay_19/expected
new file mode 100644
index 0000000000..f077169c88
--- /dev/null
+++ b/src/test/conf_examples/relay_19/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 1
+PublishServerDescriptor
diff --git a/src/test/conf_examples/relay_19/expected_log b/src/test/conf_examples/relay_19/expected_log
new file mode 100644
index 0000000000..17656ba2cf
--- /dev/null
+++ b/src/test/conf_examples/relay_19/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_19[./]*torrc
diff --git a/src/test/conf_examples/relay_19/torrc b/src/test/conf_examples/relay_19/torrc
new file mode 100644
index 0000000000..fd2cd91fa5
--- /dev/null
+++ b/src/test/conf_examples/relay_19/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Minimal PublishServerDescriptor
+ORPort 1
+PublishServerDescriptor
diff --git a/src/test/conf_examples/relay_20/error b/src/test/conf_examples/relay_20/error
new file mode 100644
index 0000000000..e5a81637f8
--- /dev/null
+++ b/src/test/conf_examples/relay_20/error
@@ -0,0 +1 @@
+Unrecognized value in PublishServerDescriptor
diff --git a/src/test/conf_examples/relay_20/error_no_dirauth_relay b/src/test/conf_examples/relay_20/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_20/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_20/torrc b/src/test/conf_examples/relay_20/torrc
new file mode 100644
index 0000000000..87dd74fdc1
--- /dev/null
+++ b/src/test/conf_examples/relay_20/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Invalid PublishServerDescriptor
+ORPort 1
+PublishServerDescriptor bad
diff --git a/src/test/conf_examples/relay_21/error_no_dirauth_relay b/src/test/conf_examples/relay_21/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_21/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_21/expected b/src/test/conf_examples/relay_21/expected
new file mode 100644
index 0000000000..9bcead1402
--- /dev/null
+++ b/src/test/conf_examples/relay_21/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 1
+PublishServerDescriptor v1,v2,hidserv
diff --git a/src/test/conf_examples/relay_21/expected_log b/src/test/conf_examples/relay_21/expected_log
new file mode 100644
index 0000000000..ba0d56fb1c
--- /dev/null
+++ b/src/test/conf_examples/relay_21/expected_log
@@ -0,0 +1 @@
+PublishServerDescriptor v1 has no effect
diff --git a/src/test/conf_examples/relay_21/torrc b/src/test/conf_examples/relay_21/torrc
new file mode 100644
index 0000000000..97f032f626
--- /dev/null
+++ b/src/test/conf_examples/relay_21/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Ignored PublishServerDescriptor values
+ORPort 1
+PublishServerDescriptor v1,v2,hidserv
diff --git a/src/test/conf_examples/relay_22/error b/src/test/conf_examples/relay_22/error
new file mode 100644
index 0000000000..c47dd8c4c6
--- /dev/null
+++ b/src/test/conf_examples/relay_22/error
@@ -0,0 +1 @@
+Invalid BridgeDistribution value
diff --git a/src/test/conf_examples/relay_22/error_no_dirauth_relay b/src/test/conf_examples/relay_22/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_22/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_22/torrc b/src/test/conf_examples/relay_22/torrc
new file mode 100644
index 0000000000..e83c83260e
--- /dev/null
+++ b/src/test/conf_examples/relay_22/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Bad BridgeDistribution characters
+ORPort 1
+BridgeRelay 1
+BridgeDistribution *$%()@!
diff --git a/src/test/conf_examples/relay_23/error b/src/test/conf_examples/relay_23/error
new file mode 100644
index 0000000000..f76bbe77c4
--- /dev/null
+++ b/src/test/conf_examples/relay_23/error
@@ -0,0 +1 @@
+Relays must use 'auto' for the ConnectionPadding setting
diff --git a/src/test/conf_examples/relay_23/error_no_dirauth_relay b/src/test/conf_examples/relay_23/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_23/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_23/torrc b/src/test/conf_examples/relay_23/torrc
new file mode 100644
index 0000000000..3d28a1e27c
--- /dev/null
+++ b/src/test/conf_examples/relay_23/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ConnectionPadding
+ORPort 1
+ConnectionPadding 1
diff --git a/src/test/conf_examples/relay_24/error b/src/test/conf_examples/relay_24/error
new file mode 100644
index 0000000000..f76bbe77c4
--- /dev/null
+++ b/src/test/conf_examples/relay_24/error
@@ -0,0 +1 @@
+Relays must use 'auto' for the ConnectionPadding setting
diff --git a/src/test/conf_examples/relay_24/error_no_dirauth_relay b/src/test/conf_examples/relay_24/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_24/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_24/torrc b/src/test/conf_examples/relay_24/torrc
new file mode 100644
index 0000000000..1206e59e09
--- /dev/null
+++ b/src/test/conf_examples/relay_24/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ConnectionPadding
+ORPort 1
+ConnectionPadding 0
diff --git a/src/test/conf_examples/relay_25/error b/src/test/conf_examples/relay_25/error
new file mode 100644
index 0000000000..bac681e6cc
--- /dev/null
+++ b/src/test/conf_examples/relay_25/error
@@ -0,0 +1 @@
+Relays cannot set ReducedConnectionPadding
diff --git a/src/test/conf_examples/relay_25/error_no_dirauth_relay b/src/test/conf_examples/relay_25/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_25/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_25/torrc b/src/test/conf_examples/relay_25/torrc
new file mode 100644
index 0000000000..ab862a16f3
--- /dev/null
+++ b/src/test/conf_examples/relay_25/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ReducedConnectionPadding 1
+ORPort 1
+ReducedConnectionPadding 1
diff --git a/src/test/conf_examples/relay_26/error b/src/test/conf_examples/relay_26/error
new file mode 100644
index 0000000000..94334935e3
--- /dev/null
+++ b/src/test/conf_examples/relay_26/error
@@ -0,0 +1 @@
+Relays cannot set CircuitPadding to 0
diff --git a/src/test/conf_examples/relay_26/error_no_dirauth_relay b/src/test/conf_examples/relay_26/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_26/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_26/torrc b/src/test/conf_examples/relay_26/torrc
new file mode 100644
index 0000000000..5dd6d68dc4
--- /dev/null
+++ b/src/test/conf_examples/relay_26/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set CircuitPadding to 0
+ORPort 1
+CircuitPadding 0
diff --git a/src/test/conf_examples/relay_27/error b/src/test/conf_examples/relay_27/error
new file mode 100644
index 0000000000..e26ce46914
--- /dev/null
+++ b/src/test/conf_examples/relay_27/error
@@ -0,0 +1 @@
+Relays cannot set ReducedCircuitPadding
diff --git a/src/test/conf_examples/relay_27/error_no_dirauth_relay b/src/test/conf_examples/relay_27/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_27/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_27/torrc b/src/test/conf_examples/relay_27/torrc
new file mode 100644
index 0000000000..8556b2f351
--- /dev/null
+++ b/src/test/conf_examples/relay_27/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ReducedCircuitPadding 1
+ORPort 1
+ReducedCircuitPadding 1
diff --git a/src/test/conf_examples/relay_28/error b/src/test/conf_examples/relay_28/error
new file mode 100644
index 0000000000..3f14df975b
--- /dev/null
+++ b/src/test/conf_examples/relay_28/error
@@ -0,0 +1 @@
+SigningKeyLifetime is too short
diff --git a/src/test/conf_examples/relay_28/error_no_dirauth_relay b/src/test/conf_examples/relay_28/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_28/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_28/torrc b/src/test/conf_examples/relay_28/torrc
new file mode 100644
index 0000000000..3e2c895bb7
--- /dev/null
+++ b/src/test/conf_examples/relay_28/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Short key lifetimes
+ORPort 1
+SigningKeyLifetime 1
diff --git a/src/test/conf_examples/relay_29/error_no_dirauth_relay b/src/test/conf_examples/relay_29/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_29/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_29/expected b/src/test/conf_examples/relay_29/expected
new file mode 100644
index 0000000000..1d25374ed1
--- /dev/null
+++ b/src/test/conf_examples/relay_29/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_29/expected_log b/src/test/conf_examples/relay_29/expected_log
new file mode 100644
index 0000000000..f46c609c0c
--- /dev/null
+++ b/src/test/conf_examples/relay_29/expected_log
@@ -0,0 +1 @@
+Linelist option 'MyFamily' has no value\. Skipping
diff --git a/src/test/conf_examples/relay_29/torrc b/src/test/conf_examples/relay_29/torrc
new file mode 100644
index 0000000000..4181d5acc2
--- /dev/null
+++ b/src/test/conf_examples/relay_29/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# MyFamily normalisation: empty MyFamily
+ORPort 1
+MyFamily
diff --git a/src/test/conf_examples/relay_30/error_no_dirauth_relay b/src/test/conf_examples/relay_30/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_30/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_30/expected b/src/test/conf_examples/relay_30/expected
new file mode 100644
index 0000000000..3a4e9feb3f
--- /dev/null
+++ b/src/test/conf_examples/relay_30/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort auto
diff --git a/src/test/conf_examples/relay_30/expected_log b/src/test/conf_examples/relay_30/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_30/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_30/torrc b/src/test/conf_examples/relay_30/torrc
new file mode 100644
index 0000000000..bf8487fe16
--- /dev/null
+++ b/src/test/conf_examples/relay_30/torrc
@@ -0,0 +1,3 @@
+# Relay tests
+# default (IPv4) ORPort auto
+ORPort auto
diff --git a/src/test/conf_examples/relay_31/error_no_dirauth_relay b/src/test/conf_examples/relay_31/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_31/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_31/expected b/src/test/conf_examples/relay_31/expected
new file mode 100644
index 0000000000..9a40cdd588
--- /dev/null
+++ b/src/test/conf_examples/relay_31/expected
@@ -0,0 +1,3 @@
+DirPort auto
+Nickname Unnamed
+ORPort auto
diff --git a/src/test/conf_examples/relay_31/expected_log b/src/test/conf_examples/relay_31/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_31/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_31/torrc b/src/test/conf_examples/relay_31/torrc
new file mode 100644
index 0000000000..e662bb71b3
--- /dev/null
+++ b/src/test/conf_examples/relay_31/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# default (IPv4) ORPort and DirPort auto
+ORPort auto
+DirPort auto
diff --git a/src/test/conf_examples/relay_32/error_no_dirauth_relay b/src/test/conf_examples/relay_32/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_32/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_32/expected b/src/test/conf_examples/relay_32/expected
new file mode 100644
index 0000000000..14b36c8259
--- /dev/null
+++ b/src/test/conf_examples/relay_32/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_32/expected_log b/src/test/conf_examples/relay_32/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_32/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_32/torrc b/src/test/conf_examples/relay_32/torrc
new file mode 100644
index 0000000000..95a66c4852
--- /dev/null
+++ b/src/test/conf_examples/relay_32/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# default (IPv4) ORPort auto and IPv6 ORPort auto
+ORPort auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_33/error_no_dirauth_relay b/src/test/conf_examples/relay_33/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_33/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_33/expected b/src/test/conf_examples/relay_33/expected
new file mode 100644
index 0000000000..22567cbe2e
--- /dev/null
+++ b/src/test/conf_examples/relay_33/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_33/expected_log b/src/test/conf_examples/relay_33/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_33/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_33/torrc b/src/test/conf_examples/relay_33/torrc
new file mode 100644
index 0000000000..44d16ad31a
--- /dev/null
+++ b/src/test/conf_examples/relay_33/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# explicit IPv4 ORPort auto and IPv6 ORPort auto
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_34/error_no_dirauth_relay b/src/test/conf_examples/relay_34/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_34/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_34/expected b/src/test/conf_examples/relay_34/expected
new file mode 100644
index 0000000000..bccde684f3
--- /dev/null
+++ b/src/test/conf_examples/relay_34/expected
@@ -0,0 +1,4 @@
+DirPort 127.0.0.1:auto
+Nickname Unnamed
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_34/expected_log b/src/test/conf_examples/relay_34/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_34/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_34/torrc b/src/test/conf_examples/relay_34/torrc
new file mode 100644
index 0000000000..01010a5c38
--- /dev/null
+++ b/src/test/conf_examples/relay_34/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# explicit IPv4 ORPort and DirPort auto and IPv6 ORPort auto
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
+DirPort 127.0.0.1:auto
diff --git a/src/test/conf_examples/relpath_rad/error b/src/test/conf_examples/relpath_rad/error
new file mode 100644
index 0000000000..e131744475
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/error
@@ -0,0 +1 @@
+RunAsDaemon is not compatible with relative paths.
diff --git a/src/test/conf_examples/relpath_rad/torrc b/src/test/conf_examples/relpath_rad/torrc
new file mode 100644
index 0000000000..fe02441c3f
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/torrc
@@ -0,0 +1,4 @@
+
+# Relative-path data directories are incompatible with RunAsDaemon
+DataDirectory ./datadir
+RunAsDaemon 1
diff --git a/src/test/conf_failures/README b/src/test/conf_failures/README
new file mode 100644
index 0000000000..0da470eeb4
--- /dev/null
+++ b/src/test/conf_failures/README
@@ -0,0 +1,5 @@
+This directory contains typical test_parseconf.sh failure cases.
+
+If these directories are copied into conf_examples, test_parseconf.sh will
+fail. Use these failure cases to make sure test_parseconf.sh handles failures
+correctly, and produces useful output.
diff --git a/src/test/conf_failures/fail-error-success/error b/src/test/conf_failures/fail-error-success/error
new file mode 100644
index 0000000000..569a631e86
--- /dev/null
+++ b/src/test/conf_failures/fail-error-success/error
@@ -0,0 +1 @@
+Tor
diff --git a/src/test/conf_failures/fail-error-success/torrc b/src/test/conf_failures/fail-error-success/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-error-success/torrc
diff --git a/src/test/conf_failures/fail-error/error b/src/test/conf_failures/fail-error/error
new file mode 100644
index 0000000000..4c0be97832
--- /dev/null
+++ b/src/test/conf_failures/fail-error/error
@@ -0,0 +1 @@
+no match
diff --git a/src/test/conf_failures/fail-error/torrc b/src/test/conf_failures/fail-error/torrc
new file mode 100644
index 0000000000..bb6fe186a4
--- /dev/null
+++ b/src/test/conf_failures/fail-error/torrc
@@ -0,0 +1 @@
+bad bad bad
diff --git a/src/test/conf_failures/fail-expected-error/expected b/src/test/conf_failures/fail-expected-error/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-error/expected
diff --git a/src/test/conf_failures/fail-expected-error/torrc b/src/test/conf_failures/fail-expected-error/torrc
new file mode 100644
index 0000000000..bb6fe186a4
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-error/torrc
@@ -0,0 +1 @@
+bad bad bad
diff --git a/src/test/conf_failures/fail-expected-log/expected b/src/test/conf_failures/fail-expected-log/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-log/expected
diff --git a/src/test/conf_failures/fail-expected-log/expected_log b/src/test/conf_failures/fail-expected-log/expected_log
new file mode 100644
index 0000000000..bb6fe186a4
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-log/expected_log
@@ -0,0 +1 @@
+bad bad bad
diff --git a/src/test/conf_failures/fail-expected-log/torrc b/src/test/conf_failures/fail-expected-log/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-log/torrc
diff --git a/src/test/conf_failures/fail-expected/expected b/src/test/conf_failures/fail-expected/expected
new file mode 100644
index 0000000000..67be85f127
--- /dev/null
+++ b/src/test/conf_failures/fail-expected/expected
@@ -0,0 +1 @@
+bad
diff --git a/src/test/conf_failures/fail-expected/torrc b/src/test/conf_failures/fail-expected/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected/torrc
diff --git a/src/test/ed25519_exts_ref.py b/src/test/ed25519_exts_ref.py
index 75562184b5..ae537ff15b 100644
--- a/src/test/ed25519_exts_ref.py
+++ b/src/test/ed25519_exts_ref.py
@@ -8,6 +8,11 @@
    Includes self-tester and test vector generator.
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import slow_ed25519
 from slow_ed25519 import *
 
@@ -147,7 +152,7 @@ class SelfTest(unittest.TestCase):
         # Check that identities match
         assert(identity == identity2)
         # Check that identity is the point (0,1)
-        assert(identity == [0L,1L])
+        assert(identity == [0,1])
 
         # Check identity element: a*E = E, where a is a random scalar
         scalar = random_scalar(os.urandom)
@@ -181,22 +186,22 @@ BLINDING_PARAMS = [
 PREFIX = "ED25519_"
 
 def writeArray(name, array):
-    print "static const char *{prefix}{name}[] = {{".format(
-        prefix=PREFIX,name=name)
+    print("static const char *{prefix}{name}[] = {{".format(
+        prefix=PREFIX,name=name))
     for a in array:
         h = binascii.b2a_hex(a)
         if len(h) > 70:
             h1 = h[:70]
             h2 = h[70:]
-            print '  "{0}"\n      "{1}",'.format(h1,h2)
+            print('  "{0}"\n      "{1}",'.format(h1,h2))
         else:
-            print '  "{0}",'.format(h)
-    print "};\n"
+            print('  "{0}",'.format(h))
+    print("};\n")
 
 def comment(text, initial="/**"):
-    print initial
-    print textwrap.fill(text,initial_indent=" * ",subsequent_indent=" * ")
-    print " */"
+    print(initial)
+    print(textwrap.fill(text,initial_indent=" * ",subsequent_indent=" * "))
+    print(" */")
 
 def makeTestVectors():
     comment("""Test vectors for our ed25519 implementation and related
@@ -252,7 +257,7 @@ def makeTestVectors():
 if __name__ == '__main__':
     import sys
     if len(sys.argv) == 1 or sys.argv[1] not in ("SelfTest", "MakeVectors"):
-        print "You should specify one of 'SelfTest' or 'MakeVectors'"
+        print("You should specify one of 'SelfTest' or 'MakeVectors'")
         sys.exit(1)
     if sys.argv[1] == 'SelfTest':
         unittest.main()
diff --git a/src/test/fakechans.h b/src/test/fakechans.h
index 4006e1bec4..c3accb1637 100644
--- a/src/test/fakechans.h
+++ b/src/test/fakechans.h
@@ -1,4 +1,4 @@
- /* Copyright (c) 2014-2019, The Tor Project, Inc. */
+ /* Copyright (c) 2014-2020, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 #ifndef TOR_FAKECHANS_H
diff --git a/src/test/fakecircs.c b/src/test/fakecircs.c
new file mode 100644
index 0000000000..4d5b97197e
--- /dev/null
+++ b/src/test/fakecircs.c
@@ -0,0 +1,91 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fakecircs.c
+ * \brief Fake circuits API for unit test.
+ **/
+
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CRYPT_PATH_PRIVATE
+
+#include "core/or/or.h"
+
+#include "core/crypto/relay_crypto.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/crypt_path.h"
+#include "core/or/relay.h"
+#include "core/or/relay_crypto_st.h"
+
+#include "test/fakecircs.h"
+
+/** Return newly allocated OR circuit using the given nchan and pchan. It must
+ * be freed with the free_fake_orcirc(). */
+or_circuit_t *
+new_fake_orcirc(channel_t *nchan, channel_t *pchan)
+{
+  or_circuit_t *orcirc = NULL;
+  circuit_t *circ = NULL;
+  crypt_path_t tmp_cpath;
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+
+  orcirc = tor_malloc_zero(sizeof(*orcirc));
+  circ = &(orcirc->base_);
+  circ->magic = OR_CIRCUIT_MAGIC;
+
+  circuit_set_n_circid_chan(circ, get_unique_circ_id_by_chan(nchan), nchan);
+  cell_queue_init(&(circ->n_chan_cells));
+
+  circ->n_hop = NULL;
+  circ->streams_blocked_on_n_chan = 0;
+  circ->streams_blocked_on_p_chan = 0;
+  circ->n_delete_pending = 0;
+  circ->p_delete_pending = 0;
+  circ->received_destroy = 0;
+  circ->state = CIRCUIT_STATE_OPEN;
+  circ->purpose = CIRCUIT_PURPOSE_OR;
+  circ->package_window = CIRCWINDOW_START_MAX;
+  circ->deliver_window = CIRCWINDOW_START_MAX;
+  circ->n_chan_create_cell = NULL;
+
+  circuit_set_p_circid_chan(orcirc, get_unique_circ_id_by_chan(pchan), pchan);
+  cell_queue_init(&(orcirc->p_chan_cells));
+
+  memset(&tmp_cpath, 0, sizeof(tmp_cpath));
+  if (cpath_init_circuit_crypto(&tmp_cpath, whatevs_key,
+                                sizeof(whatevs_key), 0, 0)<0) {
+    log_warn(LD_BUG,"Circuit initialization failed");
+    return NULL;
+  }
+  orcirc->crypto = tmp_cpath.pvt_crypto;
+
+  return orcirc;
+}
+
+/** Free fake OR circuit which MUST be created by new_fake_orcirc(). */
+void
+free_fake_orcirc(or_circuit_t *orcirc)
+{
+  if (!orcirc) {
+    return;
+  }
+
+  circuit_t *circ = TO_CIRCUIT(orcirc);
+
+  relay_crypto_clear(&orcirc->crypto);
+
+  circpad_circuit_free_all_machineinfos(circ);
+
+  if (orcirc->p_chan && orcirc->p_chan->cmux) {
+    circuitmux_detach_circuit(orcirc->p_chan->cmux, circ);
+  }
+  if (circ->n_chan && circ->n_chan->cmux) {
+    circuitmux_detach_circuit(circ->n_chan->cmux, circ);
+  }
+
+  tor_free_(circ);
+}
diff --git a/src/test/fakecircs.h b/src/test/fakecircs.h
new file mode 100644
index 0000000000..ed8a150a3f
--- /dev/null
+++ b/src/test/fakecircs.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fakecircs.h
+ * \brief Declarations for fake circuits for test suite use.
+ **/
+
+#ifndef TOR_FAKECIRCS_H
+#define TOR_FAKECIRCS_H
+
+#include "core/or/or_circuit_st.h"
+
+or_circuit_t *new_fake_orcirc(channel_t *nchan, channel_t *pchan);
+void free_fake_orcirc(or_circuit_t *orcirc);
+
+#endif /* !defined(TOR_FAKECIRCS_H) */
diff --git a/src/test/fuzz/.may_include b/src/test/fuzz/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/test/fuzz/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh
index 68efc1abc5..f730d532a5 100755
--- a/src/test/fuzz/fixup_filenames.sh
+++ b/src/test/fuzz/fixup_filenames.sh
@@ -8,9 +8,9 @@ if [ ! -d "$1" ] ; then
 fi
 
 for fn in "$1"/* ; do
-    prev=`basename "$fn"`
-    post=`sha256sum "$fn" | sed -e 's/ .*//;'`
-    if [ "$prev" == "$post" ] ; then
+    prev=$(basename "$fn")
+    post=$(sha256sum "$fn" | sed -e 's/ .*//;')
+    if [ "$prev" = "$post" ] ; then
       echo "OK $prev"
     else
       echo "mv $prev $post"
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
index 5947a3f48c..f2bf29ea78 100644
--- a/src/test/fuzz/fuzz_consensus.c
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define SIGCOMMON_PRIVATE
 #include "core/or/or.h"
@@ -61,13 +61,13 @@ int
 fuzz_main(const uint8_t *data, size_t sz)
 {
   networkstatus_t *ns;
-  char *str = tor_memdup_nulterm(data, sz);
   const char *eos = NULL;
   networkstatus_type_t tp = NS_TYPE_CONSENSUS;
   if (tor_memstr(data, MIN(sz, 1024), "tus vote"))
     tp = NS_TYPE_VOTE;
   const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote";
-  ns = networkstatus_parse_vote_from_string(str,
+  ns = networkstatus_parse_vote_from_string((const char *)data,
+                                            sz,
                                             &eos,
                                             tp);
   if (ns) {
@@ -76,6 +76,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   } else {
     log_debug(LD_GENERAL, "Parsing as %s failed", what);
   }
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c
index 58ee3dbc35..eb4d4d507f 100644
--- a/src/test/fuzz/fuzz_descriptor.c
+++ b/src/test/fuzz/fuzz_descriptor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define SIGCOMMON_PRIVATE
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c
index 1bc60e50ee..9cd2116245 100644
--- a/src/test/fuzz/fuzz_diff.c
+++ b/src/test/fuzz/fuzz_diff.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFF_PRIVATE
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -42,28 +44,34 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_generate(c1, c2);
+  const char *cp = memchr(c1, 0, c1_len);
+  if (cp)
+    c1_len = cp - c1;
+
+  cp = memchr(c2, 0, c2_len);
+  if (cp)
+    c2_len = cp - c2;
+
+  char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len);
 
   if (c3) {
-    char *c4 = consensus_diff_apply(c1, c3);
+    char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3));
     tor_assert(c4);
-    if (strcmp(c2, c4)) {
-      printf("%s\n", escaped(c1));
-      printf("%s\n", escaped(c2));
+    int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len);
+    if (! equal) {
+      //printf("%s\n", escaped(c1));
+      //printf("%s\n", escaped(c2));
       printf("%s\n", escaped(c3));
       printf("%s\n", escaped(c4));
     }
-    tor_assert(! strcmp(c2, c4));
+    tor_assert(equal);
     tor_free(c3);
     tor_free(c4);
   }
-  tor_free(c1);
-  tor_free(c2);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c
index 9bd3cb0bf8..a819c73338 100644
--- a/src/test/fuzz/fuzz_diff_apply.c
+++ b/src/test/fuzz/fuzz_diff_apply.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFF_PRIVATE
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_apply(c1, c2);
+  char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len);
 
-  tor_free(c1);
-  tor_free(c2);
   tor_free(c3);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c
index f18bd68d65..ad21254e3e 100644
--- a/src/test/fuzz/fuzz_extrainfo.c
+++ b/src/test/fuzz/fuzz_extrainfo.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define SIGCOMMON_PRIVATE
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c
index 34639b237c..81d9e5f00e 100644
--- a/src/test/fuzz/fuzz_hsdescv2.c
+++ b/src/test/fuzz/fuzz_hsdescv2.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #include "core/or/or.h"
 #include "feature/dirparse/unparseable.h"
diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c
index 2cbd655898..8d7eab1a8d 100644
--- a/src/test/fuzz/fuzz_hsdescv3.c
+++ b/src/test/fuzz/fuzz_hsdescv3.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define HS_DESCRIPTOR_PRIVATE
@@ -35,16 +35,21 @@ mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert,
 
 static size_t
 mock_decrypt_desc_layer(const hs_descriptor_t *desc,
-                        const uint8_t *encrypted_blob,
-                        size_t encrypted_blob_size,
                         const uint8_t *descriptor_cookie,
-                        int is_superencrypted_layer,
+                        bool is_superencrypted_layer,
                         char **decrypted_out)
 {
   (void)is_superencrypted_layer;
   (void)desc;
   (void)descriptor_cookie;
   const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
+
   if (encrypted_blob_size < overhead)
     return 0;
   *decrypted_out = tor_memdup_nulterm(
@@ -80,12 +85,12 @@ int
 fuzz_main(const uint8_t *data, size_t sz)
 {
   hs_descriptor_t *desc = NULL;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   char *fuzzing_data = tor_memdup_nulterm(data, sz);
-  memset(subcredential, 'A', sizeof(subcredential));
+  memset(&subcredential, 'A', sizeof(subcredential));
 
-  hs_desc_decode_descriptor(fuzzing_data, subcredential, NULL, &desc);
+  hs_desc_decode_descriptor(fuzzing_data, &subcredential, NULL, &desc);
   if (desc) {
     log_debug(LD_GENERAL, "Decoding okay");
     hs_descriptor_free(desc);
@@ -96,4 +101,3 @@ fuzz_main(const uint8_t *data, size_t sz)
   tor_free(fuzzing_data);
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
index 2798c47d23..a4fd182f1e 100644
--- a/src/test/fuzz/fuzz_http.c
+++ b/src/test/fuzz/fuzz_http.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/dircache/dircache.h"
diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c
index a60fc36804..9e5a48ba4d 100644
--- a/src/test/fuzz/fuzz_http_connect.c
+++ b/src/test/fuzz/fuzz_http_connect.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c
index 76fa3c164e..bc51ffcdb8 100644
--- a/src/test/fuzz/fuzz_iptsv2.c
+++ b/src/test/fuzz/fuzz_iptsv2.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c
index 28fdc5e24d..3fc709183b 100644
--- a/src/test/fuzz/fuzz_microdesc.c
+++ b/src/test/fuzz/fuzz_microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh
index b4a17ed8cb..406ab498d9 100755
--- a/src/test/fuzz/fuzz_multi.sh
+++ b/src/test/fuzz/fuzz_multi.sh
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 MEMLIMIT_BYTES=21990500990976
 
 N_CPUS=1
@@ -6,9 +8,9 @@ if [ $# -ge 1 ]; then
     shift
 fi
 
-FILTER=echo
+FILTER="echo"
 
-for i in `seq -w "$N_CPUS"`; do
+for i in $(seq -w "$N_CPUS"); do
     if [ "$i" -eq 1 ]; then
         if [ "$N_CPUS" -eq 1 ]; then
             INSTANCE=""
diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c
index 06cb08391e..4e7cb4d48d 100644
--- a/src/test/fuzz/fuzz_socks.c
+++ b/src/test/fuzz/fuzz_socks.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/err/backtrace.h"
 #include "lib/log/log.h"
 #include "core/proto/proto_socks.h"
diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c
new file mode 100644
index 0000000000..05d9101e72
--- /dev/null
+++ b/src/test/fuzz/fuzz_strops.c
@@ -0,0 +1,253 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fuzz_strops.c
+ * \brief Fuzzers for various string encoding/decoding operations
+ **/
+
+#include "orconfig.h"
+
+#include "lib/cc/torint.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/confline.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/muldiv.h"
+
+#include "test/fuzz/fuzzing.h"
+
+#include <stdio.h>
+#include <string.h>
+
+int
+fuzz_init(void)
+{
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+typedef struct chunk_t {
+  uint8_t *buf;
+  size_t len;
+} chunk_t;
+
+#define chunk_free(ch)                          \
+  FREE_AND_NULL(chunk_t, chunk_free_, (ch))
+
+static chunk_t *
+chunk_new(size_t len)
+{
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = tor_malloc(len);
+  ch->len = len;
+  return ch;
+}
+static void
+chunk_free_(chunk_t *ch)
+{
+  if (!ch)
+    return;
+  tor_free(ch->buf);
+  tor_free(ch);
+}
+static bool
+chunk_eq(const chunk_t *a, const chunk_t *b)
+{
+  return a->len == b->len && fast_memeq(a->buf, b->buf, a->len);
+}
+
+static chunk_t *
+b16_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2));
+  int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b16_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len * 2 + 1);
+  base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  return ch;
+}
+
+static chunk_t *
+b32_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX
+  int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b32_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(base32_encoded_size(inp->len));
+  base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+b64_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter.
+  int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b64_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len));
+  base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+c_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  char *r = NULL;
+  (void) unescape_string(s, &r, &ch->len);
+  tor_free(s);
+  ch->buf = (uint8_t*) r;
+  if (!ch->buf) {
+    tor_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+c_enc(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = (uint8_t*)esc_for_log(s);
+  tor_free(s);
+  ch->len = strlen((char*)ch->buf);
+  return ch;
+}
+
+static int kv_flags = 0;
+static config_line_t *
+kv_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  config_line_t *res = kvline_parse(s, kv_flags);
+  tor_free(s);
+  return res;
+}
+static chunk_t *
+kv_enc(const config_line_t *inp)
+{
+  char *s = kvline_encode(inp, kv_flags);
+  if (!s)
+    return NULL;
+  chunk_t *res = tor_malloc(sizeof(chunk_t));
+  res->buf = (uint8_t*)s;
+  res->len = strlen(s);
+  return res;
+}
+
+/* Given an encoder function, a decoder function, and a function to free
+ * the decoded object, check whether any string that successfully decoded
+ * will then survive an encode-decode-encode round-trip unchanged.
+ */
+#define ENCODE_ROUNDTRIP(E,D,FREE)              \
+  STMT_BEGIN {                                  \
+    bool err = false;                           \
+    a = D(&inp);                                \
+    if (!a)                                     \
+      return 0;                                 \
+    b = E(a);                                   \
+    tor_assert(b);                              \
+    c = D(b);                                   \
+    tor_assert(c);                              \
+    d = E(c);                                   \
+    tor_assert(d);                              \
+    if (!chunk_eq(b,d)) {                       \
+      printf("Unequal chunks: %s\n",            \
+             hex_str((char*)b->buf, b->len));   \
+      printf("             vs %s\n",            \
+             hex_str((char*)d->buf, d->len));   \
+      err = true;                               \
+    }                                           \
+    FREE(a);                                    \
+    chunk_free(b);                              \
+    FREE(c);                                    \
+    chunk_free(d);                              \
+    tor_assert(!err);                           \
+  } STMT_END
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  if (!data_size)
+    return 0;
+
+  chunk_t inp = { (uint8_t*)stdin_buf, data_size };
+  chunk_t *b=NULL,*d=NULL;
+  void *a=NULL,*c=NULL;
+
+  switch (stdin_buf[0]) {
+    case 0:
+      ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_);
+      break;
+    case 1:
+      ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_);
+      break;
+    case 2:
+      ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_);
+      break;
+    case 3:
+      ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_);
+      break;
+    case 5:
+      kv_flags = KV_QUOTED|KV_OMIT_KEYS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 6:
+      kv_flags = 0;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 7:
+      kv_flags = KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 8:
+      kv_flags = KV_QUOTED;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 9:
+      kv_flags = KV_QUOTED|KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    }
+
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
index 967397d1af..d6e88e59e7 100644
--- a/src/test/fuzz/fuzz_vrs.c
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -1,8 +1,9 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define NS_PARSE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #include "core/or/or.h"
+#include "feature/dirauth/dirvote.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/dirparse/unparseable.h"
 #include "lib/memarea/memarea.h"
@@ -35,9 +36,12 @@ fuzz_init(void)
   dummy_vote = tor_malloc_zero(sizeof(*dummy_vote));
   dummy_vote->known_flags = smartlist_new();
   smartlist_split_string(dummy_vote->known_flags,
-                         "Authority BadExit Exit Fast Guard HSDir "
-                         "NoEdConsensus Running Stable V2Dir Valid",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          " ", 0, 0);
+  smartlist_split_string(dummy_vote->known_flags,
+                         DIRVOTE_OPTIONAL_FLAGS,
+                         " ", 0, 0);
+  smartlist_sort_strings(dummy_vote->known_flags);
   return 0;
 }
 
@@ -53,24 +57,24 @@ fuzz_cleanup(void)
 int
 fuzz_main(const uint8_t *data, size_t sz)
 {
-  char *str = tor_memdup_nulterm(data, sz);
   const char *s;
   routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL;
   vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs));
   smartlist_t *tokens = smartlist_new();
+  const char *eos = (const char *)data + sz;
 
-  s = str;
-  rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_md = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_MICRODESC);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                               dummy_vote, vrs, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
@@ -82,6 +86,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   vote_routerstatus_free(vrs);
   memarea_clear(area);
   smartlist_free(tokens);
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h
index 150ac4aa7d..6cbcdc41ad 100644
--- a/src/test/fuzz/fuzzing.h
+++ b/src/test/fuzz/fuzzing.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #ifndef FUZZING_H
 #define FUZZING_H
@@ -9,5 +9,5 @@ int fuzz_main(const uint8_t *data, size_t sz);
 
 void disable_signature_checking(void);
 
-#endif /* FUZZING_H */
+#endif /* !defined(FUZZING_H) */
 
diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c
index 8ea4898522..d9719074ad 100644
--- a/src/test/fuzz/fuzzing_common.c
+++ b/src/test/fuzz/fuzzing_common.c
@@ -1,14 +1,17 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define CRYPTO_ED25519_PRIVATE
+#define CONFIG_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
+#include "app/main/subsysmgr.h"
 #include "lib/err/backtrace.h"
 #include "app/config/config.h"
 #include "test/fuzz/fuzzing.h"
 #include "lib/compress/compress.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 static or_options_t *mock_options = NULL;
 static const or_options_t *
@@ -94,12 +97,10 @@ disable_signature_checking(void)
 static void
 global_init(void)
 {
-  tor_threads_init();
-  tor_compress_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
 
-  /* Initialise logging first */
-  init_logging(1);
-  configure_backtrace_handler(get_version());
+  tor_compress_init();
 
   if (crypto_global_init(0, NULL, NULL) < 0)
     abort();
@@ -111,7 +112,7 @@ global_init(void)
   }
 
   /* set up the options. */
-  mock_options = tor_malloc_zero(sizeof(or_options_t));
+  mock_options = options_new();
   MOCK(get_options, mock_get_options);
 
   /* Make BUG() and nonfatal asserts crash */
@@ -137,7 +138,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
   return fuzz_main(Data, Size);
 }
 
-#else /* Not LLVM_FUZZ, so AFL. */
+#else /* !defined(LLVM_FUZZ) */
 
 int
 main(int argc, char **argv)
@@ -166,7 +167,7 @@ main(int argc, char **argv)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
 
@@ -189,9 +190,9 @@ main(int argc, char **argv)
   if (fuzz_cleanup() < 0)
     abort();
 
-  tor_free(mock_options);
+  or_options_free(mock_options);
   UNMOCK(get_options);
   return 0;
 }
 
-#endif
+#endif /* defined(LLVM_FUZZ) */
diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am
index 27eeced8c5..d0711f05d6 100644
--- a/src/test/fuzz/include.am
+++ b/src/test/fuzz/include.am
@@ -153,6 +153,16 @@ src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_strops_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_strops.c
+src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_fuzz_vrs_SOURCES = \
 	src/test/fuzz/fuzzing_common.c \
 	src/test/fuzz/fuzz_vrs.c
@@ -176,6 +186,7 @@ FUZZERS = \
 	src/test/fuzz/fuzz-iptsv2 \
 	src/test/fuzz/fuzz-microdesc \
 	src/test/fuzz/fuzz-socks \
+	src/test/fuzz/fuzz-strops \
 	src/test/fuzz/fuzz-vrs
 endif
 
@@ -291,6 +302,15 @@ src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_strops_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_lf_fuzz_vrs_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
@@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \
 	src/test/fuzz/lf-fuzz-iptsv2 \
 	src/test/fuzz/lf-fuzz-microdesc \
 	src/test/fuzz/lf-fuzz-socks \
+	src/test/fuzz/lf-fuzz-strops \
 	src/test/fuzz/lf-fuzz-vrs
 
 else
@@ -406,6 +427,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
@@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \
 	src/test/fuzz/liboss-fuzz-iptsv2.a \
 	src/test/fuzz/liboss-fuzz-microdesc.a \
 	src/test/fuzz/liboss-fuzz-socks.a \
+	src/test/fuzz/liboss-fuzz-strops.a \
 	src/test/fuzz/liboss-fuzz-vrs.a
 
 else
diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh
index 87d3dda13c..ce43812bb8 100755
--- a/src/test/fuzz/minimize.sh
+++ b/src/test/fuzz/minimize.sh
@@ -7,7 +7,7 @@ if [ ! -d "$1" ] ; then
     exit 1
 fi
 
-which=`basename "$1"`
+which=$(basename "$1")
 
 mkdir "$1.out"
 afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}"
diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh
index f7b3adffb1..b883352402 100755
--- a/src/test/fuzz_static_testcases.sh
+++ b/src/test/fuzz_static_testcases.sh
@@ -14,7 +14,7 @@ fi
 
 
 for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do
-    f=`basename $fuzzer`
+    f=$(basename "$fuzzer")
     case="${f#fuzz-}"
     if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then
         echo "Running tests for ${case}"
diff --git a/src/test/hs_build_address.py b/src/test/hs_build_address.py
index 7ff22c3a9a..91864eabcb 100644
--- a/src/test/hs_build_address.py
+++ b/src/test/hs_build_address.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/hs_indexes.py b/src/test/hs_indexes.py
index af0b81f8de..5c6d893a66 100644
--- a/src/test/hs_indexes.py
+++ b/src/test/hs_indexes.py
@@ -7,6 +7,11 @@
 # store/fetch the descriptor on the hashring. (hs_build_hs_index()).
 #
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/hs_ntor_ref.py b/src/test/hs_ntor_ref.py
index d58ac3ca23..98025dd584 100644
--- a/src/test/hs_ntor_ref.py
+++ b/src/test/hs_ntor_ref.py
@@ -41,6 +41,11 @@ The whole logic and concept for this test suite was taken from ntor_ref.py.
                 *** DO NOT USE THIS IN PRODUCTION. ***
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import struct
 import os, sys
 import binascii
diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c
index f2ae8398df..5116fc7169 100644
--- a/src/test/hs_test_helpers.c
+++ b/src/test/hs_test_helpers.c
@@ -1,17 +1,34 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define HS_CLIENT_PRIVATE
+
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "test/test.h"
 #include "feature/nodelist/torcert.h"
 
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_service.h"
 #include "test/hs_test_helpers.h"
 
+/**
+ * Create an introduction point taken straight out of an HSv3 descriptor.
+ *
+ * Use 'signing_kp' to sign the introduction point certificates.
+ *
+ * If 'intro_auth_kp' is provided use that as the introduction point
+ * authentication keypair, otherwise generate one on the fly.
+ *
+ * If 'intro_enc_kp' is provided use that as the introduction point encryption
+ * keypair, otherwise generate one on the fly.
+ */
 hs_desc_intro_point_t *
 hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
-                            const char *addr, int legacy)
+                            const char *addr, int legacy,
+                            const ed25519_keypair_t *intro_auth_kp,
+                            const curve25519_keypair_t *intro_enc_kp)
 {
   int ret;
   ed25519_keypair_t auth_kp;
@@ -21,30 +38,43 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
   /* For a usable intro point we need at least two link specifiers: One legacy
    * keyid and one ipv4 */
   {
-    hs_desc_link_specifier_t *ls_legacy = tor_malloc_zero(sizeof(*ls_legacy));
-    hs_desc_link_specifier_t *ls_v4 = tor_malloc_zero(sizeof(*ls_v4));
-    ls_legacy->type = LS_LEGACY_ID;
-    memcpy(ls_legacy->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8",
-           DIGEST_LEN);
-    ls_v4->u.ap.port = 9001;
-    int family = tor_addr_parse(&ls_v4->u.ap.addr, addr);
+    tor_addr_t a;
+    tor_addr_make_unspec(&a);
+    link_specifier_t *ls_legacy = link_specifier_new();
+    link_specifier_t *ls_ip = link_specifier_new();
+    link_specifier_set_ls_type(ls_legacy, LS_LEGACY_ID);
+    memset(link_specifier_getarray_un_legacy_id(ls_legacy), 'C',
+           link_specifier_getlen_un_legacy_id(ls_legacy));
+    int family = tor_addr_parse(&a, addr);
     switch (family) {
     case AF_INET:
-          ls_v4->type = LS_IPV4;
+          link_specifier_set_ls_type(ls_ip, LS_IPV4);
+          link_specifier_set_un_ipv4_addr(ls_ip, tor_addr_to_ipv4h(&a));
+          link_specifier_set_un_ipv4_port(ls_ip, 9001);
           break;
         case AF_INET6:
-          ls_v4->type = LS_IPV6;
+          link_specifier_set_ls_type(ls_ip, LS_IPV6);
+          memcpy(link_specifier_getarray_un_ipv6_addr(ls_ip),
+                 tor_addr_to_in6_addr8(&a),
+                 link_specifier_getlen_un_ipv6_addr(ls_ip));
+          link_specifier_set_un_ipv6_port(ls_ip, 9001);
           break;
         default:
-          /* Stop the test, not suppose to have an error. */
-          tt_int_op(family, OP_EQ, AF_INET);
+          /* Stop the test, not supposed to have an error.
+           * Compare with -1 to show the actual family.
+           */
+          tt_int_op(family, OP_EQ, -1);
     }
     smartlist_add(ip->link_specifiers, ls_legacy);
-    smartlist_add(ip->link_specifiers, ls_v4);
+    smartlist_add(ip->link_specifiers, ls_ip);
   }
 
-  ret = ed25519_keypair_generate(&auth_kp, 0);
-  tt_int_op(ret, ==, 0);
+  if (intro_auth_kp) {
+    memcpy(&auth_kp, intro_auth_kp, sizeof(ed25519_keypair_t));
+  } else {
+    ret = ed25519_keypair_generate(&auth_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+  }
   ip->auth_key_cert = tor_cert_create(signing_kp, CERT_TYPE_AUTH_HS_IP_KEY,
                                       &auth_kp.pubkey, now,
                                       HS_DESC_CERT_LIFETIME,
@@ -55,7 +85,7 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
     ip->legacy.key = crypto_pk_new();
     tt_assert(ip->legacy.key);
     ret = crypto_pk_generate_key(ip->legacy.key);
-    tt_int_op(ret, ==, 0);
+    tt_int_op(ret, OP_EQ, 0);
     ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
                                     &signing_kp->pubkey, ip->legacy.key,
                                     now + HS_DESC_CERT_LIFETIME,
@@ -72,8 +102,12 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
     ed25519_keypair_t ed25519_kp;
     tor_cert_t *cross_cert;
 
-    ret = curve25519_keypair_generate(&curve25519_kp, 0);
-    tt_int_op(ret, ==, 0);
+    if (intro_enc_kp) {
+      memcpy(&curve25519_kp, intro_enc_kp, sizeof(curve25519_keypair_t));
+    } else {
+      ret = curve25519_keypair_generate(&curve25519_kp, 0);
+      tt_int_op(ret, OP_EQ, 0);
+    }
     ed25519_keypair_from_curve25519_keypair(&ed25519_kp, &signbit,
                                             &curve25519_kp);
     cross_cert = tor_cert_create(signing_kp, CERT_TYPE_CROSS_HS_IP_KEYS,
@@ -82,6 +116,8 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
                                  CERT_FLAG_INCLUDE_SIGNING_KEY);
     tt_assert(cross_cert);
     ip->enc_key_cert = cross_cert;
+    memcpy(ip->enc_key.public_key, curve25519_kp.pubkey.public_key,
+           CURVE25519_PUBKEY_LEN);
   }
 
   intro_point = ip;
@@ -127,11 +163,11 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip,
   desc->plaintext_data.lifetime_sec = 3 * 60 * 60;
 
   hs_get_subcredential(&signing_kp->pubkey, &blinded_kp.pubkey,
-                    desc->subcredential);
+                    &desc->subcredential);
 
   /* Setup superencrypted data section. */
   ret = curve25519_keypair_generate(&auth_ephemeral_kp, 0);
-  tt_int_op(ret, ==, 0);
+  tt_int_op(ret, OP_EQ, 0);
   memcpy(&desc->superencrypted_data.auth_ephemeral_pubkey,
          &auth_ephemeral_kp.pubkey,
          sizeof(curve25519_public_key_t));
@@ -152,13 +188,17 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip,
   if (!no_ip) {
     /* Add four intro points. */
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0));
+                  hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0,
+                                              NULL, NULL));
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0));
+                  hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0,
+                                              NULL, NULL));
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1));
+                  hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1,
+                                              NULL, NULL));
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "5.6.7.8", 1));
+                  hs_helper_build_intro_point(signing_kp, now, "5.6.7.8", 1,
+                                              NULL, NULL));
   }
 
   descp = desc;
@@ -173,7 +213,7 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip,
  *  an HS. Used to decrypt descriptors in unittests. */
 void
 hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
-                                            uint8_t *subcred_out)
+                                            hs_subcredential_t *subcred_out)
 {
   ed25519_keypair_t blinded_kp;
   uint64_t current_time_period = hs_get_time_period_num(approx_time());
@@ -198,11 +238,39 @@ hs_helper_build_hs_desc_no_ip(const ed25519_keypair_t *signing_kp)
   return hs_helper_build_hs_desc_impl(1, signing_kp);
 }
 
+hs_descriptor_t *
+hs_helper_build_hs_desc_with_client_auth(
+                        const uint8_t *descriptor_cookie,
+                        const curve25519_public_key_t *client_pk,
+                        const ed25519_keypair_t *signing_kp)
+{
+  curve25519_keypair_t auth_ephemeral_kp;
+  hs_descriptor_t *desc = hs_helper_build_hs_desc_impl(0, signing_kp);
+  hs_desc_authorized_client_t *desc_client;
+
+  /* The number of client authorized auth has tobe a multiple of
+   * HS_DESC_AUTH_CLIENT_MULTIPLE so remove one that we'll replace. */
+  desc_client = smartlist_get(desc->superencrypted_data.clients, 0);
+  smartlist_remove(desc->superencrypted_data.clients, desc_client);
+  hs_desc_authorized_client_free(desc_client);
+
+  desc_client = tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
+
+  curve25519_keypair_generate(&auth_ephemeral_kp, 0);
+  memcpy(&desc->superencrypted_data.auth_ephemeral_pubkey,
+         &auth_ephemeral_kp.pubkey, sizeof(curve25519_public_key_t));
+
+  hs_desc_build_authorized_client(&desc->subcredential, client_pk,
+                                  &auth_ephemeral_kp.seckey,
+                                  descriptor_cookie, desc_client);
+  smartlist_add(desc->superencrypted_data.clients, desc_client);
+  return desc;
+}
+
 void
 hs_helper_desc_equal(const hs_descriptor_t *desc1,
                      const hs_descriptor_t *desc2)
 {
-  char *addr1 = NULL, *addr2 = NULL;
   /* Plaintext data section. */
   tt_int_op(desc1->plaintext_data.version, OP_EQ,
             desc2->plaintext_data.version);
@@ -216,7 +284,7 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   tt_mem_op(desc1->plaintext_data.blinded_pubkey.pubkey, OP_EQ,
             desc2->plaintext_data.blinded_pubkey.pubkey,
             ED25519_PUBKEY_LEN);
-  tt_u64_op(desc1->plaintext_data.revision_counter, ==,
+  tt_u64_op(desc1->plaintext_data.revision_counter, OP_EQ,
             desc2->plaintext_data.revision_counter);
 
   /* NOTE: We can't compare the encrypted blob because when encoding the
@@ -233,7 +301,7 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   {
     tt_assert(desc1->superencrypted_data.clients);
     tt_assert(desc2->superencrypted_data.clients);
-    tt_int_op(smartlist_len(desc1->superencrypted_data.clients), ==,
+    tt_int_op(smartlist_len(desc1->superencrypted_data.clients), OP_EQ,
               smartlist_len(desc2->superencrypted_data.clients));
     for (int i=0;
          i < smartlist_len(desc1->superencrypted_data.clients);
@@ -251,15 +319,15 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   }
 
   /* Encrypted data section. */
-  tt_uint_op(desc1->encrypted_data.create2_ntor, ==,
+  tt_uint_op(desc1->encrypted_data.create2_ntor, OP_EQ,
              desc2->encrypted_data.create2_ntor);
 
   /* Authentication type. */
-  tt_int_op(!!desc1->encrypted_data.intro_auth_types, ==,
+  tt_int_op(!!desc1->encrypted_data.intro_auth_types, OP_EQ,
             !!desc2->encrypted_data.intro_auth_types);
   if (desc1->encrypted_data.intro_auth_types &&
       desc2->encrypted_data.intro_auth_types) {
-    tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), ==,
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), OP_EQ,
               smartlist_len(desc2->encrypted_data.intro_auth_types));
     for (int i = 0;
          i < smartlist_len(desc1->encrypted_data.intro_auth_types);
@@ -273,7 +341,7 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   {
     tt_assert(desc1->encrypted_data.intro_points);
     tt_assert(desc2->encrypted_data.intro_points);
-    tt_int_op(smartlist_len(desc1->encrypted_data.intro_points), ==,
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_points), OP_EQ,
               smartlist_len(desc2->encrypted_data.intro_points));
     for (int i=0; i < smartlist_len(desc1->encrypted_data.intro_points); i++) {
       hs_desc_intro_point_t *ip1 = smartlist_get(desc1->encrypted_data
@@ -288,38 +356,76 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
         tt_mem_op(&ip1->enc_key, OP_EQ, &ip2->enc_key, CURVE25519_PUBKEY_LEN);
       }
 
-      tt_int_op(smartlist_len(ip1->link_specifiers), ==,
+      tt_int_op(smartlist_len(ip1->link_specifiers), OP_EQ,
                 smartlist_len(ip2->link_specifiers));
       for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) {
-        hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
-                                 *ls2 = smartlist_get(ip2->link_specifiers, j);
-        tt_int_op(ls1->type, ==, ls2->type);
-        switch (ls1->type) {
+        link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
+                         *ls2 = smartlist_get(ip2->link_specifiers, j);
+        tt_int_op(link_specifier_get_ls_type(ls1), OP_EQ,
+                  link_specifier_get_ls_type(ls2));
+        switch (link_specifier_get_ls_type(ls1)) {
           case LS_IPV4:
+            {
+              uint32_t addr1 = link_specifier_get_un_ipv4_addr(ls1);
+              uint32_t addr2 = link_specifier_get_un_ipv4_addr(ls2);
+              tt_int_op(addr1, OP_EQ, addr2);
+              uint16_t port1 = link_specifier_get_un_ipv4_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv4_port(ls2);
+              tt_int_op(port1, OP_EQ, port2);
+            }
+            break;
           case LS_IPV6:
             {
-              addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr);
-              addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr);
-              tt_str_op(addr1, OP_EQ, addr2);
-              tor_free(addr1);
-              tor_free(addr2);
-              tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port);
+              const uint8_t *addr1 =
+                link_specifier_getconstarray_un_ipv6_addr(ls1);
+              const uint8_t *addr2 =
+                link_specifier_getconstarray_un_ipv6_addr(ls2);
+              tt_int_op(link_specifier_getlen_un_ipv6_addr(ls1), OP_EQ,
+                        link_specifier_getlen_un_ipv6_addr(ls2));
+              tt_mem_op(addr1, OP_EQ, addr2,
+                        link_specifier_getlen_un_ipv6_addr(ls1));
+              uint16_t port1 = link_specifier_get_un_ipv6_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv6_port(ls2);
+              tt_int_op(port1, OP_EQ, port2);
             }
             break;
           case LS_LEGACY_ID:
-            tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id,
-                      sizeof(ls1->u.legacy_id));
+            {
+              const uint8_t *id1 =
+                link_specifier_getconstarray_un_legacy_id(ls1);
+              const uint8_t *id2 =
+                link_specifier_getconstarray_un_legacy_id(ls2);
+              tt_int_op(link_specifier_getlen_un_legacy_id(ls1), OP_EQ,
+                        link_specifier_getlen_un_legacy_id(ls2));
+              tt_mem_op(id1, OP_EQ, id2,
+                        link_specifier_getlen_un_legacy_id(ls1));
+            }
             break;
           default:
             /* Unknown type, caught it and print its value. */
-            tt_int_op(ls1->type, OP_EQ, -1);
+            tt_int_op(link_specifier_get_ls_type(ls1), OP_EQ, -1);
         }
       }
     }
   }
 
  done:
-  tor_free(addr1);
-  tor_free(addr2);
+  ;
 }
 
+void
+hs_helper_add_client_auth(const ed25519_public_key_t *service_pk,
+                          const curve25519_secret_key_t *client_sk)
+{
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  if (client_auths == NULL) {
+    client_auths = digest256map_new();
+    set_hs_client_auths_map(client_auths);
+  }
+
+  hs_client_service_authorization_t *auth =
+    tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+  memcpy(&auth->enc_seckey, client_sk, sizeof(curve25519_secret_key_t));
+  hs_build_address(service_pk, HS_VERSION_THREE, auth->onion_address);
+  digest256map_set(client_auths, service_pk->pubkey, auth);
+}
diff --git a/src/test/hs_test_helpers.h b/src/test/hs_test_helpers.h
index 9662a83ba8..23d11f2a4a 100644
--- a/src/test/hs_test_helpers.h
+++ b/src/test/hs_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_HS_TEST_HELPERS_H
@@ -8,18 +8,26 @@
 #include "feature/hs/hs_descriptor.h"
 
 /* Set of functions to help build and test descriptors. */
-hs_desc_intro_point_t *hs_helper_build_intro_point(
-                          const ed25519_keypair_t *signing_kp, time_t now,
-                          const char *addr, int legacy);
+hs_desc_intro_point_t *
+hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
+                            const char *addr, int legacy,
+                            const ed25519_keypair_t *intro_auth_kp,
+                            const curve25519_keypair_t *intro_enc_kp);
 hs_descriptor_t *hs_helper_build_hs_desc_no_ip(
                                  const ed25519_keypair_t *signing_kp);
 hs_descriptor_t *hs_helper_build_hs_desc_with_ip(
                                  const ed25519_keypair_t *signing_kp);
+hs_descriptor_t *hs_helper_build_hs_desc_with_client_auth(
+                                 const uint8_t *descriptor_cookie,
+                                 const curve25519_public_key_t *client_pk,
+                                 const ed25519_keypair_t *signing_kp);
 void hs_helper_desc_equal(const hs_descriptor_t *desc1,
                           const hs_descriptor_t *desc2);
-void
-hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
-                                            uint8_t *subcred_out);
+struct hs_subcredential_t;
+void hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
+                                      struct hs_subcredential_t *subcred_out);
 
-#endif /* !defined(TOR_HS_TEST_HELPERS_H) */
+void hs_helper_add_client_auth(const ed25519_public_key_t *service_pk,
+                               const curve25519_secret_key_t *client_sk);
 
+#endif /* !defined(TOR_HS_TEST_HELPERS_H) */
diff --git a/src/test/include.am b/src/test/include.am
index 75861fb9ef..7814dbca89 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -24,6 +24,8 @@ TESTSCRIPTS = \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh \
 	src/test/test_switch_id.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh \
         src/test/unittest_part1.sh \
         src/test/unittest_part2.sh \
         src/test/unittest_part3.sh \
@@ -39,9 +41,26 @@ TESTSCRIPTS += \
 endif
 
 if USEPYTHON
-TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += \
+	src/test/test_ntor.sh \
+	src/test/test_hs_ntor.sh \
+	src/test/test_bt.sh \
+	scripts/maint/practracker/test_practracker.sh \
+	scripts/maint/run_check_subsystem_order.sh
+
+if COVERAGE_ENABLED
+# ...
+else
+# Only do this when coverage is not on, since it invokes lots of code
+# in a kind of unpredictable way.
 TESTSCRIPTS += src/test/test_rebind.sh
 endif
+endif
+
+if USE_PERL
+TESTSCRIPTS += \
+        scripts/maint/checkSpaceTest.sh
+endif
 
 TESTS += src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue \
@@ -51,15 +70,25 @@ TESTS += src/test/test-slow src/test/test-memwipe \
 	$(TESTSCRIPTS)
 
 # These flavors are run using automake's test-driver and test-network.sh
-TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \
-	single-onion-v23
+
+# run a quick test or two
+# this test only uses IPv4
+TEST_CHUTNEY_FLAVOR_QUICK = bridges+hs-v23
 # only run if we can ping6 ::1 (localhost)
-# IPv6-only v3 single onion services don't work yet, so we don't test the
-# single-onion-v23-ipv6-md flavor
+TEST_CHUTNEY_FLAVOR_QUICK_IPV6 = single-onion-v23-ipv6-md
+
+# run a basic set of tests, which only use IPv4
+TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v23-min single-onion-v23
+
+# only run if we can ping ::1 (localhost)
 TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \
-	single-onion-ipv6-md
+	single-onion-v23-ipv6-md
+
 # only run if we can find a stable (or simply another) version of tor
-TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2
+TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v23
+
+# only run if IPv6 and mixed networks are run
+TEST_CHUTNEY_FLAVORS_IPV6_MIXED = mixed+hs-v23-ipv6
 
 ### This is a lovely feature, but it requires automake >= 1.12, and Tor
 ###  doesn't require that yet.
@@ -73,10 +102,11 @@ noinst_PROGRAMS+= \
 	src/test/test \
 	src/test/test-slow \
 	src/test/test-memwipe \
-	src/test/test-child \
+	src/test/test-process \
 	src/test/test_workqueue \
 	src/test/test-switch-id \
-	src/test/test-timers
+	src/test/test-timers \
+	src/test/test-rng
 endif
 
 src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
@@ -92,31 +122,42 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
 src_test_test_SOURCES =
 
 if UNITTESTS_ENABLED
+
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_test_test_SOURCES += \
+	src/test/fakecircs.c \
 	src/test/log_test_helpers.c \
 	src/test/hs_test_helpers.c \
+	src/test/opts_test_helpers.c \
 	src/test/rend_test_helpers.c \
+	src/test/resolve_test_helpers.c \
+	src/test/rng_test_helpers.c \
 	src/test/test.c \
 	src/test/test_accounting.c \
 	src/test/test_addr.c \
 	src/test/test_address.c \
 	src/test/test_address_set.c \
 	src/test/test_bridges.c \
+	src/test/test_btrack.c \
 	src/test/test_buffers.c \
 	src/test/test_bwmgt.c \
 	src/test/test_cell_formats.c \
 	src/test/test_cell_queue.c \
 	src/test/test_channel.c \
 	src/test/test_channelpadding.c \
+	src/test/test_circuitpadding.c \
 	src/test/test_channeltls.c \
 	src/test/test_checkdir.c \
 	src/test/test_circuitlist.c \
 	src/test/test_circuitmux.c \
+	src/test/test_circuitmux_ewma.c \
 	src/test/test_circuitbuild.c \
 	src/test/test_circuituse.c \
 	src/test/test_circuitstats.c \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
+	src/test/test_confmgr.c \
+	src/test/test_confparse.c \
 	src/test/test_connection.c \
 	src/test/test_conscache.c \
 	src/test/test_consdiff.c \
@@ -126,10 +167,12 @@ src_test_test_SOURCES += \
 	src/test/test_controller_events.c \
 	src/test/test_crypto.c \
 	src/test/test_crypto_ope.c \
+	src/test/test_crypto_rng.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
 	src/test/test_dir_common.c \
 	src/test/test_dir_handle_get.c \
+	src/test/test_dispatch.c \
 	src/test/test_dos.c \
 	src/test/test_entryconn.c \
 	src/test/test_entrynodes.c \
@@ -145,27 +188,38 @@ src_test_test_SOURCES += \
 	src/test/test_hs_client.c  \
 	src/test/test_hs_intropoint.c \
 	src/test/test_hs_control.c \
+	src/test/test_hs_ob.c \
 	src/test/test_handles.c \
 	src/test/test_hs_cache.c \
 	src/test/test_hs_descriptor.c \
+	src/test/test_hs_dos.c \
 	src/test/test_introduce.c \
 	src/test/test_keypin.c \
 	src/test/test_link_handshake.c \
 	src/test/test_logging.c \
 	src/test/test_mainloop.c \
 	src/test/test_microdesc.c \
+	src/test/test_namemap.c \
+	src/test/test_netinfo.c \
 	src/test/test_nodelist.c \
 	src/test/test_oom.c \
 	src/test/test_oos.c \
 	src/test/test_options.c \
+	src/test/test_options_act.c \
 	src/test/test_pem.c \
 	src/test/test_periodic_event.c \
 	src/test/test_policy.c \
+	src/test/test_process.c \
+	src/test/test_process_descs.c \
+	src/test/test_prob_distr.c \
 	src/test/test_procmon.c \
+	src/test/test_proto_haproxy.c \
 	src/test/test_proto_http.c \
 	src/test/test_proto_misc.c \
 	src/test/test_protover.c \
 	src/test/test_pt.c \
+	src/test/test_pubsub_build.c \
+	src/test/test_pubsub_msg.c \
 	src/test/test_relay.c \
 	src/test/test_relaycell.c \
 	src/test/test_relaycrypt.c \
@@ -176,19 +230,24 @@ src_test_test_SOURCES += \
 	src/test/test_routerlist.c \
 	src/test/test_routerset.c \
 	src/test/test_scheduler.c \
+	src/test/test_sendme.c \
 	src/test/test_shared_random.c \
 	src/test/test_socks.c \
+	src/test/test_stats.c \
 	src/test/test_status.c \
 	src/test/test_storagedir.c \
 	src/test/test_threads.c \
+	src/test/test_token_bucket.c \
 	src/test/test_tortls.c \
 	src/test/test_util.c \
 	src/test/test_util_format.c \
 	src/test/test_util_process.c \
+	src/test/test_voting_flags.c \
 	src/test/test_voting_schedule.c \
 	src/test/test_x509.c \
 	src/test/test_helpers.c \
 	src/test/test_dns.c \
+	src/test/test_parsecommon.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -206,9 +265,13 @@ endif
 src_test_test_slow_SOURCES =
 if UNITTESTS_ENABLED
 src_test_test_slow_SOURCES += \
+	src/test/rng_test_helpers.c \
 	src/test/test_slow.c \
 	src/test/test_crypto_slow.c \
-	src/test/test_util_slow.c \
+	src/test/test_process_slow.c \
+	src/test/test_prob_distr.c \
+	src/test/ptr_helpers.c \
+	src/test/test_ptr_slow.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -259,6 +322,12 @@ src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS)
 src_test_test_slow_LDADD = $(src_test_test_LDADD)
 src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS)
 
+src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS)
+src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS)
+src_test_test_rng_SOURCES = src/test/test_rng.c
+src_test_test_rng_LDFLAGS =  $(src_test_test_LDFLAGS)
+src_test_test_rng_LDADD = $(src_test_test_LDADD)
+
 src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS)
 # Don't use bugtrap cflags here: memwipe tests require memory violations.
 src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS)
@@ -300,12 +369,18 @@ src_test_test_timers_LDADD = \
 	@TOR_LZMA_LIBS@
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS+= \
 	src/test/fakechans.h \
+	src/test/fakecircs.h \
 	src/test/hs_test_helpers.h \
 	src/test/log_test_helpers.h \
+	src/test/opts_test_helpers.h \
 	src/test/rend_test_helpers.h \
+	src/test/resolve_test_helpers.h \
+	src/test/rng_test_helpers.h \
 	src/test/test.h \
+	src/test/ptr_helpers.h \
 	src/test/test_helpers.h \
 	src/test/test_dir_common.h \
 	src/test/test_connection.h \
@@ -364,6 +439,7 @@ EXTRA_DIST += \
 	src/test/test_rebind.sh \
 	src/test/test_rebind.py \
 	src/test/zero_length_keys.sh \
+	scripts/maint/run_check_subsystem_order.sh \
 	src/test/rust_supp.txt \
 	src/test/test_keygen.sh \
 	src/test/test_key_expiration.sh \
@@ -378,6 +454,8 @@ EXTRA_DIST += \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh \
         src/test/unittest_part1.sh \
         src/test/unittest_part2.sh \
         src/test/unittest_part3.sh \
diff --git a/src/test/log_test_helpers.c b/src/test/log_test_helpers.c
index 03c52dd6bd..5e60d6b282 100644
--- a/src/test/log_test_helpers.c
+++ b/src/test/log_test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define LOG_PRIVATE
 #include "lib/log/log.h"
diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h
index 5d1c3c1914..c2d71c6bcd 100644
--- a/src/test/log_test_helpers.h
+++ b/src/test/log_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -78,7 +78,7 @@ void mock_dump_saved_logs(void);
                          mock_saved_log_n_entries() == 1,               \
                          ("expected log to contain exactly 1 message \"%s\"", \
                           str));                                        \
-  } while (0);
+  } while (0)
 
 #define expect_single_log_msg_containing(str) \
   do {                                                                  \
@@ -86,30 +86,30 @@ void mock_dump_saved_logs(void);
                          mock_saved_log_n_entries() == 1 ,              \
                     ("expected log to contain 1 message, containing \"%s\"",\
                      str));                                             \
-  } while (0);
+  } while (0)
 
 #define expect_no_log_msg(str) \
   assert_log_predicate(!mock_saved_log_has_message(str), \
-                       ("expected log to not contain \"%s\"",str));
+                       ("expected log to not contain \"%s\"",str))
 
 #define expect_no_log_msg_containing(str) \
   assert_log_predicate(!mock_saved_log_has_message_containing(str), \
-                       ("expected log to not contain \"%s\"", str));
+                       ("expected log to not contain \"%s\"", str))
 
 #define expect_log_severity(severity) \
   assert_log_predicate(mock_saved_log_has_severity(severity), \
-                       ("expected log to contain severity " # severity));
+                       ("expected log to contain severity " # severity))
 
 #define expect_no_log_severity(severity) \
   assert_log_predicate(!mock_saved_log_has_severity(severity), \
-                       ("expected log to not contain severity " # severity));
+                       ("expected log to not contain severity " # severity))
 
 #define expect_log_entry() \
   assert_log_predicate(mock_saved_log_has_entry(), \
-                       ("expected log to contain entries"));
+                       ("expected log to contain entries"))
 
 #define expect_no_log_entry() \
   assert_log_predicate(!mock_saved_log_has_entry(), \
-                       ("expected log to not contain entries"));
+                       ("expected log to not contain entries"))
 
 #endif /* !defined(TOR_LOG_TEST_HELPERS_H) */
diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py
index 204f05e2ad..e3307430e1 100755
--- a/src/test/ntor_ref.py
+++ b/src/test/ntor_ref.py
@@ -27,6 +27,11 @@ commands:
 
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import binascii
 try:
     import curve25519
@@ -99,7 +104,7 @@ else:
    def int2byte(i):
       return bytes([i])
 
-def  kdf_rfc5869(key, salt, info, n):
+def kdf_rfc5869(key, salt, info, n):
 
     prk = HMAC(key=salt, msg=key)
 
diff --git a/src/test/ope_ref.py b/src/test/ope_ref.py
index f9bd97c546..61a86b57bb 100644
--- a/src/test/ope_ref.py
+++ b/src/test/ope_ref.py
@@ -1,9 +1,14 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # Copyright 2018-2019, The Tor Project, Inc. See LICENSE for licensing info.
 
 # Reference implementation for our rudimentary OPE code, used to
 # generate test vectors. See crypto_ope.c for more details.
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.primitives.ciphers.algorithms import AES
 from cryptography.hazmat.backends import default_backend
diff --git a/src/test/opts_test_helpers.c b/src/test/opts_test_helpers.c
new file mode 100644
index 0000000000..619ca40733
--- /dev/null
+++ b/src/test/opts_test_helpers.c
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file opts_testing_helpers.c
+ * @brief Helper functions to access module-specific config options.
+ **/
+
+#include "orconfig.h"
+#include "test/opts_test_helpers.h"
+
+#define CONFIG_PRIVATE
+#include "core/or/or.h"
+#include "lib/confmgt/confmgt.h"
+#include "app/main/subsysmgr.h"
+#include "app/config/config.h"
+
+#include "lib/crypt_ops/crypto_sys.h"
+#include "feature/dirauth/dirauth_sys.h"
+
+struct dirauth_options_t *
+get_dirauth_options(struct or_options_t *opt)
+{
+  int idx = subsystems_get_options_idx(&sys_dirauth);
+  tor_assert(idx >= 0);
+  return config_mgr_get_obj_mutable(get_options_mgr(), opt, idx);
+}
+
+struct crypto_options_t *
+get_crypto_options(struct or_options_t *opt)
+{
+  int idx = subsystems_get_options_idx(&sys_crypto);
+  tor_assert(idx >= 0);
+  return config_mgr_get_obj_mutable(get_options_mgr(), opt, idx);
+}
diff --git a/src/test/opts_test_helpers.h b/src/test/opts_test_helpers.h
new file mode 100644
index 0000000000..f925194e63
--- /dev/null
+++ b/src/test/opts_test_helpers.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file opts_testing_helpers.h
+ * @brief Header for test/opts_test_helpers.c
+ **/
+
+#ifndef TOR_TEST_OPTS_TESTING_HELPERS_H
+#define TOR_TEST_OPTS_TESTING_HELPERS_H
+
+struct crypto_options_t;
+struct dirauth_options_t;
+struct or_options_t;
+
+struct crypto_options_t *get_crypto_options(struct or_options_t *opt);
+struct dirauth_options_t *get_dirauth_options(struct or_options_t *opt);
+
+#endif /* !defined(TOR_TEST_OPTS_TESTING_HELPERS_H) */
diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c
new file mode 100644
index 0000000000..de4179c4e0
--- /dev/null
+++ b/src/test/prob_distr_mpfr_ref.c
@@ -0,0 +1,64 @@
+/* Copyright 2012-2020, The Tor Project, Inc
+ * See LICENSE for licensing information */
+
+/** prob_distr_mpfr_ref.c
+ *
+ * Example reference file for GNU MPFR vectors tested in test_prob_distr.c .
+ * Code by Riastradh.
+ */
+
+#include <complex.h>
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+
+/* Must come after <stdio.h> so we get mpfr_printf.  */
+#include <mpfr.h>
+
+/*  gcc -o mpfr prob_distr_mpfr_ref.c -lmpfr -lm */
+
+/* Computes logit(p) for p = .49999 */
+int
+main(void)
+{
+  mpfr_t p, q, r;
+  mpfr_init(p);
+  mpfr_set_prec(p, 200);
+  mpfr_init(q);
+  mpfr_set_prec(q, 200);
+  mpfr_init(r);
+  mpfr_set_prec(r, 200);
+  mpfr_set_d(p, .49999, MPFR_RNDN);
+  mpfr_set_d(q, 1, MPFR_RNDN);
+  /* r := q - p = 1 - p */
+  mpfr_sub(r, q, p, MPFR_RNDN);
+  /* q := p/r = p/(1 - p) */
+  mpfr_div(q, p, r, MPFR_RNDN);
+  /* r := log(q) = log(p/(1 - p)) */
+  mpfr_log(r, q, MPFR_RNDN);
+  mpfr_printf("mpfr 200-bit\t%.128Rg\n", r);
+
+  /*
+   * Print a double approximation to logit three different ways.  All
+   * three agree bit for bit on the libms I tried, with the nextafter
+   * adjustment (which is well within the 10 eps relative error bound
+   * advertised).  Apparently I must have used the Goldberg expression
+   * for what I wrote down in the test case.
+   */
+  printf("mpfr 53-bit\t%.17g\n", nextafter(mpfr_get_d(r, MPFR_RNDN), 0), 0);
+  volatile double p0 = .49999;
+  printf("log1p\t\t%.17g\n", nextafter(-log1p((1 - 2*p0)/p0), 0));
+  volatile double x = (1 - 2*p0)/p0;
+  volatile double xp1 = x + 1;
+  printf("Goldberg\t%.17g\n", -x*log(xp1)/(xp1 - 1));
+
+  /*
+   * Print a bad approximation, using the naive expression, to see a
+   * lot of wrong digits, far beyond the 10 eps relative error attained
+   * by -log1p((1 - 2*p)/p).
+   */
+  printf("naive\t\t%.17g\n", log(p0/(1 - p0)));
+
+  fflush(stdout);
+  return ferror(stdout);
+}
diff --git a/src/test/ptr_helpers.c b/src/test/ptr_helpers.c
new file mode 100644
index 0000000000..0e0995df7c
--- /dev/null
+++ b/src/test/ptr_helpers.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/ptr_helpers.h"
+
+/**
+ * Cast <b> (inptr_t value) to a void pointer.
+ */
+void *
+cast_intptr_to_voidstar(intptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to inptr_t value.
+ */
+intptr_t
+cast_voidstar_to_intptr(void *x)
+{
+  intptr_t r = (intptr_t)x;
+
+  return r;
+}
+
+/**
+ * Cast x (uinptr_t value) to void pointer.
+ */
+void *
+cast_uintptr_to_voidstar(uintptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to uinptr_t value.
+ */
+uintptr_t
+cast_voidstar_to_uintptr(void *x)
+{
+  uintptr_t r = (uintptr_t)x;
+
+  return r;
+}
diff --git a/src/test/ptr_helpers.h b/src/test/ptr_helpers.h
new file mode 100644
index 0000000000..0999fdf5d2
--- /dev/null
+++ b/src/test/ptr_helpers.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PTR_HELPERS_H
+#define TOR_PTR_HELPERS_H
+
+#include <stdint.h>
+
+void *
+cast_intptr_to_voidstar(intptr_t x);
+
+intptr_t
+cast_voidstar_to_intptr(void *x);
+
+void *
+cast_uintptr_to_voidstar(uintptr_t x);
+
+uintptr_t
+cast_voidstar_to_uintptr(void *x);
+
+#endif /* !defined(TOR_PTR_HELPERS_H) */
diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c
index f12d193cc5..61bacb4d2e 100644
--- a/src/test/rend_test_helpers.c
+++ b/src/test/rend_test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/rend_test_helpers.h b/src/test/rend_test_helpers.h
index c10da52cd7..b1078ce866 100644
--- a/src/test/rend_test_helpers.h
+++ b/src/test/rend_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/resolve_test_helpers.c b/src/test/resolve_test_helpers.c
new file mode 100644
index 0000000000..ed5853c359
--- /dev/null
+++ b/src/test/resolve_test_helpers.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.c
+ * @brief Helper functions for mocking libc's blocking hostname lookup
+ *   facilities.
+ **/
+
+#define RESOLVE_PRIVATE
+#include "orconfig.h"
+#include "test/resolve_test_helpers.h"
+#include "lib/net/address.h"
+#include "lib/net/resolve.h"
+#include "test/test.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * Mock replacement for our getaddrinfo/gethostbyname wrapper.
+ **/
+static int
+replacement_host_lookup(const char *name, uint16_t family, tor_addr_t *addr)
+{
+  static const struct lookup_table_ent {
+    const char *name;
+    const char *ipv4;
+    const char *ipv6;
+  } entries[] = {
+    { "localhost", "127.0.0.1", "::1" },
+    { "torproject.org", "198.51.100.6", "2001:DB8::700" },
+    { NULL, NULL, NULL },
+  };
+
+  int r = -1;
+
+  for (unsigned i = 0; entries[i].name != NULL; ++i) {
+    if (!strcasecmp(name, entries[i].name)) {
+      if (family == AF_INET6) {
+        int s = tor_addr_parse(addr, entries[i].ipv6);
+        tt_int_op(s, OP_EQ, AF_INET6);
+      } else {
+        int s = tor_addr_parse(addr, entries[i].ipv4);
+        tt_int_op(s, OP_EQ, AF_INET);
+      }
+      r = 0;
+      break;
+    }
+  }
+
+  log_debug(LD_GENERAL, "resolve(%s,%d) => %s",
+            name, family, r == 0 ? fmt_addr(addr) : "-1");
+
+  return r;
+ done:
+  return -1;
+}
+
+/**
+ * Set up a mock replacement for our wrapper on libc's resolver code.
+ *
+ * According to our replacement, only "localhost" and "torproject.org"
+ * are real addresses; everything else doesn't exist.
+ *
+ * Use this function to avoid using the DNS resolver during unit tests;
+ * call unmock_hostname_resolver() when you're done.
+ **/
+void
+mock_hostname_resolver(void)
+{
+  MOCK(tor_addr_lookup_host_impl, replacement_host_lookup);
+}
+
+/**
+ * Unmock our wrappers for libc's blocking hostname resolver code.
+ **/
+void
+unmock_hostname_resolver(void)
+{
+  UNMOCK(tor_addr_lookup_host_impl);
+}
diff --git a/src/test/resolve_test_helpers.h b/src/test/resolve_test_helpers.h
new file mode 100644
index 0000000000..ca642d6c63
--- /dev/null
+++ b/src/test/resolve_test_helpers.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.h
+ * @brief Header for test/resolve_test_helpers.c
+ **/
+
+#ifndef TOR_TEST_RESOLVE_TEST_HELPERS_H
+#define TOR_TEST_RESOLVE_TEST_HELPERS_H
+
+void mock_hostname_resolver(void);
+void unmock_hostname_resolver(void);
+
+#endif /* !defined(TOR_TEST_RESOLVE_TEST_HELPERS_H) */
diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c
new file mode 100644
index 0000000000..b7d7cb0dfa
--- /dev/null
+++ b/src/test/rng_test_helpers.c
@@ -0,0 +1,259 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rng_test_helpers.c
+ * \brief Helpers for overriding PRNGs during unit tests.
+ *
+ * We define two PRNG overrides: a "reproducible PRNG" where the seed is
+ * chosen randomly but the stream can be replayed later on in case a bug is
+ * found, and a "deterministic PRNG" where the seed is fixed in the unit
+ * tests.
+ *
+ * Obviously, this code is testing-only.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "ext/tinytest.h"
+
+#include "test/rng_test_helpers.h"
+
+#ifndef TOR_UNIT_TESTS
+#error "No. Never link this code into Tor proper."
+#endif
+
+/**
+ * True iff the RNG is currently replaced.  Prevents double-replacement.
+ **/
+static bool rng_is_replaced = false;
+
+/**
+ * Mutex to protect deterministic prng.
+ *
+ * Note that if you actually _use_ the prng from two threads at the same time,
+ * the results will probably be nondeterministic anyway.
+ */
+static tor_mutex_t *rng_mutex = NULL;
+
+/**
+ * Cached old value for the thread prng.
+ **/
+static crypto_fast_rng_t *stored_fast_rng = NULL;
+
+/** replacement for crypto_strongest_rand that delegates to crypto_rand. */
+static void
+mock_crypto_strongest_rand(uint8_t *out, size_t len)
+{
+  crypto_rand((char *)out, len);
+}
+
+/* This is the seed of the deterministic randomness. */
+static uint8_t rng_seed[16];
+static crypto_xof_t *rng_xof = NULL;
+
+/**
+ * Print the seed for our PRNG to stdout.  We use this when we're failed
+ * test that had a reproducible RNG set.
+ **/
+void
+testing_dump_reproducible_rng_seed(void)
+{
+  printf("\n"
+         "Seed: %s\n",
+         hex_str((const char*)rng_seed, sizeof(rng_seed)));
+}
+
+/** Produce deterministic randomness for the stochastic tests using the global
+ * rng_xof output.
+ *
+ * This function produces deterministic data over multiple calls iff it's
+ * called in the same call order with the same 'n' parameter.
+ * If not, outputs will deviate. */
+static void
+crypto_rand_deterministic(char *out, size_t n)
+{
+  tor_assert(rng_xof);
+  tor_mutex_acquire(rng_mutex);
+  crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n);
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Implementation helper: override our crypto_rand() PRNG with a given seed of
+ * length <b>seed_len</b>.  Overlong seeds are truncated; short ones are
+ * padded.
+ **/
+static void
+enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len)
+{
+  tor_assert(!rng_is_replaced);
+  tor_assert(crypto_rand == crypto_rand__real);
+
+  memset(rng_seed, 0, sizeof(rng_seed));
+  memcpy(rng_seed, seed, MIN(seed_len, sizeof(rng_seed)));
+
+  rng_mutex = tor_mutex_new();
+
+  crypto_xof_free(rng_xof);
+  rng_xof = crypto_xof_new();
+  crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed));
+  MOCK(crypto_rand, crypto_rand_deterministic);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+
+  uint8_t fast_rng_seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(fast_rng_seed, 0xff, sizeof(fast_rng_seed));
+  memcpy(fast_rng_seed, rng_seed, MIN(sizeof(rng_seed),
+                                      sizeof(fast_rng_seed)));
+  crypto_fast_rng_t *fast_rng = crypto_fast_rng_new_from_seed(fast_rng_seed);
+  crypto_fast_rng_disable_reseed(fast_rng);
+  stored_fast_rng = crypto_replace_thread_fast_rng(fast_rng);
+
+  rng_is_replaced = true;
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a randomly chosen seed.  In the event of an
+ * error, you can log the seed later on with
+ * testing_dump_reproducible_rng_seed.
+ **/
+void
+testing_enable_reproducible_rng(void)
+{
+  const char *provided_seed = getenv("TOR_TEST_RNG_SEED");
+  if (provided_seed) {
+    size_t hexlen = strlen(provided_seed);
+    size_t seedlen = hexlen / 2;
+    uint8_t *seed = tor_malloc(hexlen / 2);
+    if (base16_decode((char*)seed, seedlen, provided_seed, hexlen) < 0) {
+      puts("Cannot decode value in TOR_TEST_RNG_SEED");
+      exit(1);
+    }
+    enable_deterministic_rng_impl(seed, seedlen);
+    tor_free(seed);
+  } else {
+    uint8_t seed[16];
+    crypto_rand((char*)seed, sizeof(seed));
+    enable_deterministic_rng_impl(seed, sizeof(seed));
+  }
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a fixed seed.  This variant is mainly useful
+ * for cases when we don't want coverage to change between runs.
+ *
+ * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of
+ * this "rng".  If you need a specific output, use
+ * testing_enable_prefilled_rng() instead.
+ **/
+void
+testing_enable_deterministic_rng(void)
+{
+  static const uint8_t quotation[] =
+    "What will it be? A tree? A weed? "
+    "Each one is started from a seed."; // -- Mary Ann Hoberman
+  enable_deterministic_rng_impl(quotation, sizeof(quotation));
+}
+
+static uint8_t *prefilled_rng_buffer = NULL;
+static size_t prefilled_rng_buflen;
+static size_t prefilled_rng_idx;
+
+/**
+ * crypto_rand() replacement that returns canned data.
+ **/
+static void
+crypto_rand_prefilled(char *out, size_t n)
+{
+  tor_mutex_acquire(rng_mutex);
+  while (n) {
+    size_t n_to_copy = MIN(prefilled_rng_buflen - prefilled_rng_idx, n);
+    memcpy(out, prefilled_rng_buffer + prefilled_rng_idx, n_to_copy);
+    out += n_to_copy;
+    n -= n_to_copy;
+    prefilled_rng_idx += n_to_copy;
+
+    if (prefilled_rng_idx == prefilled_rng_buflen) {
+      prefilled_rng_idx = 0;
+    }
+  }
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant
+ * that yields output from a buffer.  If it reaches the end of the buffer, it
+ * starts over.
+ *
+ * Note: the get_thread_fast_rng() prng is not replaced by this; we'll need
+ * more code to support that.
+ **/
+void
+testing_enable_prefilled_rng(const void *buffer, size_t buflen)
+{
+  tor_assert(buflen > 0);
+  tor_assert(!rng_mutex);
+  rng_mutex = tor_mutex_new();
+
+  tor_mutex_acquire(rng_mutex);
+
+  prefilled_rng_buffer = tor_memdup(buffer, buflen);
+  prefilled_rng_buflen = buflen;
+  prefilled_rng_idx = 0;
+
+  tor_mutex_release(rng_mutex);
+
+  MOCK(crypto_rand, crypto_rand_prefilled);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+}
+
+/**
+ * Reset the position in the prefilled RNG buffer to the start.
+ */
+void
+testing_prefilled_rng_reset(void)
+{
+  tor_mutex_acquire(rng_mutex);
+  prefilled_rng_idx = 0;
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Undo the overrides for our PRNG.  To be used at the end of testing.
+ *
+ * Note that this function should be safe to call even if the rng has not
+ * yet been replaced.
+ **/
+void
+testing_disable_rng_override(void)
+{
+  crypto_xof_free(rng_xof);
+  tor_free(prefilled_rng_buffer);
+  UNMOCK(crypto_rand);
+  UNMOCK(crypto_strongest_rand_);
+  tor_mutex_free(rng_mutex);
+
+  crypto_fast_rng_t *rng = crypto_replace_thread_fast_rng(stored_fast_rng);
+  crypto_fast_rng_free(rng);
+
+  rng_is_replaced = false;
+}
+
+/**
+ * As testing_disable_rng_override(), but dump the seed if the current
+ * test has failed.
+ */
+void
+testing_disable_reproducible_rng(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    testing_dump_reproducible_rng_seed();
+  }
+  testing_disable_rng_override();
+}
diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h
new file mode 100644
index 0000000000..6fcdaa2653
--- /dev/null
+++ b/src/test/rng_test_helpers.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_RNG_TEST_HELPERS_H
+#define TOR_RNG_TEST_HELPERS_H
+
+#include "core/or/or.h"
+
+void testing_enable_deterministic_rng(void);
+void testing_enable_reproducible_rng(void);
+void testing_enable_prefilled_rng(const void *buffer, size_t buflen);
+
+void testing_prefilled_rng_reset(void);
+
+void testing_disable_rng_override(void);
+
+void testing_disable_reproducible_rng(void);
+#define testing_disable_deterministic_rng() \
+  testing_disable_rng_override()
+#define testing_disable_prefilled_rng() \
+  testing_disable_rng_override()
+
+void testing_dump_reproducible_rng_seed(void);
+
+#endif /* !defined(TOR_RNG_TEST_HELPERS_H) */
diff --git a/src/test/slow_ed25519.py b/src/test/slow_ed25519.py
index f44708b200..be4eeab857 100644
--- a/src/test/slow_ed25519.py
+++ b/src/test/slow_ed25519.py
@@ -1,5 +1,5 @@
 # This is the ed25519 implementation from
-#     http://ed25519.cr.yp.to/python/ed25519.py .
+#     https://ed25519.cr.yp.to/python/ed25519.py .
 # It is in the public domain.
 #
 # It isn't constant-time.  Don't use it except for testing.  Also, see
@@ -8,6 +8,11 @@
 #
 # Don't edit this file.  Mess with ed25519_ref.py
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import hashlib
 
 b = 256
diff --git a/src/test/slownacl_curve25519.py b/src/test/slownacl_curve25519.py
index 4dabab61b6..0cafe0e71f 100644
--- a/src/test/slownacl_curve25519.py
+++ b/src/test/slownacl_curve25519.py
@@ -6,10 +6,15 @@
 # Nick got the slownacl source from:
 #        https://github.com/mdempsky/dnscurve/tree/master/slownacl
 
-__all__ = ['smult_curve25519_base', 'smult_curve25519']
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 import sys
 
+__all__ = ['smult_curve25519_base', 'smult_curve25519']
+
 P = 2 ** 255 - 19
 A = 486662
 
diff --git a/src/test/sr_commit_calc_ref.py b/src/test/sr_commit_calc_ref.py
index 45e629cfb0..c4cb72d87f 100644
--- a/src/test/sr_commit_calc_ref.py
+++ b/src/test/sr_commit_calc_ref.py
@@ -12,6 +12,11 @@
 #      COMMIT = base64-encode( TIMESTAMP || H(REVEAL) )
 #
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/sr_srv_calc_ref.py b/src/test/sr_srv_calc_ref.py
index 492ca62b15..a3752b15cc 100644
--- a/src/test/sr_srv_calc_ref.py
+++ b/src/test/sr_srv_calc_ref.py
@@ -10,6 +10,11 @@
 #                     HASHED_REVEALS | previous_SRV)
 #
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/test-child.c b/src/test/test-child.c
deleted file mode 100644
index 11a1695cad..0000000000
--- a/src/test/test-child.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#include <stdio.h>
-#ifdef _WIN32
-#define WINDOWS_LEAN_AND_MEAN
-#include <windows.h>
-#else
-#include <unistd.h>
-#endif /* defined(_WIN32) */
-#include <string.h>
-
-#ifdef _WIN32
-#define SLEEP(sec) Sleep((sec)*1000)
-#else
-#define SLEEP(sec) sleep(sec)
-#endif
-
-/** Trivial test program which prints out its command line arguments so we can
- * check if tor_spawn_background() works */
-int
-main(int argc, char **argv)
-{
-  int i;
-  int delay = 1;
-  int fast = 0;
-
-  if (argc > 1)  {
-    if (!strcmp(argv[1], "--hang")) {
-      delay = 60;
-    } else if (!strcmp(argv[1], "--fast")) {
-      fast = 1;
-      delay = 0;
-    }
-  }
-
-  fprintf(stdout, "OUT\n");
-  fprintf(stderr, "ERR\n");
-  for (i = 1; i < argc; i++)
-    fprintf(stdout, "%s\n", argv[i]);
-  if (!fast)
-    fprintf(stdout, "SLEEPING\n");
-  /* We need to flush stdout so that test_util_spawn_background_partial_read()
-     succeed. Otherwise ReadFile() will get the entire output in one */
-  // XXX: Can we make stdio flush on newline?
-  fflush(stdout);
-  if (!fast)
-    SLEEP(1);
-  fprintf(stdout, "DONE\n");
-  fflush(stdout);
-  if (fast)
-    return 0;
-
-  while (--delay) {
-    SLEEP(1);
-  }
-
-  return 0;
-}
-
diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c
index 43754ed1c2..4faf7bc5a1 100644
--- a/src/test/test-memwipe.c
+++ b/src/test/test-memwipe.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -49,7 +49,7 @@ const char *s = NULL;
  * us do bad things, such as access freed buffers, without crashing. */
 extern const char *malloc_options;
 const char *malloc_options = "sufjj";
-#endif
+#endif /* defined(OpenBSD) */
 
 static unsigned
 fill_a_buffer_memset(void)
diff --git a/src/test/test-network.sh b/src/test/test-network.sh
index b7a9f1b3c0..5ef995f1a4 100755
--- a/src/test/test-network.sh
+++ b/src/test/test-network.sh
@@ -5,7 +5,7 @@
 # If we already know CHUTNEY_PATH, don't bother with argument parsing
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     # we can't produce any output, because we might be --quiet
     # this preserves arguments with spaces correctly
     exec "$TEST_NETWORK" "$@"
@@ -16,34 +16,16 @@ fi
 # Do we output anything at all?
 ECHO="${ECHO:-echo}"
 # Output is prefixed with the name of the script
-myname=$(basename $0)
-
-# Save the arguments before we destroy them
-# This might not preserve arguments with spaces in them
-ORIGINAL_ARGS="$@"
+myname=$(basename "$0")
 
 # We need to find CHUTNEY_PATH, so that we can call the version of this script
 # in chutney/tools with the same arguments. We also need to respect --quiet.
-until [ -z "$1" ]
-do
-  case "$1" in
-    --chutney-path)
-      CHUTNEY_PATH="$2"
-      shift
-    ;;
-    --tor-path)
-      TOR_DIR="$2"
-      shift
-    ;;
-    --quiet)
-      ECHO=true
-    ;;
-    *)
-      # maybe chutney's test-network.sh can handle it
-    ;;
-  esac
-  shift
-done
+CHUTNEY_PATH=$(echo "$@" | awk -F '--chutney-path ' '{sub(" .*","",$2); print $2}')
+TOR_DIR=$(echo "$@" | awk -F '--tor-dir ' '{sub(" .*","",$2); print $2}')
+
+if echo "$@" | grep -e "--quiet" > /dev/null; then
+  ECHO=true
+fi
 
 # optional: $TOR_DIR is the tor build directory
 # it's used to find the location of tor binaries
@@ -52,12 +34,12 @@ done
 #  - if $PWD looks like a tor build directory, set it to $PWD, or
 #  - unset $TOR_DIR, and let chutney fall back to finding tor binaries in $PATH
 if [ ! -d "$TOR_DIR" ]; then
-    if [ -d "$BUILDDIR/src/core/or" -a -d "$BUILDDIR/src/tools" ]; then
+    if [ -d "$BUILDDIR/src/core/or" ] && [ -d "$BUILDDIR/src/tools" ]; then
         # Choose the build directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$BUILDDIR"
         TOR_DIR="$BUILDDIR"
-    elif [ -d "$PWD/src/core/or" -a -d "$PWD/src/tools" ]; then
+    elif [ -d "$PWD/src/core/or" ] && [ -d "$PWD/src/tools" ]; then
         # Guess the tor directory is the current directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$PWD"
@@ -73,12 +55,12 @@ fi
 #  - if $PWD looks like a chutney directory, set it to $PWD, or
 #  - set it based on $TOR_DIR, expecting chutney to be next to tor, or
 #  - fail and tell the user how to clone the chutney repository
-if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then
+if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then
     if [ -x "$PWD/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$PWD"
         CHUTNEY_PATH="$PWD"
-    elif [ -d "$TOR_DIR" -a -d "$TOR_DIR/../chutney" -a \
-           -x "$TOR_DIR/../chutney/chutney" ]; then
+    elif [ -d "$TOR_DIR" ] && [ -d "$TOR_DIR/../chutney" ] && \
+          [ -x "$TOR_DIR/../chutney/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney"
         CHUTNEY_PATH="$TOR_DIR/../chutney"
     else
@@ -94,12 +76,12 @@ fi
 
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     $ECHO "$myname: Calling newer chutney script $TEST_NETWORK"
     # this may fail if some arguments have spaces in them
     # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces
     # will be handled correctly
-    exec "$TEST_NETWORK" $ORIGINAL_ARGS
+    exec "$TEST_NETWORK" "$@"
 else
     $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH."
     $ECHO "$myname: Please update your chutney using 'git pull'."
diff --git a/src/test/test-process.c b/src/test/test-process.c
new file mode 100644
index 0000000000..f5a1f1a54e
--- /dev/null
+++ b/src/test/test-process.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include <stdio.h>
+#ifdef _WIN32
+#define WINDOWS_LEAN_AND_MEAN
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif /* defined(_WIN32) */
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define SLEEP(sec) Sleep((sec)*1000)
+#else
+#define SLEEP(sec) sleep(sec)
+#endif
+
+/* Trivial test program to test process_t. */
+int
+main(int argc, char **argv)
+{
+  /* Does our process get the right arguments? */
+  for (int i = 0; i < argc; ++i) {
+    fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]);
+    fflush(stdout);
+  }
+
+  /* Make sure our process got our environment variable. */
+  fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n",
+          getenv("TOR_TEST_ENV"));
+  fflush(stdout);
+
+  /* Test line handling on stdout and stderr. */
+  fprintf(stdout, "Output on stdout\nThis is a new line\n");
+  fflush(stdout);
+
+  fprintf(stderr, "Output on stderr\nThis is a new line\n");
+  fflush(stderr);
+
+  fprintf(stdout, "Partial line on stdout ...");
+  fflush(stdout);
+
+  fprintf(stderr, "Partial line on stderr ...");
+  fflush(stderr);
+
+  SLEEP(2);
+
+  fprintf(stdout, "end of partial line on stdout\n");
+  fflush(stdout);
+  fprintf(stderr, "end of partial line on stderr\n");
+  fflush(stderr);
+
+  /* Echo input from stdin. */
+  char buffer[1024];
+
+  int count = 0;
+
+  while (fgets(buffer, sizeof(buffer), stdin)) {
+    /* Strip the newline. */
+    size_t size = strlen(buffer);
+
+    if (size >= 1 && buffer[size - 1] == '\n') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    if (size >= 1 && buffer[size - 1] == '\r') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    fprintf(stdout, "Read line from stdin: '%s'\n", buffer);
+    fflush(stdout);
+
+    if (++count == 3)
+      break;
+  }
+
+  fprintf(stdout, "We are done for here, thank you!\n");
+
+  return 0;
+}
diff --git a/src/test/test-timers.c b/src/test/test-timers.c
index c80fb1e305..18e2191a09 100644
--- a/src/test/test-timers.c
+++ b/src/test/test-timers.c
@@ -1,4 +1,4 @@
-/* Copyright 2016-2019, The Tor Project, Inc. */
+/* Copyright 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -59,7 +59,7 @@ main(int argc, char **argv)
 {
   (void)argc;
   (void)argv;
-  tor_libevent_cfg cfg;
+  tor_libevent_cfg_t cfg;
   memset(&cfg, 0, sizeof(cfg));
   tor_libevent_initialize(&cfg);
   timers_initialize();
diff --git a/src/test/test.c b/src/test/test.c
index 58b468775c..4b6082ce4f 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "app/config/or_state_st.h"
+#include "test/rng_test_helpers.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -37,7 +38,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "lib/compress/compress.h"
@@ -54,7 +55,6 @@
 #include "core/crypto/onion_fast.h"
 #include "core/crypto/onion_tap.h"
 #include "core/or/policies.h"
-#include "feature/stats/rephist.h"
 #include "app/config/statefile.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 
@@ -283,7 +283,7 @@ test_fast_handshake(void *arg)
   /* First, test an entire handshake. */
   memset(client_handshake, 0, sizeof(client_handshake));
   tt_int_op(0, OP_EQ, fast_onionskin_create(&state, client_handshake));
-  tt_assert(! tor_mem_is_zero((char*)client_handshake,
+  tt_assert(! fast_mem_is_zero((char*)client_handshake,
                               sizeof(client_handshake)));
 
   tt_int_op(0, OP_EQ,
@@ -354,18 +354,6 @@ test_onion_queues(void *arg)
   tor_free(onionskin);
 }
 
-static crypto_cipher_t *crypto_rand_aes_cipher = NULL;
-
-// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR
-// cipher in <b>crypto_rand_aes_cipher</b>.
-static void
-crypto_rand_deterministic_aes(char *out, size_t n)
-{
-  tor_assert(crypto_rand_aes_cipher);
-  memset(out, 0, n);
-  crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n);
-}
-
 static void
 test_circuit_timeout(void *arg)
 {
@@ -397,8 +385,7 @@ test_circuit_timeout(void *arg)
 
   // Use a deterministic RNG here, or else we'll get nondeterministic
   // coverage in some of the circuitstats functions.
-  MOCK(crypto_rand, crypto_rand_deterministic_aes);
-  crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover");
+  testing_enable_deterministic_rng();
 
   circuitbuild_running_unit_tests();
 #define timeout0 (build_time_t)(30*1000.0)
@@ -534,8 +521,8 @@ test_circuit_timeout(void *arg)
   circuit_build_times_free_timeouts(&final);
   or_state_free(state);
   teardown_periodic_events();
-  UNMOCK(crypto_rand);
-  crypto_cipher_free(crypto_rand_aes_cipher);
+
+  testing_disable_deterministic_rng();
 }
 
 /** Test encoding and parsing of rendezvous service descriptors. */
@@ -651,166 +638,6 @@ test_rend_fns(void *arg)
   tor_free(intro_points_encrypted);
 }
 
-/** Run unit tests for stats code. */
-static void
-test_stats(void *arg)
-{
-  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
-  char *s = NULL;
-  int i;
-
-  /* Start with testing exit port statistics; we shouldn't collect exit
-   * stats without initializing them. */
-  (void)arg;
-  rep_hist_note_exit_stream_opened(80);
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Initialize stats, note some streams and bytes, and generate history
-   * string. */
-  rep_hist_exit_stats_init(now);
-  rep_hist_note_exit_stream_opened(80);
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  rep_hist_note_exit_stream_opened(443);
-  rep_hist_note_exit_bytes(443, 100, 10000);
-  rep_hist_note_exit_bytes(443, 100, 10000);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "exit-kibibytes-written 80=1,443=1,other=0\n"
-             "exit-kibibytes-read 80=10,443=20,other=0\n"
-             "exit-streams-opened 80=4,443=4,other=0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Add a few bytes on 10 more ports and ensure that only the top 10
-   * ports are contained in the history string. */
-  for (i = 50; i < 60; i++) {
-    rep_hist_note_exit_bytes(i, i, i);
-    rep_hist_note_exit_stream_opened(i);
-  }
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "exit-kibibytes-written 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
-             "59=1,80=1,443=1,other=1\n"
-             "exit-kibibytes-read 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
-             "59=1,80=10,443=20,other=1\n"
-             "exit-streams-opened 52=4,53=4,54=4,55=4,56=4,57=4,58=4,"
-             "59=4,80=4,443=4,other=4\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add some bytes, and ensure we don't generate
-   * a history string. */
-  rep_hist_exit_stats_term();
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Re-start stats, add some bytes, reset stats, and see what history we
-   * get when observing no streams or bytes at all. */
-  rep_hist_exit_stats_init(now);
-  rep_hist_note_exit_stream_opened(80);
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  rep_hist_reset_exit_stats(now);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "exit-kibibytes-written other=0\n"
-             "exit-kibibytes-read other=0\n"
-             "exit-streams-opened other=0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Continue with testing connection statistics; we shouldn't collect
-   * conn stats without initializing them. */
-  rep_hist_note_or_conn_bytes(1, 20, 400, now);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Initialize stats, note bytes, and generate history string. */
-  rep_hist_conn_stats_init(now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,1,0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add some bytes, and ensure we don't generate
-   * a history string. */
-  rep_hist_conn_stats_term();
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Re-start stats, add some bytes, reset stats, and see what history we
-   * get when observing no bytes at all. */
-  rep_hist_conn_stats_init(now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  rep_hist_reset_conn_stats(now);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Continue with testing buffer statistics; we shouldn't collect buffer
-   * stats without initializing them. */
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Initialize stats, add statistics for a single circuit, and generate
-   * the history string. */
-  rep_hist_buffer_stats_init(now);
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "cell-processed-cells 20,0,0,0,0,0,0,0,0,0\n"
-             "cell-queued-cells 2.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
-                               "0.00,0.00\n"
-             "cell-time-in-queue 2,0,0,0,0,0,0,0,0,0\n"
-             "cell-circuits-per-decile 1\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Add nineteen more circuit statistics to the one that's already in the
-   * history to see that the math works correctly. */
-  for (i = 21; i < 30; i++)
-    rep_hist_add_buffer_stats(2.0, 2.0, i);
-  for (i = 20; i < 30; i++)
-    rep_hist_add_buffer_stats(3.5, 3.5, i);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "cell-processed-cells 29,28,27,26,25,24,23,22,21,20\n"
-             "cell-queued-cells 2.75,2.75,2.75,2.75,2.75,2.75,2.75,2.75,"
-                               "2.75,2.75\n"
-             "cell-time-in-queue 3,3,3,3,3,3,3,3,3,3\n"
-             "cell-circuits-per-decile 2\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add statistics for one circuit, and ensure we
-   * don't generate a history string. */
-  rep_hist_buffer_stats_term();
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Re-start stats, add statistics for one circuit, reset stats, and make
-   * sure that the history has all zeros. */
-  rep_hist_buffer_stats_init(now);
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  rep_hist_reset_buffer_stats(now);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "cell-processed-cells 0,0,0,0,0,0,0,0,0,0\n"
-             "cell-queued-cells 0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
-                               "0.00,0.00\n"
-             "cell-time-in-queue 0,0,0,0,0,0,0,0,0,0\n"
-             "cell-circuits-per-decile 0\n",OP_EQ, s);
-
- done:
-  tor_free(s);
-}
-
 #define ENT(name)                                                       \
   { #name, test_ ## name , 0, NULL, NULL }
 #define FORK(name)                                                      \
@@ -824,7 +651,6 @@ static struct testcase_t test_array[] = {
   { "fast_handshake", test_fast_handshake, 0, NULL, NULL },
   FORK(circuit_timeout),
   FORK(rend_fns),
-  FORK(stats),
 
   END_OF_TESTCASES
 };
@@ -845,18 +671,24 @@ struct testgroup_t testgroups[] = {
   { "channeltls/", channeltls_tests },
   { "checkdir/", checkdir_tests },
   { "circuitbuild/", circuitbuild_tests },
+  { "circuitpadding/", circuitpadding_tests },
   { "circuitlist/", circuitlist_tests },
   { "circuitmux/", circuitmux_tests },
-  { "circuituse/", circuituse_tests },
+  { "circuitmux_ewma/", circuitmux_ewma_tests },
   { "circuitstats/", circuitstats_tests },
+  { "circuituse/", circuituse_tests },
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
+  { "config/mgr/", confmgr_tests },
+  { "config/parse/", confparse_tests },
   { "connection/", connection_tests },
   { "conscache/", conscache_tests },
   { "consdiff/", consdiff_tests },
   { "consdiffmgr/", consdiffmgr_tests },
   { "container/", container_tests },
+  { "container/namemap/", namemap_tests },
   { "control/", controller_tests },
+  { "control/btrack/", btrack_tests },
   { "control/event/", controller_event_tests },
   { "crypto/", crypto_tests },
   { "crypto/ope/", crypto_ope_tests },
@@ -864,42 +696,57 @@ struct testgroup_t testgroups[] = {
   { "crypto/openssl/", crypto_openssl_tests },
 #endif
   { "crypto/pem/", pem_tests },
+  { "crypto/rng/", crypto_rng_tests },
   { "dir/", dir_tests },
-  { "dir_handle_get/", dir_handle_get_tests },
+  { "dir/auth/process_descs/", process_descs_tests },
   { "dir/md/", microdesc_tests },
-  { "dir/voting-schedule/", voting_schedule_tests },
+  { "dir/voting/flags/", voting_flags_tests },
+  { "dir/voting/schedule/", voting_schedule_tests },
+  { "dir_handle_get/", dir_handle_get_tests },
+  { "dispatch/", dispatch_tests, },
+  { "dns/", dns_tests },
   { "dos/", dos_tests },
   { "entryconn/", entryconn_tests },
   { "entrynodes/", entrynodes_tests },
-  { "guardfraction/", guardfraction_tests },
   { "extorport/", extorport_tests },
   { "geoip/", geoip_tests },
-  { "legacy_hs/", hs_tests },
+  { "guardfraction/", guardfraction_tests },
   { "hs_cache/", hs_cache },
   { "hs_cell/", hs_cell_tests },
+  { "hs_client/", hs_client_tests },
   { "hs_common/", hs_common_tests },
   { "hs_config/", hs_config_tests },
   { "hs_control/", hs_control_tests },
   { "hs_descriptor/", hs_descriptor },
+  { "hs_dos/", hs_dos_tests },
+  { "hs_intropoint/", hs_intropoint_tests },
   { "hs_ntor/", hs_ntor_tests },
+  { "hs_ob/", hs_ob_tests },
   { "hs_service/", hs_service_tests },
-  { "hs_client/", hs_client_tests },
-  { "hs_intropoint/", hs_intropoint_tests },
   { "introduce/", introduce_tests },
   { "keypin/", keypin_tests },
+  { "legacy_hs/", hs_tests },
   { "link-handshake/", link_handshake_tests },
   { "mainloop/", mainloop_tests },
+  { "netinfo/", netinfo_tests },
   { "nodelist/", nodelist_tests },
   { "oom/", oom_tests },
   { "oos/", oos_tests },
   { "options/", options_tests },
+  { "options/act/", options_act_tests },
+  { "parsecommon/", parsecommon_tests },
   { "periodic-event/" , periodic_event_tests },
   { "policy/" , policy_tests },
+  { "prob_distr/", prob_distr_tests },
   { "procmon/", procmon_tests },
+  { "process/", process_tests },
+  { "proto/haproxy/", proto_haproxy_tests },
   { "proto/http/", proto_http_tests },
   { "proto/misc/", proto_misc_tests },
   { "protover/", protover_tests },
   { "pt/", pt_tests },
+  { "pubsub/build/", pubsub_build_tests },
+  { "pubsub/msg/", pubsub_msg_tests },
   { "relay/" , relay_tests },
   { "relaycell/", relaycell_tests },
   { "relaycrypt/", relaycrypt_tests },
@@ -910,10 +757,13 @@ struct testgroup_t testgroups[] = {
   { "routerlist/", routerlist_tests },
   { "routerset/" , routerset_tests },
   { "scheduler/", scheduler_tests },
-  { "socks/", socks_tests },
+  { "sendme/", sendme_tests },
   { "shared-random/", sr_tests },
+  { "socks/", socks_tests },
+  { "stats/", stats_tests },
   { "status/" , status_tests },
   { "storagedir/", storagedir_tests },
+  { "token_bucket/", token_bucket_tests },
   { "tortls/", tortls_tests },
 #ifndef ENABLE_NSS
   { "tortls/openssl/", tortls_openssl_tests },
@@ -921,10 +771,9 @@ struct testgroup_t testgroups[] = {
   { "tortls/x509/", x509_tests },
   { "util/", util_tests },
   { "util/format/", util_format_tests },
+  { "util/handle/", handle_tests },
   { "util/logging/", logging_tests },
   { "util/process/", util_process_tests },
   { "util/thread/", thread_tests },
-  { "util/handle/", handle_tests },
-  { "dns/", dns_tests },
   END_OF_GROUPS
 };
diff --git a/src/test/test.h b/src/test/test.h
index aacc9dba87..18987719d0 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_H
@@ -37,6 +37,7 @@
 
 #define test_memeq_hex(expr1, hex) test_mem_op_hex(expr1, OP_EQ, hex)
 
+#ifndef COCCI
 #define tt_double_op(a,op,b)                                            \
   tt_assert_test_type(a,b,#a" "#op" "#b,double,(val1_ op val2_),"%g",   \
                       TT_EXIT_TEST_FUNCTION)
@@ -64,6 +65,7 @@
   tt_assert_test_fmt_type(a,b,#a" "#op" "#b,int64_t,(val1_ op val2_),  \
     int64_t, "%"PRId64,                                                \
     {print_ = (int64_t) value_;}, {}, TT_EXIT_TEST_FUNCTION)
+#endif /* !defined(COCCI) */
 
 /**
  * Declare that the test is done, even though no tt___op() calls were made.
@@ -79,169 +81,98 @@ struct crypto_pk_t *pk_generate(int idx);
 void init_pregenerated_keys(void);
 void free_pregenerated_keys(void);
 
-#define US2_CONCAT_2__(a, b) a ## __ ## b
-#define US_CONCAT_2__(a, b) a ## _ ## b
-#define US_CONCAT_3__(a, b, c) a ## _ ## b ## _ ## c
-#define US_CONCAT_2_(a, b) US_CONCAT_2__(a, b)
-#define US_CONCAT_3_(a, b, c) US_CONCAT_3__(a, b, c)
-
-/*
- * These macros are helpful for streamlining the authorship of several test
- * cases that use mocks.
- *
- * The pattern is as follows.
- * * Declare a top level namespace:
- *       #define NS_MODULE foo
- *
- * * For each test case you want to write, create a new submodule in the
- *   namespace. All mocks and other information should belong to a single
- *   submodule to avoid interference with other test cases.
- *   You can simply name the submodule after the function in the module you
- *   are testing:
- *       #define NS_SUBMODULE some_function
- *   or, if you're wanting to write several tests against the same function,
- *   ie., you are testing an aspect of that function, you can use:
- *       #define NS_SUBMODULE ASPECT(some_function, behavior)
- *
- * * Declare all the mocks you will use. The NS_DECL macro serves to declare
- *   the mock in the current namespace (defined by NS_MODULE and NS_SUBMODULE).
- *   It behaves like MOCK_DECL:
- *       NS_DECL(int, dependent_function, (void *));
- *   Here, dependent_function must be declared and implemented with the
- *   MOCK_DECL and MOCK_IMPL macros. The NS_DECL macro also defines an integer
- *   global for use for tracking how many times a mock was called, and can be
- *   accessed by CALLED(mock_name). For example, you might put
- *       CALLED(dependent_function)++;
- *   in your mock body.
- *
- * * Define a function called NS(main) that will contain the body of the
- *   test case. The NS macro can be used to reference a name in the current
- *   namespace.
- *
- * * In NS(main), indicate that a mock function in the current namespace,
- *   declared with NS_DECL is to override that in the global namespace,
- *   with the NS_MOCK macro:
- *       NS_MOCK(dependent_function)
- *   Unmock with:
- *       NS_UNMOCK(dependent_function)
- *
- * * Define the mocks with the NS macro, eg.,
- *       int
- *       NS(dependent_function)(void *)
- *       {
- *           CALLED(dependent_function)++;
- *       }
- *
- * * In the struct testcase_t array, you can use the TEST_CASE and
- *   TEST_CASE_ASPECT macros to define the cases without having to do so
- *   explicitly nor without having to reset NS_SUBMODULE, eg.,
- *       struct testcase_t foo_tests[] = {
- *         TEST_CASE_ASPECT(some_function, behavior),
- *         ...
- *         END_OF_TESTCASES
- *   which will define a test case named "some_function__behavior".
- */
-
-#define NAME_TEST_(name) #name
-#define NAME_TEST(name) NAME_TEST_(name)
-#define ASPECT(test_module, test_name) US2_CONCAT_2__(test_module, test_name)
-#define TEST_CASE(function)  \
-  {  \
-      NAME_TEST(function),  \
-      NS_FULL(NS_MODULE, function, test_main),  \
-      TT_FORK,  \
-      NULL,  \
-      NULL,  \
-  }
-#define TEST_CASE_ASPECT(function, aspect)  \
-  {  \
-      NAME_TEST(ASPECT(function, aspect)),  \
-      NS_FULL(NS_MODULE, ASPECT(function, aspect), test_main),  \
-      TT_FORK,  \
-      NULL,  \
-      NULL,  \
-  }
-
-#define NS(name) US_CONCAT_3_(NS_MODULE, NS_SUBMODULE, name)
-#define NS_FULL(module, submodule, name) US_CONCAT_3_(module, submodule, name)
-
-#define CALLED(mock_name) US_CONCAT_2_(NS(mock_name), called)
-#define NS_DECL(retval, mock_fn, args) \
-    extern int CALLED(mock_fn);        \
-    static retval NS(mock_fn) args; int CALLED(mock_fn) = 0
-#define NS_MOCK(name) MOCK(name, NS(name))
-#define NS_UNMOCK(name) UNMOCK(name)
-
 extern const struct testcase_setup_t passthrough_setup;
 extern const struct testcase_setup_t ed25519_test_setup;
 
 extern struct testcase_t accounting_tests[];
 extern struct testcase_t addr_tests[];
-extern struct testcase_t address_tests[];
 extern struct testcase_t address_set_tests[];
+extern struct testcase_t address_tests[];
 extern struct testcase_t bridges_tests[];
-extern struct testcase_t bwmgt_tests[];
+extern struct testcase_t btrack_tests[];
 extern struct testcase_t buffer_tests[];
+extern struct testcase_t bwmgt_tests[];
 extern struct testcase_t cell_format_tests[];
 extern struct testcase_t cell_queue_tests[];
 extern struct testcase_t channel_tests[];
 extern struct testcase_t channelpadding_tests[];
+extern struct testcase_t circuitpadding_tests[];
 extern struct testcase_t channeltls_tests[];
 extern struct testcase_t checkdir_tests[];
 extern struct testcase_t circuitbuild_tests[];
 extern struct testcase_t circuitlist_tests[];
 extern struct testcase_t circuitmux_tests[];
-extern struct testcase_t circuituse_tests[];
+extern struct testcase_t circuitmux_ewma_tests[];
 extern struct testcase_t circuitstats_tests[];
+extern struct testcase_t circuituse_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
+extern struct testcase_t confmgr_tests[];
+extern struct testcase_t confparse_tests[];
 extern struct testcase_t connection_tests[];
 extern struct testcase_t conscache_tests[];
 extern struct testcase_t consdiff_tests[];
 extern struct testcase_t consdiffmgr_tests[];
 extern struct testcase_t container_tests[];
-extern struct testcase_t controller_tests[];
 extern struct testcase_t controller_event_tests[];
-extern struct testcase_t crypto_tests[];
+extern struct testcase_t controller_tests[];
 extern struct testcase_t crypto_ope_tests[];
 extern struct testcase_t crypto_openssl_tests[];
-extern struct testcase_t dir_tests[];
+extern struct testcase_t crypto_rng_tests[];
+extern struct testcase_t crypto_tests[];
 extern struct testcase_t dir_handle_get_tests[];
+extern struct testcase_t dir_tests[];
+extern struct testcase_t dispatch_tests[];
+extern struct testcase_t dns_tests[];
 extern struct testcase_t dos_tests[];
 extern struct testcase_t entryconn_tests[];
 extern struct testcase_t entrynodes_tests[];
-extern struct testcase_t guardfraction_tests[];
 extern struct testcase_t extorport_tests[];
 extern struct testcase_t geoip_tests[];
-extern struct testcase_t hs_tests[];
+extern struct testcase_t guardfraction_tests[];
+extern struct testcase_t handle_tests[];
 extern struct testcase_t hs_cache[];
 extern struct testcase_t hs_cell_tests[];
+extern struct testcase_t hs_client_tests[];
 extern struct testcase_t hs_common_tests[];
 extern struct testcase_t hs_config_tests[];
 extern struct testcase_t hs_control_tests[];
 extern struct testcase_t hs_descriptor[];
+extern struct testcase_t hs_dos_tests[];
+extern struct testcase_t hs_intropoint_tests[];
 extern struct testcase_t hs_ntor_tests[];
+extern struct testcase_t hs_ob_tests[];
 extern struct testcase_t hs_service_tests[];
-extern struct testcase_t hs_client_tests[];
-extern struct testcase_t hs_intropoint_tests[];
+extern struct testcase_t hs_tests[];
 extern struct testcase_t introduce_tests[];
 extern struct testcase_t keypin_tests[];
 extern struct testcase_t link_handshake_tests[];
 extern struct testcase_t logging_tests[];
 extern struct testcase_t mainloop_tests[];
 extern struct testcase_t microdesc_tests[];
+extern struct testcase_t namemap_tests[];
+extern struct testcase_t netinfo_tests[];
 extern struct testcase_t nodelist_tests[];
 extern struct testcase_t oom_tests[];
 extern struct testcase_t oos_tests[];
 extern struct testcase_t options_tests[];
+extern struct testcase_t options_act_tests[];
+extern struct testcase_t parsecommon_tests[];
 extern struct testcase_t pem_tests[];
 extern struct testcase_t periodic_event_tests[];
 extern struct testcase_t policy_tests[];
+extern struct testcase_t prob_distr_tests[];
+extern struct testcase_t slow_stochastic_prob_distr_tests[];
 extern struct testcase_t procmon_tests[];
+extern struct testcase_t process_tests[];
+extern struct testcase_t process_descs_tests[];
+extern struct testcase_t proto_haproxy_tests[];
 extern struct testcase_t proto_http_tests[];
 extern struct testcase_t proto_misc_tests[];
 extern struct testcase_t protover_tests[];
 extern struct testcase_t pt_tests[];
+extern struct testcase_t pubsub_build_tests[];
+extern struct testcase_t pubsub_msg_tests[];
 extern struct testcase_t relay_tests[];
 extern struct testcase_t relaycell_tests[];
 extern struct testcase_t relaycrypt_tests[];
@@ -252,23 +183,26 @@ extern struct testcase_t routerkeys_tests[];
 extern struct testcase_t routerlist_tests[];
 extern struct testcase_t routerset_tests[];
 extern struct testcase_t scheduler_tests[];
-extern struct testcase_t storagedir_tests[];
+extern struct testcase_t sendme_tests[];
 extern struct testcase_t socks_tests[];
+extern struct testcase_t sr_tests[];
+extern struct testcase_t stats_tests[];
 extern struct testcase_t status_tests[];
+extern struct testcase_t storagedir_tests[];
 extern struct testcase_t thread_tests[];
-extern struct testcase_t tortls_tests[];
+extern struct testcase_t token_bucket_tests[];
 extern struct testcase_t tortls_openssl_tests[];
-extern struct testcase_t util_tests[];
+extern struct testcase_t tortls_tests[];
 extern struct testcase_t util_format_tests[];
 extern struct testcase_t util_process_tests[];
+extern struct testcase_t util_tests[];
+extern struct testcase_t voting_flags_tests[];
 extern struct testcase_t voting_schedule_tests[];
-extern struct testcase_t dns_tests[];
-extern struct testcase_t handle_tests[];
-extern struct testcase_t sr_tests[];
 extern struct testcase_t x509_tests[];
 
 extern struct testcase_t slow_crypto_tests[];
-extern struct testcase_t slow_util_tests[];
+extern struct testcase_t slow_process_tests[];
+extern struct testcase_t slow_ptr_tests[];
 
 extern struct testgroup_t testgroups[];
 
diff --git a/src/test/test_accounting.c b/src/test/test_accounting.c
index 8ae8fe4343..7933df5e35 100644
--- a/src/test/test_accounting.c
+++ b/src/test/test_accounting.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -11,19 +11,16 @@
 
 #include "app/config/or_state_st.h"
 
-#define NS_MODULE accounting
-
-#define NS_SUBMODULE limits
-
 /*
  * Test to make sure accounting triggers hibernation
  * correctly with both sum or max rules set
  */
 
 static or_state_t *or_state;
-NS_DECL(or_state_t *, get_or_state, (void));
+static or_state_t * acct_limits_get_or_state(void);
+ATTR_UNUSED static int acct_limits_get_or_state_called = 0;
 static or_state_t *
-NS(get_or_state)(void)
+acct_limits_get_or_state(void)
 {
   return or_state;
 }
@@ -35,7 +32,8 @@ test_accounting_limits(void *arg)
   time_t fake_time = time(NULL);
   (void) arg;
 
-  NS_MOCK(get_or_state);
+  MOCK(get_or_state,
+       acct_limits_get_or_state);
   or_state = or_state_new();
 
   options->AccountingMax = 100;
@@ -94,12 +92,10 @@ test_accounting_limits(void *arg)
 
   goto done;
  done:
-  NS_UNMOCK(get_or_state);
+  UNMOCK(get_or_state);
   or_state_free(or_state);
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t accounting_tests[] = {
   { "bwlimits", test_accounting_limits, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
diff --git a/src/test/test_addr.c b/src/test/test_addr.c
index 8868edce25..cf5aad7e71 100644
--- a/src/test/test_addr.c
+++ b/src/test/test_addr.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESSMAP_PRIVATE
@@ -11,6 +11,8 @@
 #include "feature/client/addressmap.h"
 #include "test/log_test_helpers.h"
 #include "lib/net/resolve.h"
+#include "test/rng_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #ifdef HAVE_SYS_UN_H
 #include <sys/un.h>
@@ -50,6 +52,7 @@ test_addr_basic(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define test_op_ip6_(a,op,b,e1,e2)                               \
   STMT_BEGIN                                                     \
   tt_assert_test_fmt_type(a,b,e1" "#op" "e2,struct in6_addr*,    \
@@ -67,6 +70,7 @@ test_addr_basic(void *arg)
     TT_EXIT_TEST_FUNCTION                                        \
   );                                                             \
   STMT_END
+#endif /* !defined(COCCI) */
 
 /** Helper: Assert that two strings both decode as IPv6 addresses with
  * tor_inet_pton(), and both decode to the same address. */
@@ -106,9 +110,10 @@ test_addr_basic(void *arg)
     tt_int_op(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), OP_EQ, 1); \
     t1.family = AF_INET6;                                      \
     if (tor_addr_is_internal(&t1, for_listening))              \
-      TT_DIE(("%s was not internal", a));                      \
+      TT_DIE(("%s was internal", a));                      \
   STMT_END
 
+#ifndef COCCI
 /** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by
  * tor_inet_pton(), give addresses that compare in the order defined by
  * <b>op</b> with tor_addr_compare(). */
@@ -133,6 +138,7 @@ test_addr_basic(void *arg)
       TT_DIE(("Failed: tor_addr_compare_masked(%s,%s,%d) %s 0", \
               a, b, m, #op));                                   \
   STMT_END
+#endif /* !defined(COCCI) */
 
 /** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with
  * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address
@@ -239,7 +245,7 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(0,OP_EQ, tor_addr_lookup("9000::5", AF_UNSPEC, &t1));
   tt_int_op(AF_INET6,OP_EQ, tor_addr_family(&t1));
   tt_int_op(0x90,OP_EQ, tor_addr_to_in6_addr8(&t1)[0]);
-  tt_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
+  tt_assert(fast_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
   tt_int_op(0x05,OP_EQ, tor_addr_to_in6_addr8(&t1)[15]);
 
   /* === Test pton: valid af_inet6 */
@@ -653,12 +659,7 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x01010202);
   r=tor_addr_parse_mask_ports("3.4.16.032:1-2",0,&t1, &mask, &port1, &port2);
-  tt_int_op(r, OP_EQ, AF_INET);
-  tt_int_op(mask,OP_EQ,32);
-  tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
-  tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x03041020);
-  tt_uint_op(port1, OP_EQ, 1);
-  tt_uint_op(port2, OP_EQ, 2);
+  tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("1.1.2.3/255.255.128.0",0,&t1, &mask,NULL,NULL);
   tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,17);
@@ -696,7 +697,7 @@ test_addr_ip6_helpers(void *arg)
                               &t1,&mask,&port1,&port2);
   tt_int_op(r,OP_EQ,AF_INET6);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET6);
-  tt_assert(tor_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
+  tt_assert(fast_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
   tt_int_op(mask,OP_EQ,0);
   tt_int_op(port1,OP_EQ,1);
   tt_int_op(port2,OP_EQ,65535);
@@ -723,104 +724,559 @@ test_addr_ip6_helpers(void *arg)
   ;
 }
 
-/** Test tor_addr_port_parse(). */
+/* Test that addr_str successfully parses, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_PARSE_FMT(addr_str, expect_family, fmt_decorated, \
+                            expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that addr_str fails to parse, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_PARSE_XFAIL(addr_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str and default_port successfully parse, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, expect_family, \
+                                 fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that addr_port_str and default_port fail to parse, and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_PARSE_XFAIL(addr_port_str, default_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as an IPv4 address using
+ * tor_lookup_hostname(), and:
+ *  - the fmt_addr32() of the result is expect_str.
+ */
+#define TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, expect_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(addr_str, &addr32h); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_str_op(fmt_addr32(addr32h), OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse using tor_lookup_hostname(), with a
+ * permanent failure, and:
+ *  - the returned address is 0.
+ */
+#define TEST_ADDR_V4_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(bad_str, &addr32h); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_str as an IPv4 address using tor_lookup_hostname()
+ * does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is 0.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_V4_LOOKUP(host_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(host_str, &addr32h); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a require_family IP address using
+ * tor_addr_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_LOOKUP_FMT(addr_str, require_family, expect_family, \
+                             fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(addr_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse as a require_family IP address using
+ * tor_addr_lookup(), with a permanent failure, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_LOOKUP_XFAIL(bad_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(bad_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that looking up host_string as a require_family IP address using
+ * tor_addr_lookup(),  does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is null.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_LOOKUP(host_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(host_str, require_family, &addr); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, expect_family, \
+                                  fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(addr_port_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that bad_str fails to parse as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(bad_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_port_str as an IP address using
+ * tor_addr_port_lookup(),  does something sensible:
+ *  - the result is -1 or 0.
+ *  - if the result is a failure, the returned address is null, and the
+ *    returned port is zero,
+ *  - if the result is a success, the returned port is expect_success_port,
+ *    and the returned family is AF_INET or AF_INET6.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(host_port_str, &addr, &port); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 0); \
+    if (r == -1) { \
+      tt_assert(tor_addr_is_null(&addr)); \
+      tt_int_op(port, OP_EQ, 0); \
+    } else { \
+      tt_assert(tor_addr_family(&addr) == AF_INET || \
+                tor_addr_family(&addr) == AF_INET6); \
+      tt_int_op(port, OP_EQ, expect_success_port); \
+    } \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical IPv4 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup() with AF_INET,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_port_lookup(), because there is no port.
+ */
+#define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \
+                             addr_str, 111); \
+    TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET, 0, addr_str, 0); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET, AF_INET, 0, addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical fmt_decorated
+ * IPv6 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \
+                             addr_str, 222); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, addr_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_str successfully parses, and the fmt_decorated canonical
+ * IPv6 string is expect_str.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port.
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \
+                             expect_str, 333); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, expect_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical IPv4 address
+ * string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port.
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical undecorated
+ * IPv6 address string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port, and because it only
+ *    supports IPv4,
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET6, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+  STMT_END
+
+/* Test that bad_str fails to parse due to a bad address or port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup().
+ */
+#define TEST_ADDR_PARSE_XFAIL_MALFORMED(bad_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_XFAIL(bad_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str, 666); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(bad_str); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET6); \
+    TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str); \
+  STMT_END
+
+/* Test that host_str is treated as a hostname, and not an address.
+ * Check for success or failure using the network-dependent functions:
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup(), expecting a zero port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port.
+ */
+#define TEST_HOSTNAME(host_str) \
+  STMT_BEGIN \
+    TEST_HOST_V4_LOOKUP(host_str); \
+    TEST_HOST_LOOKUP(host_str, AF_UNSPEC); \
+    TEST_HOST_LOOKUP(host_str, AF_INET); \
+    TEST_HOST_LOOKUP(host_str, AF_INET6); \
+    TEST_HOST_PORT_LOOKUP(host_str, 0); \
+    TEST_ADDR_PARSE_XFAIL(host_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \
+  STMT_END
+
+/* Test that host_port_str is treated as a hostname and port, and not a
+ * hostname or an address.
+ * Check for success or failure using the network-dependent function:
+ *  - tor_addr_port_lookup(), expecting expect_success_port if the lookup is
+ *    successful.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(), because it doesn't support ports,
+ *  - tor_addr_lookup(), regardless of the address family, because it doesn't
+ *    support ports.
+ */
+#define TEST_HOSTNAME_PORT(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port); \
+    TEST_ADDR_PARSE_XFAIL(host_port_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET6); \
+  STMT_END
+
+static void
+test_addr_parse_canonical(void *arg)
+{
+  int r;
+  tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
+  char buf[TOR_ADDR_BUF_LEN];
+
+  (void)arg;
+
+  /* Correct calls. */
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.1");
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2");
+
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::33:44]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[2::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1);
+
+  /* Allow IPv6 without square brackets, when there is no port, but only if
+   * there is a default port */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::1", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("2::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0);
+ done:
+  ;
+}
+
+/** Test tor_addr_parse() and tor_addr_port_parse(). */
 static void
 test_addr_parse(void *arg)
 {
+
   int r;
   tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
   char buf[TOR_ADDR_BUF_LEN];
-  uint16_t port = 0;
 
-  /* Correct call. */
   (void)arg;
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.1:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "192.0.2.1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "::1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  /* Domain name. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
 
-  /* Only IP. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  mock_hostname_resolver();
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,200);
+  /* IPv6-mapped IPv4 addresses. Tor doesn't really use these. */
+  TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0,
+                     "11:22:33:44:55:66:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_ADDR_V6_PARSE("11:22::33:44:1.2.3.4", 0,
+                     "11:22::33:44:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, 400);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,400);
+  /* Ports. */
+  TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234);
+  TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234);
 
-  /* Bad port. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  /* Host names. */
+  TEST_HOSTNAME("localhost");
+  TEST_HOSTNAME_PORT("localhost:1234", 1234);
+  TEST_HOSTNAME_PORT("localhost:0", 0);
 
-  /* Only domain name */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("torproject.org");
+  TEST_HOSTNAME_PORT("torproject.org:56", 56);
 
-  /* Bad IP address */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("probably-not-a-valid-dns.name-tld");
+  TEST_HOSTNAME_PORT("probably-not-a-valid-dns.name-tld:789", 789);
+
+  /* Malformed addresses. */
+  /* Empty string. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("");
+
+  /* Square brackets around IPv4 address. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.1]");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345");
+
+  /* Only left square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[11:22::33:44");
+
+  /* Only right square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44]");
 
-  /* Make sure that the default port has lower priority than the real
-     one */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:1337",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1337);
+  /* Leading colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED(":11:22::33:44");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1369",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1369);
+  /* Trailing colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:");
+
+  /* Bad port. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:77777");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:88888");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:99999");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:-2");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:-3");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:-4");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:1 bad");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:bad-port-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port-1");
+
+  /* Bad hostname */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid:22222");
+
+  /* Ambiguous cases */
+  /* Too many hex words in IPv4-mapped IPv6 address.
+   * But some OS host lookup routines accept it as a hostname, or
+   * as an IP address?? (I assume they discard unused characters). */
+  TEST_HOSTNAME("11:22:33:44:55:66:77:88:1.2.3.4");
+
+  /* IPv6 address with port and no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address:port ? */
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+  /* Is it a port, or are there too many hex words?
+   * We reject it either way, but some OS host lookup routines accept it as an
+   * IPv6 address:port */
+  TEST_HOSTNAME_PORT("11:22:33:44:55:66:77:88:99", 99);
+  /* But we accept it if it has square brackets. */
+  TEST_ADDR_V6_PORT_PARSE("[11:22:33:44:55:66:77:88]:99",
+                           "11:22:33:44:55:66:77:88",99);
+
+  /* Bad IPv4 address
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv4 address[:port], with a zero last octet */
+  TEST_HOSTNAME("192.0.1");
+  TEST_HOSTNAME_PORT("192.0.2:1234", 1234);
+
+  /* More bad IPv6 addresses and ports: no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address[:port] */
+  TEST_HOSTNAME_PORT("::1:12345", 12345);
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+
+  /* And this is an ambiguous case, which is interpreted as an IPv6 address. */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::88:99", 0);
+  /* Use square brackets to resolve the ambiguity */
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::88:99]", 1);
+  TEST_ADDR_V6_PORT_PARSE("[11:22::88]:99",
+                           "11:22::88",99);
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -891,27 +1347,6 @@ test_virtaddrmap(void *data)
   ;
 }
 
-static const char *canned_data = NULL;
-static size_t canned_data_len = 0;
-
-/* Mock replacement for crypto_rand() that returns canned data from
- * canned_data above. */
-static void
-crypto_canned(char *ptr, size_t n)
-{
-  if (canned_data_len) {
-    size_t to_copy = MIN(n, canned_data_len);
-    memcpy(ptr, canned_data, to_copy);
-    canned_data += to_copy;
-    canned_data_len -= to_copy;
-    n -= to_copy;
-    ptr += to_copy;
-  }
-  if (n) {
-    crypto_rand_unmocked(ptr, n);
-  }
-}
-
 static void
 test_virtaddrmap_persist(void *data)
 {
@@ -919,6 +1354,8 @@ test_virtaddrmap_persist(void *data)
   const char *a, *b, *c;
   tor_addr_t addr;
   char *ones = NULL;
+  const char *canned_data;
+  size_t canned_data_len;
 
   addressmap_init();
 
@@ -937,7 +1374,7 @@ test_virtaddrmap_persist(void *data)
                 "1234567890" // the second call returns this.
                 "abcdefghij"; // the third call returns this.
   canned_data_len = 30;
-  MOCK(crypto_rand, crypto_canned);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
 
   a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
                                           tor_strdup("quuxit.baz"));
@@ -947,9 +1384,9 @@ test_virtaddrmap_persist(void *data)
   tt_assert(b);
   tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual");
   tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual");
+  testing_disable_prefilled_rng();
 
   // Now try something to get us an ipv4 address
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16",
                                                 AF_INET, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -966,22 +1403,23 @@ test_virtaddrmap_persist(void *data)
 
   // Try some canned entropy and verify all the we discard duplicates,
   // addresses that end with 0, and addresses that end with 255.
-  MOCK(crypto_rand, crypto_canned);
   canned_data = "\x01\x02\x03\x04" // okay
                 "\x01\x02\x03\x04" // duplicate
                 "\x03\x04\x00\x00" // bad ending 1
                 "\x05\x05\x00\xff" // bad ending 2
                 "\x05\x06\x07\xf0"; // okay
   canned_data_len = 20;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumble.onion"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumpus.onion"));
   tt_str_op(a, OP_EQ, "192.168.3.4");
   tt_str_op(b, OP_EQ, "192.168.7.240");
+  testing_disable_prefilled_rng();
 
   // Now try IPv6!
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20",
                                                 AF_INET6, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -997,7 +1435,7 @@ test_virtaddrmap_persist(void *data)
   tt_assert(!strcmpstart(b, "[1010:f"));
 
   // Try IPv6 with canned entropy, to make sure we detect duplicates.
-  MOCK(crypto_rand, crypto_canned);
+
   canned_data = "acanthopterygian" // okay
                 "cinematographist" // okay
                 "acanthopterygian" // duplicate
@@ -1006,6 +1444,8 @@ test_virtaddrmap_persist(void *data)
                 "cinematographist" // duplicate
                 "coadministration"; // okay
   canned_data_len = 16 * 7;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
                                           tor_strdup("wuffle.baz"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -1018,9 +1458,11 @@ test_virtaddrmap_persist(void *data)
 
   // Try address exhaustion: make sure we can actually fail if we
   // get too many already-existing addresses.
+  testing_disable_prefilled_rng();
   canned_data_len = 128*1024;
   canned_data = ones = tor_malloc(canned_data_len);
   memset(ones, 1, canned_data_len);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
   // There is some chance this one will fail if a previous random
   // allocation gave out the address already.
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -1037,7 +1479,7 @@ test_virtaddrmap_persist(void *data)
   expect_single_log_msg_containing("Ran out of virtual addresses!");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(ones);
   addressmap_free_all();
   teardown_capture_of_logs();
@@ -1206,13 +1648,169 @@ test_addr_rfc6598(void *arg)
   ;
 }
 
+#define TEST_ADDR_ATON(a, rv) STMT_BEGIN \
+    struct in_addr addr; \
+    tt_int_op(tor_inet_aton(a, &addr), OP_EQ, rv); \
+  STMT_END;
+
+static void
+test_addr_octal(void *arg)
+{
+  (void)arg;
+
+  /* Test non-octal IP addresses. */
+  TEST_ADDR_ATON("0.1.2.3", 1);
+  TEST_ADDR_ATON("1.0.2.3", 1);
+  TEST_ADDR_ATON("1.2.3.0", 1);
+
+  /* Test octal IP addresses. */
+  TEST_ADDR_ATON("01.1.2.3", 0);
+  TEST_ADDR_ATON("1.02.3.4", 0);
+  TEST_ADDR_ATON("1.2.3.04", 0);
+ done:
+  ;
+}
+
+#define get_ipv4(test_addr, str, iprv) STMT_BEGIN               \
+    test_addr = tor_malloc(sizeof(tor_addr_t));                 \
+    test_addr->family = AF_INET;                                \
+    iprv = tor_inet_aton(str, &test_addr->addr.in_addr);        \
+    tor_assert(iprv);                                           \
+  STMT_END;
+
+#define get_ipv6(test_addr, str, iprv) STMT_BEGIN                       \
+    test_addr = tor_malloc(sizeof(tor_addr_t));                         \
+    test_addr->family = AF_INET6;                                       \
+    iprv = tor_inet_pton(AF_INET6, str, &test_addr->addr.in6_addr);     \
+    tor_assert(iprv);                                                   \
+  STMT_END;
+
+#define get_af_unix(test_addr) STMT_BEGIN                       \
+    test_addr = tor_malloc_zero(sizeof(tor_addr_t));            \
+    test_addr->family = AF_UNIX;                                \
+  STMT_END;
+
+#define get_af_unspec(test_addr) STMT_BEGIN                     \
+    test_addr = tor_malloc_zero(sizeof(tor_addr_t));            \
+    test_addr->family = AF_UNSPEC;                              \
+  STMT_END;
+
+#define TEST_ADDR_VALIDITY(a, lis, rv) STMT_BEGIN               \
+    tor_assert(a);                                              \
+    tt_int_op(tor_addr_is_valid(a, lis), OP_EQ, rv);            \
+  STMT_END;
+
+/* Here we can change the addresses we are testing for. */
+#define IP4_TEST_ADDR "123.98.45.1"
+#define IP6_TEST_ADDR "2001:0DB8:AC10:FE01::"
+
+static void
+test_addr_is_valid(void *arg)
+{
+  (void)arg;
+  tor_addr_t *test_addr;
+  int iprv;
+
+  /* Tests for IPv4 addresses. */
+
+  /* Test for null IPv4 address. */
+  get_ipv4(test_addr, "0.0.0.0", iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv4 address. */
+  get_ipv4(test_addr, IP4_TEST_ADDR, iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 1);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Tests for IPv6 addresses. */
+
+  /* Test for null IPv6 address. */
+  get_ipv6(test_addr, "::", iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv6 address. */
+  get_ipv6(test_addr, IP6_TEST_ADDR, iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 1);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Test for address of type AF_UNIX. */
+
+  get_af_unix(test_addr);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 0);
+  tor_free(test_addr);
+
+  /* Test for address of type AF_UNSPEC. */
+
+  get_af_unspec(test_addr);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 0);
+
+ done:
+  tor_free(test_addr);
+}
+
+#define TEST_ADDR_IS_NULL(a, rv) STMT_BEGIN                 \
+    tor_assert(a);                                          \
+    tt_int_op(tor_addr_is_null(a), OP_EQ, rv);              \
+  STMT_END;
+
+static void
+test_addr_is_null(void *arg)
+{
+  (void)arg;
+  tor_addr_t *test_addr;
+  int iprv;
+
+  /* Test for null IPv4. */
+  get_ipv4(test_addr, "0.0.0.0", iprv);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv4. */
+  get_ipv4(test_addr, IP4_TEST_ADDR, iprv);
+  TEST_ADDR_IS_NULL(test_addr, 0);
+  tor_free(test_addr);
+
+  /* Test for null IPv6. */
+  get_ipv6(test_addr, "::", iprv);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv6. */
+  get_ipv6(test_addr, IP6_TEST_ADDR, iprv);
+  TEST_ADDR_IS_NULL(test_addr, 0);
+  tor_free(test_addr);
+
+  /* Test for address family AF_UNIX. */
+  get_af_unix(test_addr);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+  tor_free(test_addr);
+
+  /* Test for address family AF_UNSPEC. */
+  get_af_unspec(test_addr);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+
+ done:
+  tor_free(test_addr);
+}
+
+#ifndef COCCI
 #define ADDR_LEGACY(name)                                               \
   { #name, test_addr_ ## name , 0, NULL, NULL }
+#endif
 
 struct testcase_t addr_tests[] = {
   ADDR_LEGACY(basic),
   ADDR_LEGACY(ip6_helpers),
   ADDR_LEGACY(parse),
+  ADDR_LEGACY(parse_canonical),
   { "virtaddr", test_virtaddrmap, 0, NULL, NULL },
   { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL },
   { "localname", test_addr_localname, 0, NULL, NULL },
@@ -1221,5 +1819,8 @@ struct testcase_t addr_tests[] = {
   { "is_loopback", test_addr_is_loopback, 0, NULL, NULL },
   { "make_null", test_addr_make_null, 0, NULL, NULL },
   { "rfc6598", test_addr_rfc6598, 0, NULL, NULL },
+  { "octal", test_addr_octal, 0, NULL, NULL },
+  { "address_validity", test_addr_is_valid, 0, NULL, NULL },
+  { "address_is_null", test_addr_is_null, 0, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_address.c b/src/test/test_address.c
index c33c30aee5..4cedbda347 100644
--- a/src/test/test_address.c
+++ b/src/test/test_address.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESS_PRIVATE
@@ -24,6 +24,10 @@
 #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/node_st.h"
 #include "feature/nodelist/nodelist.h"
 #include "lib/net/address.h"
 #include "test/test.h"
@@ -456,7 +460,7 @@ test_address_ifreq_to_smartlist(void *arg)
   ifc->ifc_len = sizeof(struct ifreq);
   ifc->ifc_ifcu.ifcu_req = ifr;
 
-  results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
+  results = ifreq_to_smartlist((const uint8_t *)ifc->ifc_buf,ifc->ifc_len);
   tt_int_op(smartlist_len(results),OP_EQ,1);
 
   tor_addr = smartlist_get(results, 0);
@@ -479,7 +483,7 @@ test_address_ifreq_to_smartlist(void *arg)
   SMARTLIST_FOREACH(results, tor_addr_t *, t, tor_free(t));
   smartlist_free(results);
 
-  results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
+  results = ifreq_to_smartlist((const uint8_t *)ifc->ifc_buf,ifc->ifc_len);
   tt_int_op(smartlist_len(results),OP_EQ,2);
 
   tor_addr = smartlist_get(results, 0);
@@ -1170,8 +1174,167 @@ test_address_tor_addr_in_same_network_family(void *ignored)
   return;
 }
 
+static node_t *
+helper_create_mock_node(char id_char)
+{
+  node_t *node = tor_malloc_zero(sizeof(node_t));
+  routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t));
+  tor_addr_make_null(&ri->ipv6_addr, AF_INET6);
+  node->ri = ri;
+  memset(node->identity, id_char, sizeof(node->identity));
+  return node;
+}
+
+static void
+helper_free_mock_node(node_t *node)
+{
+  if (!node)
+    return;
+  tor_free(node->ri);
+  tor_free(node);
+}
+
+#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, ipv4_addr); \
+    node->ri->addr = tor_addr_to_ipv4h(&addr); \
+    node->ri->or_port = ipv4_port; \
+  }
+
+#define NODE_CLEAR_IPV4(node) { \
+    node->ri->addr = 0; \
+    node->ri->or_port = 0; \
+  }
+
+#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \
+    tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \
+    node->ri->ipv6_orport = ipv6_port; \
+  }
+
+static void
+test_address_tor_node_in_same_network_family(void *ignored)
+{
+  (void)ignored;
+  node_t *node_a = helper_create_mock_node('a');
+  node_t *node_b = helper_create_mock_node('b');
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "8.8.4.4", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "1.1.1.1", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_a);
+  NODE_SET_IPV6(node_a, "2001:470:20::2", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_b);
+  NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1);
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+ done:
+  helper_free_mock_node(node_a);
+  helper_free_mock_node(node_b);
+}
+
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+/* Test dirserv_router_has_valid_address() on a stub routerinfo, with only its
+ * address fields set. Use IPv4 ipv4_addr_str and IPv6 ipv6_addr_str.
+ * Fail if it does not return rv. */
+#define TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, ipv6_addr_str, rv) \
+  STMT_BEGIN \
+    ri = tor_malloc_zero(sizeof(routerinfo_t)); \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, (ipv4_addr_str));   \
+    ri->addr = tor_addr_to_ipv4h(&addr); \
+    tor_addr_parse(&ri->ipv6_addr, (ipv6_addr_str)); \
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, (rv)); \
+    tor_free(ri); \
+  STMT_END
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a null
+ * IPv6 address. */
+#define CHECK_RI_ADDR(ipv4_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, "::", rv)
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a non-internal
+ * IPv4 address, so that the IPv6 check is reached. */
+#define CHECK_RI_ADDR6(ipv6_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER("1.0.0.1", ipv6_addr_str, rv)
+
+static void
+test_address_dirserv_router_addr_private(void *opt_dir_allow_private)
+{
+  /* A stub routerinfo structure, with only its address fields set. */
+  routerinfo_t *ri = NULL;
+  /* The expected return value for private addresses.
+   * Modified if DirAllowPrivateAddresses is 1. */
+  int private_rv = -1;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(get_options, mock_get_options);
+
+  if (opt_dir_allow_private) {
+    mock_options.DirAllowPrivateAddresses = 1;
+    private_rv = 0;
+  }
+
+  CHECK_RI_ADDR("1.0.0.1", 0);
+  CHECK_RI_ADDR("10.0.0.1", private_rv);
+
+  CHECK_RI_ADDR6("2600::1", 0);
+  CHECK_RI_ADDR6("fe80::1", private_rv);
+
+  /* Null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  CHECK_RI_ADDR("0.0.0.0", private_rv);
+  TEST_ROUTER_VALID_ADDRESS_HELPER("0.0.0.0", "::", private_rv);
+
+  /* IPv6 null succeeds, because IPv4 is not null */
+  CHECK_RI_ADDR6("::", 0);
+
+  /* Byte-zeroed null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, private_rv);
+    tor_free(ri);
+  }
+
+  /* IPv6 null succeeds, because IPv4 is not internal */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    ri->addr = 16777217; /* 1.0.0.1 */
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, 0);
+    tor_free(ri);
+  }
+
+ done:
+  tor_free(ri);
+  UNMOCK(get_options);
+}
+
 #define ADDRESS_TEST(name, flags) \
   { #name, test_address_ ## name, flags, NULL, NULL }
+#define ADDRESS_TEST_STR_ARG(name, flags, str_arg) \
+  { #name "/" str_arg, test_address_ ## name, flags, &passthrough_setup, \
+    (void *)(str_arg) }
 
 struct testcase_t address_tests[] = {
   ADDRESS_TEST(udp_socket_trick_whitebox, TT_FORK),
@@ -1202,5 +1365,8 @@ struct testcase_t address_tests[] = {
   ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0),
   ADDRESS_TEST(tor_addr_eq_ipv4h, 0),
   ADDRESS_TEST(tor_addr_in_same_network_family, 0),
+  ADDRESS_TEST(tor_node_in_same_network_family, 0),
+  ADDRESS_TEST(dirserv_router_addr_private, 0),
+  ADDRESS_TEST_STR_ARG(dirserv_router_addr_private, 0, "allow_private"),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c
index 4e55d71ff4..c2fe718935 100644
--- a/src/test/test_address_set.c
+++ b/src/test/test_address_set.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -105,6 +105,8 @@ test_nodelist(void *arg)
        mock_networkstatus_get_latest_consensus_by_flavor);
   MOCK(get_estimated_address_per_node,
        mock_get_estimated_address_per_node);
+  MOCK(dirlist_add_trusted_dir_addresses,
+       mock_dirlist_add_trusted_dir_addresses);
 
   dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
   dummy_ns->flavor = FLAV_MICRODESC;
@@ -120,7 +122,10 @@ test_nodelist(void *arg)
    * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
   addr_per_node = 1024;
 
-  /* No node no nothing. The lookups should be empty. */
+  /* No node no nothing. The lookups should be empty. We've mocked the
+   * dirlist_add_trusted_dir_addresses in order for _no_ authorities to be
+   * added to the filter else it makes this test to trigger many false
+   * positive. */
   nodelist_set_consensus(dummy_ns);
 
   /* The address set should be empty. */
@@ -174,6 +179,7 @@ test_nodelist(void *arg)
   UNMOCK(networkstatus_get_latest_consensus);
   UNMOCK(networkstatus_get_latest_consensus_by_flavor);
   UNMOCK(get_estimated_address_per_node);
+  UNMOCK(dirlist_add_trusted_dir_addresses);
 }
 
 /** Test that the no-reentry exit filter works as intended */
diff --git a/src/test/test_bridges.c b/src/test/test_bridges.c
index 879ae6636b..f1624a529d 100644
--- a/src/test/test_bridges.c
+++ b/src/test/test_bridges.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information     */
 
 /**
@@ -550,8 +550,6 @@ test_bridges_get_transport_by_bridge_addrport_no_ptlist(void *arg)
   sweep_bridge_list();
 }
 
-#define PT_PRIVATE
-
 /**
  * Calling get_transport_by_bridge_addrport() with the address and port of a
  * configured bridge which uses a pluggable transport should return 0 and set
diff --git a/src/test/test_bt.sh b/src/test/test_bt.sh
index df8bcb8eda..312905a4e2 100755
--- a/src/test/test_bt.sh
+++ b/src/test/test_bt.sh
@@ -3,8 +3,6 @@
 
 exitcode=0
 
-ulimit -c 0
-
 export ASAN_OPTIONS="handle_segv=0:allow_user_segv_handler=1"
 "${builddir:-.}/src/test/test-bt-cl" backtraces || exit $?
 "${builddir:-.}/src/test/test-bt-cl" assert 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?"
diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c
index 0c15a02ee4..5f9a88705c 100644
--- a/src/test/test_bt_cl.c
+++ b/src/test/test_bt_cl.c
@@ -1,9 +1,12 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include <stdio.h>
 #include <stdlib.h>
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
 
 /* To prevent 'assert' from going away. */
 #undef TOR_COVERAGE
@@ -30,7 +33,7 @@ int a_tangled_web(int x) NOINLINE;
 int we_weave(int x) NOINLINE;
 
 #ifdef HAVE_CFLAG_WNULL_DEREFERENCE
-DISABLE_GCC_WARNING(null-dereference)
+DISABLE_GCC_WARNING("-Wnull-dereference")
 #endif
 int
 crash(int x)
@@ -43,7 +46,7 @@ crash(int x)
     *(volatile int *)0 = 0;
 #endif /* defined(__clang_analyzer__) || defined(__COVERITY__) */
   } else if (crashtype == 1) {
-    tor_assert(1 == 0);
+    tor_assertf(1 == 0, "%d != %d", 1, 0);
   } else if (crashtype == -1) {
     ;
   }
@@ -52,7 +55,7 @@ crash(int x)
   return crashtype;
 }
 #ifdef HAVE_CFLAG_WNULL_DEREFERENCE
-ENABLE_GCC_WARNING(null-dereference)
+ENABLE_GCC_WARNING("-Wnull-dereference")
 #endif
 
 int
@@ -88,6 +91,11 @@ main(int argc, char **argv)
     return 1;
   }
 
+#ifdef HAVE_SYS_RESOURCE_H
+  struct rlimit rlim = { .rlim_cur = 0, .rlim_max = 0 };
+  setrlimit(RLIMIT_CORE, &rlim);
+#endif
+
 #if !(defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
    defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION))
     puts("Backtrace reporting is not supported on this platform");
diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c
new file mode 100644
index 0000000000..2b2f34fc23
--- /dev/null
+++ b/src/test/test_btrack.c
@@ -0,0 +1,129 @@
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+
+static void
+send_state(const orconn_state_msg_t *msg_in)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_state_publish(msg);
+}
+
+static void
+send_status(const orconn_status_msg_t *msg_in)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_status_publish(msg);
+}
+
+static void
+send_chan(const ocirc_chan_msg_t *msg_in)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_btrack_launch(void *arg)
+{
+  orconn_state_msg_t conn;
+  ocirc_chan_msg_t circ;
+  memset(&conn, 0, sizeof(conn));
+  memset(&circ, 0, sizeof(circ));
+
+  (void)arg;
+  conn.gid = 1;
+  conn.chan = 1;
+  conn.proxy_type = PROXY_NONE;
+  conn.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 1;
+  circ.onehop = true;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1");
+  expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1");
+  teardown_capture_of_logs();
+
+  conn.gid = 2;
+  conn.chan = 2;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 2;
+  circ.onehop = false;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0");
+  expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+static void
+test_btrack_delete(void *arg)
+{
+  orconn_state_msg_t state;
+  orconn_status_msg_t status;
+  memset(&state, 0, sizeof(state));
+  memset(&status, 0, sizeof(status));
+
+  (void)arg;
+  state.gid = 1;
+  state.chan = 1;
+  state.proxy_type = PROXY_NONE;
+  state.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&state);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0");
+  teardown_capture_of_logs();
+
+  status.gid = 1;
+  status.status = OR_CONN_EVENT_CLOSED;
+  status.reason = 0;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_status(&status);
+  expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+struct testcase_t btrack_tests[] = {
+  { "launch", test_btrack_launch, TT_FORK, &helper_pubsub_setup, NULL },
+  { "delete", test_btrack_delete, TT_FORK, &helper_pubsub_setup, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c
index 3e7364a5c8..fbaa628fd7 100644
--- a/src/test/test_buffers.c
+++ b/src/test/test_buffers.c
@@ -1,12 +1,12 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define BUFFERS_PRIVATE
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/tls/tortls.h"
 #include "lib/compress/compress.h"
diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c
index 5a013aa268..117783cafc 100644
--- a/src/test/test_bwmgt.c
+++ b/src/test/test_bwmgt.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,18 +6,70 @@
  * \brief tests for bandwidth management / token bucket functions
  */
 
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
+#define DIRAUTH_SYS_PRIVATE
 #define TOKEN_BUCKET_PRIVATE
 
 #include "core/or/or.h"
-#include "test/test.h"
 
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/evloop/token_bucket.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+
+#include "app/config/or_options_st.h"
+#include "core/or/connection_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 // an imaginary time, in timestamp units. Chosen so it will roll over.
 static const uint32_t START_TS = UINT32_MAX-10;
 static const int32_t KB = 1024;
 static const uint32_t GB = (UINT64_C(1) << 30);
 
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+static networkstatus_t *dummy_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return dummy_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  tor_assert(f == FLAV_MICRODESC);
+  return dummy_ns;
+}
+
+/* Number of address a single node_t can have. Default to the production
+ * value. This is to control the size of the bloom filter. */
+static int addr_per_node = 2;
+static int
+mock_get_estimated_address_per_node(void)
+{
+  return addr_per_node;
+}
+
 static void
 test_bwmgt_token_buf_init(void *arg)
 {
@@ -220,8 +272,167 @@ test_bwmgt_token_buf_helpers(void *arg)
   ;
 }
 
+static void
+test_bwmgt_dir_conn_global_write_low(void *arg)
+{
+  bool ret;
+  int addr_family;
+  connection_t *conn = NULL;
+  routerstatus_t *rs = NULL; microdesc_t *md = NULL; routerinfo_t *ri = NULL;
+  tor_addr_t relay_addr;
+  dirauth_options_t *dirauth_opts = NULL;
+
+  (void) arg;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+  MOCK(get_estimated_address_per_node,
+       mock_get_estimated_address_per_node);
+
+  /*
+   * The following is rather complex but that is what it takes to add a dummy
+   * consensus with a valid routerlist which will populate our node address
+   * set that we need to lookup to test the known relay code path.
+   *
+   * We MUST do that before we MOCK(get_options) else it is another world of
+   * complexity.
+   */
+
+  /* This will be the address of our relay. */
+  tor_addr_parse(&relay_addr, "1.2.3.4");
+
+  /* We'll now add a relay into our routerlist and see if we let it. */
+  dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
+  dummy_ns->flavor = FLAV_MICRODESC;
+  dummy_ns->routerstatus_list = smartlist_new();
+
+  md = tor_malloc_zero(sizeof(*md));
+  ri = tor_malloc_zero(sizeof(*ri));
+  rs = tor_malloc_zero(sizeof(*rs));
+  crypto_rand(rs->identity_digest, sizeof(rs->identity_digest));
+  crypto_rand(md->digest, sizeof(md->digest));
+  memcpy(rs->descriptor_digest, md->digest, DIGEST256_LEN);
+
+  /* Set IP address. */
+  rs->addr = tor_addr_to_ipv4h(&relay_addr);
+  ri->addr = rs->addr;
+  /* Add the rs to the consensus becoming a node_t. */
+  smartlist_add(dummy_ns->routerstatus_list, rs);
+
+  /* Add all configured authorities (hardcoded) before we set the consensus so
+   * the address set exists. */
+  ret = consider_adding_dir_servers(&mock_options, &mock_options);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* This will make the nodelist bloom filter very large
+   * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
+  addr_per_node = 1024;
+
+  nodelist_set_consensus(dummy_ns);
+
+  dirauth_opts = tor_malloc_zero(sizeof(dirauth_options_t));
+  dirauth_opts->AuthDirRejectRequestsUnderLoad = 0;
+  dirauth_set_options(dirauth_opts);
+
+  /* Ok, now time to control which options we use. */
+  MOCK(get_options, mock_get_options);
+
+  /* Set ourselves as an authoritative dir. */
+  mock_options.AuthoritativeDir = 1;
+  mock_options.V3AuthoritativeDir = 1;
+  mock_options.UseDefaultFallbackDirs = 0;
+
+  /* This will set our global bucket to 1 byte and thus we will hit the
+   * banwdith limit in our test. */
+  mock_options.BandwidthRate = 1;
+  mock_options.BandwidthBurst = 1;
+
+  /* Else an IPv4 address screams. */
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Initialize the global buckets. */
+  connection_bucket_init();
+
+  /* The address "127.0.0.1" is set with this helper. */
+  conn = test_conn_get_connection(DIR_CONN_STATE_MIN_, CONN_TYPE_DIR,
+                                  DIR_PURPOSE_MIN_);
+  tt_assert(conn);
+
+  /* First try a non authority non relay IP thus a client but we are not
+   * configured to reject requests under load so we should get a false value
+   * that our limit is _not_ low. */
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Now, we will reject requests under load so try again a non authority non
+   * relay IP thus a client. We should get a warning that our limit is too
+   * low. */
+  dirauth_opts->AuthDirRejectRequestsUnderLoad = 1;
+
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Now, lets try with a connection address from moria1. It should always
+   * pass even though our limit is too low. */
+  addr_family = tor_addr_parse(&conn->addr, "128.31.0.39");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* IPv6 testing of gabelmoo. */
+  addr_family = tor_addr_parse(&conn->addr, "[2001:638:a000:4140::ffff:189]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a known relay address. It should pass. Possible due to
+   * our consensus setting above. */
+  memcpy(&conn->addr, &relay_addr, sizeof(tor_addr_t));
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a random IP that is not an authority nor a relay. */
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Finally, just make sure it still denies an IP if we are _not_ a v3
+   * directory authority. */
+  mock_options.V3AuthoritativeDir = 0;
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Random IPv6 should not be allowed. */
+  addr_family = tor_addr_parse(&conn->addr, "[CAFE::ACAB]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  connection_free_minimal(conn);
+  routerstatus_free(rs); routerinfo_free(ri); microdesc_free(md);
+  smartlist_clear(dummy_ns->routerstatus_list);
+  networkstatus_vote_free(dummy_ns);
+
+  UNMOCK(get_estimated_address_per_node);
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+  UNMOCK(get_options);
+}
+
 #define BWMGT(name)                                          \
-  { #name, test_bwmgt_ ## name , 0, NULL, NULL }
+  { #name, test_bwmgt_ ## name , TT_FORK, NULL, NULL }
 
 struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_init),
@@ -229,5 +440,7 @@ struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_dec),
   BWMGT(token_buf_refill),
   BWMGT(token_buf_helpers),
+
+  BWMGT(dir_conn_global_write_low),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c
index fc5367557d..f9ff101c98 100644
--- a/src/test/test_cell_formats.c
+++ b/src/test/test_cell_formats.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -713,16 +713,20 @@ test_cfmt_extend_cells(void *arg)
   tt_mem_op(cc->onionskin,OP_EQ, b, 99+20);
   tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
   tt_int_op(p2_cmd, OP_EQ, RELAY_COMMAND_EXTEND2);
-  /* We'll generate it minus the IPv6 address and minus the konami code */
-  tt_int_op(p2_len, OP_EQ, 89+99-34-20);
+  /* We'll generate it minus the konami code */
+  tt_int_op(p2_len, OP_EQ, 89+99-34);
   test_memeq_hex(p2,
-                 /* Two items: one that same darn IP address. */
-                 "02000612F40001F0F1"
-                 /* The next is a digest : anthropomorphization */
-                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 /* Three items */
+                 "03"
+                 /* IPv4 address */
+                 "0006" "12F40001" "F0F1"
+                 /* The next is an RSA digest: anthropomorphization */
+                 "0214" "616e7468726f706f6d6f727068697a6174696f6e"
+                 /*IPv6 address */
+                 "0112" "20020000000000000000000000f0c51e" "1112"
                  /* Now the handshake prologue */
                  "01050063");
-  tt_mem_op(p2+1+8+22+4,OP_EQ, b, 99+20);
+  tt_mem_op(p2+1+8+22+20+4, OP_EQ, b, 99+20);
   tt_int_op(0, OP_EQ, create_cell_format_relayed(&cell, cc));
 
   /* Now let's add an ed25519 key to that extend2 cell. */
@@ -732,22 +736,31 @@ test_cfmt_extend_cells(void *arg)
   /* As before, since we aren't extending by ed25519. */
   get_options_mutable()->ExtendByEd25519ID = 0;
   tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
-  tt_int_op(p2_len, OP_EQ, 89+99-34-20);
+  tt_int_op(p2_len, OP_EQ, 89+99-34);
   test_memeq_hex(p2,
-                 "02000612F40001F0F1"
+                 "03"
+                 "000612F40001F0F1"
                  "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 "011220020000000000000000000000f0c51e1112"
                  "01050063");
 
   /* Now try with the ed25519 ID. */
   get_options_mutable()->ExtendByEd25519ID = 1;
   tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
-  tt_int_op(p2_len, OP_EQ, 89+99-34-20 + 34);
+  tt_int_op(p2_len, OP_EQ, 89+99);
   test_memeq_hex(p2,
-                 "03000612F40001F0F1"
+                 /* Four items */
+                 "04"
+                 /* IPv4 address */
+                 "0006" "12F40001" "F0F1"
+                 /* The next is an RSA digest: anthropomorphization */
                  "0214616e7468726f706f6d6f727068697a6174696f6e"
-                 // ed digest follows:
+                 /* Then an ed public key: brownshoesdontmakeit/brownshoesd */
                  "0320" "62726f776e73686f6573646f6e746d616b656"
                         "9742f62726f776e73686f657364"
+                 /*IPv6 address */
+                 "0112" "20020000000000000000000000f0c51e" "1112"
+                 /* Now the handshake prologue */
                  "01050063");
   /* Can we parse that? Did the key come through right? */
   memset(&ec, 0, sizeof(ec));
@@ -756,6 +769,40 @@ test_cfmt_extend_cells(void *arg)
   tt_mem_op("brownshoesdontmakeit/brownshoesd", OP_EQ,
             ec.ed_pubkey.pubkey, 32);
 
+  /* Now try IPv6 without IPv4 */
+  memset(p, 0, sizeof(p));
+  memcpy(p, "\x02", 1);
+  memcpy(p+1, "\x02\x14" "anthropomorphization", 22);
+  memcpy(p+23, "\x01\x12" "xxxxxxxxxxxxxxxxYY", 20);
+  memcpy(p+43, "\xff\xff\x00\x20", 4);
+  tt_int_op(0, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
+                                         p, sizeof(p)));
+  tt_int_op(RELAY_COMMAND_EXTEND2, OP_EQ, ec.cell_type);
+  tt_assert(fast_mem_is_zero((const char *)&ec.orport_ipv4.addr,
+                             sizeof(tor_addr_t)));
+  tt_int_op(0, OP_EQ, ec.orport_ipv4.port);
+  tt_str_op("7878:7878:7878:7878:7878:7878:7878:7878",
+            OP_EQ, fmt_addr(&ec.orport_ipv6.addr));
+  tt_int_op(22873, OP_EQ, ec.orport_ipv6.port);
+  tt_assert(ed25519_public_key_is_zero(&ec.ed_pubkey));
+  tt_mem_op(ec.node_id,OP_EQ, "anthropomorphization", 20);
+  tt_int_op(cc->cell_type, OP_EQ, CELL_CREATE2);
+  tt_int_op(cc->handshake_type, OP_EQ, 0xffff);
+  tt_int_op(cc->handshake_len, OP_EQ, 32);
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_cmd, OP_EQ, RELAY_COMMAND_EXTEND2);
+  tt_int_op(p2_len, OP_EQ, 47+32);
+  test_memeq_hex(p2,
+                 /* Two items */
+                 "02"
+                 /* The next is an RSA digest: anthropomorphization */
+                 "0214" "616e7468726f706f6d6f727068697a6174696f6e"
+                 /*IPv6 address */
+                 "0112" "78787878787878787878787878787878" "5959"
+                 /* Now the handshake prologue */
+                 "ffff0020");
+  tt_int_op(0, OP_EQ, create_cell_format_relayed(&cell, cc));
+
   /* == Now try parsing some junk */
 
   /* Try a too-long handshake */
@@ -811,13 +858,6 @@ test_cfmt_extend_cells(void *arg)
   memcpy(p+48, "\xff\xff\x00\x20", 4);
   tt_int_op(-1, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
                                       p, sizeof(p)));
-  memset(p, 0, sizeof(p));
-  memcpy(p, "\x02", 1);
-  memcpy(p+1, "\x02\x14" "anarchoindividualist", 22);
-  memcpy(p+23, "\x01\x12" "xxxxxxxxxxxxxxxxYY", 18);
-  memcpy(p+41, "\xff\xff\x00\x20", 4);
-  tt_int_op(-1, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
-                                      p, sizeof(p)));
 
   /* Running out of space in specifiers  */
   memset(p,0,sizeof(p));
diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c
index 8fc1da031e..b778c07802 100644
--- a/src/test/test_cell_queue.c
+++ b/src/test/test_cell_queue.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITLIST_PRIVATE
diff --git a/src/test/test_channel.c b/src/test/test_channel.c
index afb7db813c..83b69cc80b 100644
--- a/src/test/test_channel.c
+++ b/src/test/test_channel.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
-#define CHANNEL_PRIVATE_
+#define CHANNEL_OBJECT_PRIVATE
+#define CHANNEL_FILE_PRIVATE
 #include "core/or/or.h"
 #include "core/or/channel.h"
 /* For channel_note_destroy_not_pending */
@@ -34,8 +34,6 @@
 static int test_chan_accept_cells = 0;
 static int test_chan_fixed_cells_recved = 0;
 static cell_t * test_chan_last_seen_fixed_cell_ptr = NULL;
-static int test_chan_var_cells_recved = 0;
-static var_cell_t * test_chan_last_seen_var_cell_ptr = NULL;
 static int test_cells_written = 0;
 static int test_doesnt_want_writes_count = 0;
 static int test_dumpstats_calls = 0;
@@ -112,24 +110,6 @@ chan_test_dumpstats(channel_t *ch, int severity)
   return;
 }
 
-/*
- * Handle an incoming variable-size cell for unit tests
- */
-
-static void
-chan_test_var_cell_handler(channel_t *ch,
-                           var_cell_t *var_cell)
-{
-  tt_assert(ch);
-  tt_assert(var_cell);
-
-  test_chan_last_seen_var_cell_ptr = var_cell;
-  ++test_chan_var_cells_recved;
-
- done:
-  return;
-}
-
 static void
 chan_test_close(channel_t *ch)
 {
@@ -487,11 +467,8 @@ test_channel_dumpstats(void *arg)
 
   /* Receive path */
   channel_set_cell_handlers(ch,
-                            chan_test_cell_handler,
-                            chan_test_var_cell_handler);
+                            chan_test_cell_handler);
   tt_ptr_op(channel_get_cell_handler(ch), OP_EQ, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), OP_EQ,
-            chan_test_var_cell_handler);
   cell = tor_malloc_zero(sizeof(*cell));
   old_count = test_chan_fixed_cells_recved;
   channel_process_cell(ch, cell);
@@ -593,7 +570,6 @@ test_channel_outbound_cell(void *arg)
   circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan);
   tt_int_op(channel_num_circuits(chan), OP_EQ, 1);
   /* Test the cmux state. */
-  tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux);
   tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
             OP_EQ, 1);
 
@@ -718,7 +694,7 @@ test_channel_inbound_cell(void *arg)
 
   /* Setup incoming cell handlers. We don't care about var cell, the channel
    * layers is not handling those. */
-  channel_set_cell_handlers(chan, chan_test_cell_handler, NULL);
+  channel_set_cell_handlers(chan, chan_test_cell_handler);
   tt_ptr_op(chan->cell_handler, OP_EQ, chan_test_cell_handler);
   /* Now process the cell, we should see it. */
   old_count = test_chan_fixed_cells_recved;
@@ -1345,7 +1321,7 @@ test_channel_for_extend(void *arg)
   channel_t *ret_chan = NULL;
   char digest[DIGEST_LEN];
   ed25519_public_key_t ed_id;
-  tor_addr_t addr;
+  tor_addr_t ipv4_addr, ipv6_addr;
   const char *msg;
   int launch;
   time_t now = time(NULL);
@@ -1355,6 +1331,9 @@ test_channel_for_extend(void *arg)
   memset(digest, 'A', sizeof(digest));
   memset(&ed_id, 'B', sizeof(ed_id));
 
+  tor_addr_make_null(&ipv4_addr, AF_INET);
+  tor_addr_make_null(&ipv6_addr, AF_INET6);
+
   chan1 = new_fake_channel();
   tt_assert(chan1);
   /* Need to be registered to get added to the id map. */
@@ -1388,7 +1367,8 @@ test_channel_for_extend(void *arg)
   tt_ptr_op(channel_find_by_remote_identity(digest, &ed_id), OP_EQ, chan1);
 
   /* The expected result is chan2 because it is older than chan1. */
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1396,16 +1376,18 @@ test_channel_for_extend(void *arg)
 
   /* Switch that around from previous test. */
   chan2->timestamp_created = chan1->timestamp_created + 1;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan1);
   tt_int_op(launch, OP_EQ, 0);
   tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
 
   /* Same creation time, num circuits will be used and they both have 0 so the
-   * channel 2 should be picked due to how channel_is_better() work. */
+   * channel 2 should be picked due to how channel_is_better() works. */
   chan2->timestamp_created = chan1->timestamp_created;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan1);
   tt_int_op(launch, OP_EQ, 0);
@@ -1416,7 +1398,8 @@ test_channel_for_extend(void *arg)
 
   /* Condemned the older channel. */
   chan1->state = CHANNEL_STATE_CLOSING;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1425,7 +1408,8 @@ test_channel_for_extend(void *arg)
 
   /* Make the older channel a client one. */
   channel_mark_client(chan1);
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1435,8 +1419,9 @@ test_channel_for_extend(void *arg)
   /* Non matching ed identity with valid digest. */
   ed25519_public_key_t dumb_ed_id;
   memset(&dumb_ed_id, 0, sizeof(dumb_ed_id));
-  ret_chan = channel_get_for_extend(digest, &dumb_ed_id, &addr, &msg,
-                                    &launch);
+  ret_chan = channel_get_for_extend(digest, &dumb_ed_id,
+                                    &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Not connected. Connecting.");
   tt_int_op(launch, OP_EQ, 1);
@@ -1445,7 +1430,8 @@ test_channel_for_extend(void *arg)
   test_chan_should_match_target = 1;
   chan1->state = CHANNEL_STATE_OPENING;
   chan2->state = CHANNEL_STATE_OPENING;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Connection in progress; waiting.");
   tt_int_op(launch, OP_EQ, 0);
@@ -1454,7 +1440,8 @@ test_channel_for_extend(void *arg)
 
   /* Mark channel 1 as bad for circuits. */
   channel_mark_bad_for_new_circs(chan1);
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1464,7 +1451,8 @@ test_channel_for_extend(void *arg)
   /* Mark both channels as unusable. */
   channel_mark_bad_for_new_circs(chan1);
   channel_mark_bad_for_new_circs(chan2);
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
                         " Launching a new one.");
@@ -1475,7 +1463,8 @@ test_channel_for_extend(void *arg)
   /* Non canonical channels. */
   test_chan_should_be_canonical = 0;
   test_chan_should_match_target = 0;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
                         " Launching a new one.");
@@ -1539,6 +1528,10 @@ test_channel_listener(void *arg)
   channel_listener_dump_statistics(chan, LOG_INFO);
 
  done:
+  if (chan) {
+    channel_listener_unregister(chan);
+    tor_free(chan);
+  }
   channel_free_all();
 }
 
diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c
index 5da2d81377..63a591583d 100644
--- a/src/test/test_channelpadding.c
+++ b/src/test/test_channelpadding.c
@@ -1,7 +1,7 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define MAINLOOP_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define TOR_TIMERS_PRIVATE
@@ -21,7 +21,7 @@
 #include "test/log_test_helpers.h"
 #include "lib/tls/tortls.h"
 #include "lib/evloop/timers.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "core/or/cell_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -289,8 +289,6 @@ test_channelpadding_timers(void *arg)
   channel_t *chans[CHANNELS_TO_TEST];
   (void)arg;
 
-  tor_libevent_postfork();
-
   if (!connection_array)
     connection_array = smartlist_new();
 
@@ -393,7 +391,6 @@ test_channelpadding_killonehop(void *arg)
   channelpadding_decision_t decision;
   int64_t new_time;
   (void)arg;
-  tor_libevent_postfork();
 
   routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
   monotime_init();
@@ -502,8 +499,6 @@ test_channelpadding_consensus(void *arg)
   int64_t new_time;
   (void)arg;
 
-  tor_libevent_postfork();
-
   /*
    * Params tested:
    *   nf_pad_before_usage
@@ -898,8 +893,6 @@ test_channelpadding_decide_to_pad_channel(void *arg)
     connection_array = smartlist_new();
   (void)arg;
 
-  tor_libevent_postfork();
-
   monotime_init();
   monotime_enable_test_mocking();
   monotime_set_mock_time_nsec(1);
diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c
index 10513e451d..f4f5cb447e 100644
--- a/src/test/test_channeltls.c
+++ b/src/test/test_channeltls.c
@@ -1,19 +1,20 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #include <math.h>
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/or.h"
 #include "lib/net/address.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 /* For init/free stuff */
 #include "core/or/scheduler.h"
 #include "lib/tls/tortls.h"
diff --git a/src/test/test_checkdir.c b/src/test/test_checkdir.c
index 1df74c390a..186a55cc8c 100644
--- a/src/test/test_checkdir.c
+++ b/src/test/test_checkdir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c
index 538f20781f..03fd176ead 100644
--- a/src/test/test_circuitbuild.c
+++ b/src/test/test_circuitbuild.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITBUILD_PRIVATE
@@ -8,18 +8,32 @@
 #define ENTRYNODES_PRIVATE
 
 #include "core/or/or.h"
+
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+
+#define CONFIG_PRIVATE
 #include "app/config/config.h"
+
+#include "core/or/channel.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
+#include "core/or/onion.h"
 
+#include "core/or/cell_st.h"
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/extend_info_st.h"
 #include "core/or/origin_circuit_st.h"
+#include "core/or/or_circuit_st.h"
 
 #include "feature/client/entrynodes.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/circuitbuild_relay.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+
+#include "feature/nodelist/node_st.h"
 
 /* Dummy nodes smartlist for testing */
 static smartlist_t dummy_nodes;
@@ -27,11 +41,11 @@ static smartlist_t dummy_nodes;
 static extend_info_t dummy_ei;
 
 static int
-mock_count_acceptable_nodes(smartlist_t *nodes)
+mock_count_acceptable_nodes(const smartlist_t *nodes, int direct)
 {
   (void)nodes;
 
-  return DEFAULT_ROUTE_LEN + 1;
+  return direct ? 1 : DEFAULT_ROUTE_LEN + 1;
 }
 
 /* Test route lengths when the caller of new_route_len() doesn't
@@ -114,6 +128,14 @@ test_new_route_len_unhandled_exit(void *arg)
   int r;
 
   (void)arg;
+#ifdef ALL_BUGS_ARE_FATAL
+  /* Coverity (and maybe clang analyser) complain that the code following
+   * tt_skip() is unconditionally unreachable. */
+#if !defined(__COVERITY__) && !defined(__clang_analyzer__)
+  tt_skip();
+#endif
+#endif /* defined(ALL_BUGS_ARE_FATAL) */
+
   MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
 
   tor_capture_bugs_(1);
@@ -125,10 +147,10 @@ test_new_route_len_unhandled_exit(void *arg)
             "!(exit_ei && !known_purpose)");
   expect_single_log_msg_containing("Unhandled purpose");
   expect_single_log_msg_containing("with a chosen exit; assuming routelen");
-  teardown_capture_of_logs();
-  tor_end_capture_bugs_();
 
  done:
+  teardown_capture_of_logs();
+  tor_end_capture_bugs_();
   UNMOCK(count_acceptable_nodes);
 }
 
@@ -167,16 +189,1366 @@ test_upgrade_from_guard_wait(void *arg)
   tt_assert(!list);
 
  done:
+  smartlist_free(list);
   circuit_free(circ);
   entry_guard_free_(guard);
 }
 
+static int server = 0;
+static int
+mock_server_mode(const or_options_t *options)
+{
+  (void)options;
+  return server;
+}
+
+/* Test the different cases in circuit_extend_state_valid_helper(). */
+static void
+test_circuit_extend_state_valid(void *arg)
+{
+  (void)arg;
+  circuit_t *circ = tor_malloc_zero(sizeof(circuit_t));
+
+  server = 0;
+  MOCK(server_mode, mock_server_mode);
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+  /* Clients can't extend */
+  server = 0;
+  tt_int_op(circuit_extend_state_valid_helper(NULL), OP_EQ, -1);
+  expect_log_msg("Got an extend cell, but running as a client. Closing.\n");
+  mock_clean_saved_logs();
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* n_chan and n_hop are NULL, this should succeed */
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  /* But clients still can't extend */
+  server = 0;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, -1);
+  expect_log_msg("Got an extend cell, but running as a client. Closing.\n");
+  mock_clean_saved_logs();
+
+  /* n_chan must be NULL */
+  circ->n_chan = tor_malloc_zero(sizeof(channel_t));
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, -1);
+  expect_log_msg("n_chan already set. Bug/attack. Closing.\n");
+  mock_clean_saved_logs();
+  tor_free(circ->n_chan);
+
+  /* n_hop must be NULL */
+  circ->n_hop = tor_malloc_zero(sizeof(extend_info_t));
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, -1);
+  expect_log_msg("conn to next hop already launched. Bug/attack. Closing.\n");
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(server_mode);
+  server = 0;
+
+  tor_free(circ->n_chan);
+  tor_free(circ->n_hop);
+  tor_free(circ);
+}
+
+static node_t *mocked_node = NULL;
+static const node_t *
+mock_node_get_by_id(const char *identity_digest)
+{
+  (void)identity_digest;
+  return mocked_node;
+}
+
+static int mocked_supports_ed25519_link_authentication = 0;
+static int
+mock_node_supports_ed25519_link_authentication(const node_t *node,
+                                                int compatible_with_us)
+{
+  (void)node;
+  (void)compatible_with_us;
+  return mocked_supports_ed25519_link_authentication;
+}
+
+static ed25519_public_key_t * mocked_ed25519_id = NULL;
+static const ed25519_public_key_t *
+mock_node_get_ed25519_id(const node_t *node)
+{
+  (void)node;
+  return mocked_ed25519_id;
+}
+
+/* Test the different cases in circuit_extend_add_ed25519_helper(). */
+static void
+test_circuit_extend_add_ed25519(void *arg)
+{
+  (void)arg;
+  extend_cell_t *ec = tor_malloc_zero(sizeof(extend_cell_t));
+  extend_cell_t *old_ec = tor_malloc_zero(sizeof(extend_cell_t));
+  extend_cell_t *zero_ec = tor_malloc_zero(sizeof(extend_cell_t));
+
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  ed25519_public_key_t *fake_ed25519_id = NULL;
+  fake_ed25519_id = tor_malloc_zero(sizeof(ed25519_public_key_t));
+
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  MOCK(node_supports_ed25519_link_authentication,
+       mock_node_supports_ed25519_link_authentication);
+  MOCK(node_get_ed25519_id, mock_node_get_ed25519_id);
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* The extend cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_add_ed25519_helper(NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!ec))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* The node id must be non-zero */
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, -1);
+  expect_log_msg(
+    "Client asked me to extend without specifying an id_digest.\n");
+  /* And nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  mock_clean_saved_logs();
+
+  /* Fill in fake node_id, and try again */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* There's no node with that id, so the ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  mock_clean_saved_logs();
+
+  /* Provide 2 out of 3 of node, supports link auth, and ed_id.
+   * The ed_id should remain zeroed. */
+
+  /* Provide node and supports link auth */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  mocked_supports_ed25519_link_authentication = 1;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Provide supports link auth and ed id */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_supports_ed25519_link_authentication = 1;
+  memset(fake_ed25519_id, 0xEE, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Provide node and ed id */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  memset(fake_ed25519_id, 0xEE, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Now do the real lookup */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  mocked_supports_ed25519_link_authentication = 1;
+  memset(fake_ed25519_id, 0xEE, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should match */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, fake_ed25519_id, sizeof(ec->ed_pubkey));
+  /* Nothing else should have changed */
+  memcpy(&ec->ed_pubkey, &old_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Now do the real lookup, but with a zeroed ed id */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  mocked_supports_ed25519_link_authentication = 1;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should match */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, fake_ed25519_id, sizeof(ec->ed_pubkey));
+  /* Nothing else should have changed */
+  memcpy(&ec->ed_pubkey, &old_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+ done:
+  UNMOCK(node_get_by_id);
+  UNMOCK(node_supports_ed25519_link_authentication);
+  UNMOCK(node_get_ed25519_id);
+
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  tor_free(ec);
+  tor_free(old_ec);
+  tor_free(zero_ec);
+
+  tor_free(fake_ed25519_id);
+  tor_free(fake_node);
+}
+
+static or_options_t *mocked_options = NULL;
+static const or_options_t *
+mock_get_options(void)
+{
+  return mocked_options;
+}
+
+#define PUBLIC_IPV4   "1.2.3.4"
+#define INTERNAL_IPV4 "0.0.0.1"
+
+#define PUBLIC_IPV6   "1234::cdef"
+#define INTERNAL_IPV6 "::1"
+
+#define VALID_PORT    0x1234
+
+/* Test the different cases in circuit_extend_lspec_valid_helper(). */
+static void
+test_circuit_extend_lspec_valid(void *arg)
+{
+  (void)arg;
+  extend_cell_t *ec = tor_malloc_zero(sizeof(extend_cell_t));
+  channel_t *p_chan = tor_malloc_zero(sizeof(channel_t));
+  or_circuit_t *or_circ = tor_malloc_zero(sizeof(or_circuit_t));
+  circuit_t *circ = TO_CIRCUIT(or_circ);
+
+  or_options_t *fake_options = options_new();
+  MOCK(get_options, mock_get_options);
+  mocked_options = fake_options;
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Extend cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(NULL, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!ec))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell and circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(NULL, NULL), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), we might not log any bugs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 2);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* IPv4 and IPv6 addr and port are all zero, this should fail */
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or unspecified address '[scrubbed]'.\n");
+  mock_clean_saved_logs();
+
+  /* Now ask for the actual address in the logs */
+  fake_options->SafeLogging_ = SAFELOG_SCRUB_NONE;
+
+  /* IPv4 port is 0, IPv6 addr and port are both zero, this should fail */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv4 address '1.2.3.4:0'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv4 addr is 0, IPv6 addr and port are both zero, this should fail */
+  ec->orport_ipv4.port = VALID_PORT;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv4 address '0.0.0.0:4660'.\n");
+  mock_clean_saved_logs();
+  ec->orport_ipv4.port = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv4 addr is internal, and port is valid.
+   * (IPv6 addr and port are both zero.)
+   * Result depends on ExtendAllowPrivateAddresses. */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv4 address '0.0.0.1'.\n");
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Now do the same tests, but for IPv6 */
+
+  /* IPv6 port is 0, IPv4 addr and port are both zero, this should fail */
+  tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv6 address '[1234::cdef]:0'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv6 addr is 0, IPv4 addr and port are both zero, this should fail */
+  ec->orport_ipv6.port = VALID_PORT;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv6 address '[::]:4660'.\n");
+  mock_clean_saved_logs();
+  ec->orport_ipv4.port = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv6 addr is internal, and port is valid.
+   * (IPv4 addr and port are both zero.)
+   * Result depends on ExtendAllowPrivateAddresses. */
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv6 address '[::1]'.\n");
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Both addresses are internal.
+   * Result depends on ExtendAllowPrivateAddresses. */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv4 address '0.0.0.1'.\n");
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv6 address '[::1]'.\n");
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* If we pass the private address check, but don't have the right
+   * OR circuit magic number, we trigger another bug */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 1;
+
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(circ->magic != 0x98ABC04Fu))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Fail again, but this time only set an IPv4 address. */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 1;
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), expect 0-1 bug logs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 1);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Now set the right magic */
+  or_circ->base_.magic = OR_CIRCUIT_MAGIC;
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* If we pass the OR circuit magic check, but don't have p_chan,
+   * we trigger another bug */
+  fake_options->ExtendAllowPrivateAddresses = 1;
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!p_chan))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  /* We can also pass the OR circuit magic check with a public address */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), expect 0-1 bug logs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 1);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  tor_addr_make_null(&ec->orport_ipv4.addr, AF_INET);
+  ec->orport_ipv4.port = 0x0000;
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Now let's fake a p_chan and the addresses */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  or_circ->p_chan = p_chan;
+
+  /* This is a trivial failure: node_id and p_chan->identity_digest are both
+   * zeroed */
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend back to the previous hop.\n");
+  mock_clean_saved_logs();
+
+  /* Let's check with non-zero identities as well */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0xAA, sizeof(p_chan->identity_digest));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend back to the previous hop.\n");
+  mock_clean_saved_logs();
+
+  memset(ec->node_id, 0, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0, sizeof(p_chan->identity_digest));
+
+  /* Let's pass the node_id test */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0xBB, sizeof(p_chan->identity_digest));
+
+  /* ed_pubkey is zero, and that's allowed, so we should succeed */
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  /* Now let's check that we warn, but succeed, when only one address is
+   * private */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv4 address '0.0.0.1'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Now with private IPv6 */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv6 address '[::1]'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Now reset to public IPv4 and IPv6 */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+
+  /* Fail on matching non-zero identities */
+  memset(&ec->ed_pubkey, 0xEE, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0xEE, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend back to the previous hop "
+                 "(by Ed25519 ID).\n");
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Succeed on different, non-zero identities */
+  memset(&ec->ed_pubkey, 0xDD, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0xEE, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Succeed if the client knows the identity, but we don't */
+  memset(&ec->ed_pubkey, 0xDD, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0x00, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Succeed if we know the identity, but the client doesn't */
+  memset(&ec->ed_pubkey, 0x00, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0xEE, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Cleanup the node ids */
+  memset(ec->node_id, 0, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0, sizeof(p_chan->identity_digest));
+
+  /* Cleanup the p_chan and the addresses */
+  tor_addr_make_null(&ec->orport_ipv4.addr, AF_UNSPEC);
+  ec->orport_ipv4.port = 0;
+  or_circ->p_chan = NULL;
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(get_options);
+  or_options_free(fake_options);
+  mocked_options = NULL;
+
+  tor_free(ec);
+  tor_free(or_circ);
+  tor_free(p_chan);
+}
+
+static bool can_extend_over_ipv6_result = false;
+static int mock_router_can_extend_over_ipv6_calls = 0;
+static bool
+mock_router_can_extend_over_ipv6(const or_options_t *options)
+{
+  (void)options;
+  mock_router_can_extend_over_ipv6_calls++;
+  return can_extend_over_ipv6_result;
+}
+
+/* Test the different cases in circuit_choose_ip_ap_for_extend(). */
+static void
+test_circuit_choose_ip_ap_for_extend(void *arg)
+{
+  (void)arg;
+  tor_addr_port_t ipv4_ap;
+  tor_addr_port_t ipv6_ap;
+
+  /* Set up valid addresses */
+  tor_addr_parse(&ipv4_ap.addr, PUBLIC_IPV4);
+  ipv4_ap.port = VALID_PORT;
+  tor_addr_parse(&ipv6_ap.addr, PUBLIC_IPV6);
+  ipv6_ap.port = VALID_PORT;
+
+  or_options_t *fake_options = options_new();
+  MOCK(get_options, mock_get_options);
+  mocked_options = fake_options;
+
+  MOCK(router_can_extend_over_ipv6,
+       mock_router_can_extend_over_ipv6);
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+
+  /* No valid addresses */
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, NULL), OP_EQ, NULL);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, NULL), OP_EQ, NULL);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  /* One valid address: IPv4 */
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(&ipv4_ap, NULL), OP_EQ, &ipv4_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(&ipv4_ap, NULL), OP_EQ, &ipv4_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  /* One valid address: IPv6 */
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, &ipv6_ap), OP_EQ, &ipv6_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, &ipv6_ap), OP_EQ, NULL);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  /* Two valid addresses */
+  const tor_addr_port_t *chosen_addr = NULL;
+
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  chosen_addr = circuit_choose_ip_ap_for_extend(&ipv4_ap, &ipv6_ap);
+  tt_assert(chosen_addr == &ipv4_ap || chosen_addr == &ipv6_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(&ipv4_ap, &ipv6_ap),
+            OP_EQ, &ipv4_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+ done:
+  UNMOCK(get_options);
+  or_options_free(fake_options);
+  mocked_options = NULL;
+
+  UNMOCK(router_can_extend_over_ipv6);
+
+  tor_free(fake_options);
+}
+
+static int mock_circuit_close_calls = 0;
+static void
+mock_circuit_mark_for_close_(circuit_t *circ, int reason,
+                             int line, const char *cfile)
+{
+  (void)circ;
+  (void)reason;
+  (void)line;
+  (void)cfile;
+  mock_circuit_close_calls++;
+}
+
+static int mock_channel_connect_calls = 0;
+static channel_t *mock_channel_connect_nchan = NULL;
+static channel_t *
+mock_channel_connect_for_circuit(const tor_addr_t *addr,
+                                 uint16_t port,
+                                 const char *id_digest,
+                                 const struct ed25519_public_key_t *ed_id)
+{
+  (void)addr;
+  (void)port;
+  (void)id_digest;
+  (void)ed_id;
+  mock_channel_connect_calls++;
+  return mock_channel_connect_nchan;
+}
+
+/* Test the different cases in circuit_open_connection_for_extend().
+ * Chooses different IP addresses depending on the first character in arg:
+ *  - 4: IPv4
+ *  - 6: IPv6
+ *  - d: IPv4 and IPv6 (dual-stack)
+ */
+static void
+test_circuit_open_connection_for_extend(void *arg)
+{
+  const char ip_version = ((const char *)arg)[0];
+  const bool use_ipv4 = (ip_version == '4' || ip_version == 'd');
+  const bool use_ipv6 = (ip_version == '6' || ip_version == 'd');
+  tor_assert(use_ipv4 || use_ipv6);
+
+  extend_cell_t *ec = tor_malloc_zero(sizeof(extend_cell_t));
+  circuit_t *circ = tor_malloc_zero(sizeof(circuit_t));
+  channel_t *fake_n_chan = tor_malloc_zero(sizeof(channel_t));
+
+  or_options_t *fake_options = options_new();
+  MOCK(get_options, mock_get_options);
+  mocked_options = fake_options;
+
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close_);
+  mock_circuit_close_calls = 0;
+  MOCK(channel_connect_for_circuit, mock_channel_connect_for_circuit);
+  mock_channel_connect_calls = 0;
+  mock_channel_connect_nchan = NULL;
+
+  MOCK(router_can_extend_over_ipv6,
+       mock_router_can_extend_over_ipv6);
+  can_extend_over_ipv6_result = true;
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(ec, NULL, 0);
+  /* We can't close a NULL circuit */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell must be non-NULL */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(NULL, circ, 0);
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!ec))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell and circuit must be non-NULL */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(NULL, NULL, 0);
+  /* We can't close a NULL circuit */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  /* Since we're using IF_BUG_ONCE(), we might not log any bugs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 2);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Fail, because neither address is valid */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(ec, circ, 0);
+  /* Close the circuit, don't connect */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_EQ, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_EQ, NULL);
+  tt_int_op(circ->state, OP_EQ, 0);
+  /* Cleanup */
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Set up valid addresses */
+  if (use_ipv4) {
+    tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+    ec->orport_ipv4.port = VALID_PORT;
+  }
+  if (use_ipv6) {
+    tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+    ec->orport_ipv6.port = VALID_PORT;
+  }
+
+  /* Succeed, but don't try to open a connection */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  circuit_open_connection_for_extend(ec, circ, 0);
+  /* If we haven't closed the circuit, that's success */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_NE, NULL);
+  tt_int_op(circ->state, OP_EQ, CIRCUIT_STATE_CHAN_WAIT);
+  /* Cleanup */
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  circ->state = 0;
+
+  /* Try to open a connection, but fail with a NULL n_chan */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  circuit_open_connection_for_extend(ec, circ, 1);
+  /* Try to connect, but fail, and close the circuit */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 1);
+  expect_log_msg("Launching n_chan failed. Closing circuit.\n");
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_NE, NULL);
+  tt_int_op(circ->state, OP_EQ, CIRCUIT_STATE_CHAN_WAIT);
+  /* Cleanup */
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  circ->state = 0;
+
+  /* Try to open a connection, and succeed, because n_chan is not NULL */
+  mock_channel_connect_nchan = fake_n_chan;
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  circuit_open_connection_for_extend(ec, circ, 1);
+  /* Connection attempt succeeded, leaving the circuit open */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 1);
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_NE, NULL);
+  tt_int_op(circ->state, OP_EQ, CIRCUIT_STATE_CHAN_WAIT);
+  /* Cleanup */
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  circ->state = 0;
+  mock_channel_connect_nchan = NULL;
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(circuit_mark_for_close_);
+  mock_circuit_close_calls = 0;
+  UNMOCK(channel_connect_for_circuit);
+  mock_channel_connect_calls = 0;
+
+  UNMOCK(get_options);
+  or_options_free(fake_options);
+  mocked_options = NULL;
+
+  UNMOCK(router_can_extend_over_ipv6);
+
+  tor_free(ec);
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  tor_free(circ);
+  tor_free(fake_n_chan);
+}
+
+/* Guaranteed to be initialised to zero. */
+static extend_cell_t mock_extend_cell_parse_cell_out;
+static int mock_extend_cell_parse_result = 0;
+static int mock_extend_cell_parse_calls = 0;
+
+static int
+mock_extend_cell_parse(extend_cell_t *cell_out,
+                       const uint8_t command,
+                       const uint8_t *payload_in,
+                       size_t payload_len)
+{
+  (void)command;
+  (void)payload_in;
+  (void)payload_len;
+
+  mock_extend_cell_parse_calls++;
+  memcpy(cell_out, &mock_extend_cell_parse_cell_out,
+         sizeof(extend_cell_t));
+  return mock_extend_cell_parse_result;
+}
+
+static int mock_channel_get_for_extend_calls = 0;
+static int mock_channel_get_for_extend_launch_out = 0;
+static channel_t *mock_channel_get_for_extend_nchan = NULL;
+static channel_t *
+mock_channel_get_for_extend(const char *rsa_id_digest,
+                            const ed25519_public_key_t *ed_id,
+                            const tor_addr_t *target_ipv4_addr,
+                            const tor_addr_t *target_ipv6_addr,
+                            const char **msg_out,
+                            int *launch_out)
+{
+  (void)rsa_id_digest;
+  (void)ed_id;
+  (void)target_ipv4_addr;
+  (void)target_ipv6_addr;
+
+  /* channel_get_for_extend() requires non-NULL arguments */
+  tt_ptr_op(msg_out, OP_NE, NULL);
+  tt_ptr_op(launch_out, OP_NE, NULL);
+
+  mock_channel_get_for_extend_calls++;
+  *msg_out = NULL;
+  *launch_out = mock_channel_get_for_extend_launch_out;
+  return mock_channel_get_for_extend_nchan;
+
+ done:
+  return NULL;
+}
+
+static const char *
+mock_channel_get_canonical_remote_descr(channel_t *chan)
+{
+  (void)chan;
+  return "mock_channel_get_canonical_remote_descr()";
+}
+
+static int mock_circuit_deliver_create_cell_calls = 0;
+static int mock_circuit_deliver_create_cell_result = 0;
+static int
+mock_circuit_deliver_create_cell(circuit_t *circ,
+                                 const struct create_cell_t *create_cell,
+                                 int relayed)
+{
+  (void)create_cell;
+
+  /* circuit_deliver_create_cell() requires non-NULL arguments,
+   * but we only check circ and circ->n_chan here. */
+  tt_ptr_op(circ, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan, OP_NE, NULL);
+
+  /* We should only ever get relayed cells from extends */
+  tt_int_op(relayed, OP_EQ, 1);
+
+  mock_circuit_deliver_create_cell_calls++;
+  return mock_circuit_deliver_create_cell_result;
+
+ done:
+  return -1;
+}
+
+/* Test the different cases in circuit_extend(). */
+static void
+test_circuit_extend(void *arg)
+{
+  (void)arg;
+  cell_t *cell = tor_malloc_zero(sizeof(cell_t));
+  channel_t *p_chan = tor_malloc_zero(sizeof(channel_t));
+  or_circuit_t *or_circ = tor_malloc_zero(sizeof(or_circuit_t));
+  circuit_t *circ = TO_CIRCUIT(or_circ);
+  channel_t *fake_n_chan = tor_malloc_zero(sizeof(channel_t));
+
+  server = 0;
+  MOCK(server_mode, mock_server_mode);
+
+  /* Mock a debug function, but otherwise ignore it */
+  MOCK(channel_get_canonical_remote_descr,
+       mock_channel_get_canonical_remote_descr);
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(cell, NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(NULL, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!cell))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell and circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(NULL, NULL), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), we might not log any bugs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 2);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Clients can't extend */
+  server = 0;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  expect_log_msg("Got an extend cell, but running as a client. Closing.\n");
+  mock_clean_saved_logs();
+
+  /* But servers can. Unpack the cell, but fail parsing. */
+  server = 1;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  expect_log_msg("Can't parse extend cell. Closing circuit.\n");
+  mock_clean_saved_logs();
+
+  /* Now mock parsing */
+  MOCK(extend_cell_parse, mock_extend_cell_parse);
+
+  /* And make parsing succeed, but fail on adding ed25519 */
+  memset(&mock_extend_cell_parse_cell_out, 0,
+             sizeof(mock_extend_cell_parse_cell_out));
+  mock_extend_cell_parse_result = 0;
+  mock_extend_cell_parse_calls = 0;
+
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  expect_log_msg(
+    "Client asked me to extend without specifying an id_digest.\n");
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+
+  /* Now add a node_id. Fail the lspec check because IPv4 and port are zero. */
+  memset(&mock_extend_cell_parse_cell_out.node_id, 0xAA,
+             sizeof(mock_extend_cell_parse_cell_out.node_id));
+
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or unspecified address '[scrubbed]'.\n");
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+
+  /* Now add a valid IPv4 and port. Fail the OR circuit magic check. */
+  tor_addr_parse(&mock_extend_cell_parse_cell_out.orport_ipv4.addr,
+                 PUBLIC_IPV4);
+  mock_extend_cell_parse_cell_out.orport_ipv4.port = VALID_PORT;
+
+#ifndef ALL_BUGS_ARE_FATAL
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(circ->magic != 0x98ABC04Fu))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Now add the right magic and a p_chan. */
+  or_circ->base_.magic = OR_CIRCUIT_MAGIC;
+  or_circ->p_chan = p_chan;
+
+  /* Mock channel_get_for_extend(), so it doesn't crash. */
+  mock_channel_get_for_extend_calls = 0;
+  MOCK(channel_get_for_extend, mock_channel_get_for_extend);
+
+  /* Test circuit not established, but don't launch another one */
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = NULL;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, 0);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+
+  /* cleanup */
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+  /* Mock channel_connect_for_circuit(), so we don't crash */
+  mock_channel_connect_calls = 0;
+  MOCK(channel_connect_for_circuit, mock_channel_connect_for_circuit);
+
+  /* Test circuit not established, and successful launch of a channel */
+  mock_channel_get_for_extend_launch_out = 1;
+  mock_channel_get_for_extend_nchan = NULL;
+  mock_channel_connect_nchan = fake_n_chan;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, 0);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 1);
+
+  /* cleanup */
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_connect_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+  /* Mock circuit_deliver_create_cell(), so it doesn't crash */
+  mock_circuit_deliver_create_cell_calls = 0;
+  MOCK(circuit_deliver_create_cell, mock_circuit_deliver_create_cell);
+
+  /* Test circuit established, re-using channel, successful delivery */
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = fake_n_chan;
+  mock_channel_connect_nchan = NULL;
+  mock_circuit_deliver_create_cell_result = 0;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, 0);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(mock_circuit_deliver_create_cell_calls, OP_EQ, 1);
+  tt_ptr_op(circ->n_chan, OP_EQ, fake_n_chan);
+
+  /* cleanup */
+  circ->n_chan = NULL;
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_connect_calls = 0;
+  mock_circuit_deliver_create_cell_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+  /* Test circuit established, re-using channel, failed delivery */
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = fake_n_chan;
+  mock_channel_connect_nchan = NULL;
+  mock_circuit_deliver_create_cell_result = -1;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(mock_circuit_deliver_create_cell_calls, OP_EQ, 1);
+  tt_ptr_op(circ->n_chan, OP_EQ, fake_n_chan);
+
+  /* cleanup */
+  circ->n_chan = NULL;
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_connect_calls = 0;
+  mock_circuit_deliver_create_cell_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(server_mode);
+  server = 0;
+
+  UNMOCK(channel_get_canonical_remote_descr);
+
+  UNMOCK(extend_cell_parse);
+  memset(&mock_extend_cell_parse_cell_out, 0,
+         sizeof(mock_extend_cell_parse_cell_out));
+  mock_extend_cell_parse_result = 0;
+  mock_extend_cell_parse_calls = 0;
+
+  UNMOCK(channel_get_for_extend);
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = NULL;
+
+  UNMOCK(channel_connect_for_circuit);
+  mock_channel_connect_calls = 0;
+  mock_channel_connect_nchan = NULL;
+
+  UNMOCK(circuit_deliver_create_cell);
+  mock_circuit_deliver_create_cell_calls = 0;
+  mock_circuit_deliver_create_cell_result = 0;
+
+  tor_free(cell);
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  tor_free(or_circ);
+  tor_free(p_chan);
+  tor_free(fake_n_chan);
+}
+
+/* Test the different cases in onionskin_answer(). */
+static void
+test_onionskin_answer(void *arg)
+{
+  (void)arg;
+  created_cell_t *created_cell = tor_malloc_zero(sizeof(created_cell_t));
+  or_circuit_t *or_circ = tor_malloc_zero(sizeof(or_circuit_t));
+  char keys[CPATH_KEY_MATERIAL_LEN] = {0};
+  uint8_t rend_circ_nonce[DIGEST_LEN] = {0};
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(NULL, created_cell,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Created cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(or_circ, NULL,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!created_cell))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Keys must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(or_circ, created_cell,
+                             NULL, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!keys))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* The rend circuit nonce must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(or_circ, created_cell,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!rend_circ_nonce))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Also, the keys length must be CPATH_KEY_MATERIAL_LEN, but we can't catch
+   * asserts in unit tests. */
+
+  /* Fail when formatting the created cell */
+  tt_int_op(onionskin_answer(or_circ, created_cell,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  expect_log_msg("couldn't format created cell (type=0, len=0).\n");
+  mock_clean_saved_logs();
+
+  /* TODO: test the rest of onionskin_answer(), see #33860 */
+  /* TODO: mock created_cell_format for the next test  */
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  tor_free(created_cell);
+  tor_free(or_circ);
+}
+
+#define TEST(name, flags, setup, cleanup) \
+  { #name, test_ ## name, flags, setup, cleanup }
+
+#define TEST_NEW_ROUTE_LEN(name, flags) \
+  { #name, test_new_route_len_ ## name, flags, NULL, NULL }
+
+#define TEST_CIRCUIT(name, flags) \
+  { #name, test_circuit_ ## name, flags, NULL, NULL }
+
+#ifndef COCCI
+#define TEST_CIRCUIT_PASSTHROUGH(name, flags, arg) \
+  { #name "/" arg, test_circuit_ ## name, flags, \
+    &passthrough_setup, (void *)(arg) }
+#endif
+
 struct testcase_t circuitbuild_tests[] = {
-  { "noexit", test_new_route_len_noexit, 0, NULL, NULL },
-  { "safe_exit", test_new_route_len_safe_exit, 0, NULL, NULL },
-  { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL },
-  { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL },
-  { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK,
-    NULL, NULL },
+  TEST_NEW_ROUTE_LEN(noexit, 0),
+  TEST_NEW_ROUTE_LEN(safe_exit, 0),
+  TEST_NEW_ROUTE_LEN(unsafe_exit, 0),
+  TEST_NEW_ROUTE_LEN(unhandled_exit, 0),
+
+  TEST(upgrade_from_guard_wait, TT_FORK, &helper_pubsub_setup, NULL),
+
+  TEST_CIRCUIT(extend_state_valid, TT_FORK),
+  TEST_CIRCUIT(extend_add_ed25519, TT_FORK),
+  TEST_CIRCUIT(extend_lspec_valid, TT_FORK),
+  TEST_CIRCUIT(choose_ip_ap_for_extend, 0),
+  TEST_CIRCUIT_PASSTHROUGH(open_connection_for_extend, TT_FORK, "4"),
+  TEST_CIRCUIT_PASSTHROUGH(open_connection_for_extend, TT_FORK, "6"),
+  TEST_CIRCUIT_PASSTHROUGH(open_connection_for_extend, TT_FORK, "dual-stack"),
+  TEST_CIRCUIT(extend, TT_FORK),
+
+  TEST(onionskin_answer, TT_FORK, NULL, NULL),
+
   END_OF_TESTCASES
 };
diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c
index 5cebdbeda0..63c4418f29 100644
--- a/src/test/test_circuitlist.c
+++ b/src/test/test_circuitlist.c
@@ -1,7 +1,7 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
 #define HS_CIRCUITMAP_PRIVATE
@@ -136,7 +136,7 @@ test_clist_maps(void *arg)
   channel_note_destroy_pending(ch2, 200);
   channel_note_destroy_pending(ch2, 205);
   channel_note_destroy_pending(ch1, 100);
-  tt_assert(circuit_id_in_use_on_channel(205, ch2))
+  tt_assert(circuit_id_in_use_on_channel(205, ch2));
   tt_assert(circuit_id_in_use_on_channel(200, ch2));
   tt_assert(circuit_id_in_use_on_channel(100, ch1));
 
diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c
index a2b3e62fe8..2c11d107d0 100644
--- a/src/test/test_circuitmux.c
+++ b/src/test/test_circuitmux.c
@@ -1,33 +1,27 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITMUX_PRIVATE
 #define CIRCUITMUX_EWMA_PRIVATE
 #define RELAY_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
+#include "core/or/destroy_cell_queue_st.h"
 #include "core/or/relay.h"
 #include "core/or/scheduler.h"
-#include "test/test.h"
 
-#include "core/or/destroy_cell_queue_st.h"
+#include "test/fakechans.h"
+#include "test/fakecircs.h"
+#include "test/test.h"
 
 #include <math.h>
 
-/* XXXX duplicated function from test_circuitlist.c */
-static channel_t *
-new_fake_channel(void)
-{
-  channel_t *chan = tor_malloc_zero(sizeof(channel_t));
-  channel_init(chan);
-  return chan;
-}
-
 static int
-has_queued_writes(channel_t *c)
+mock_has_queued_writes_true(channel_t *c)
 {
   (void) c;
   return 1;
@@ -44,16 +38,14 @@ test_cmux_destroy_cell_queue(void *arg)
   packed_cell_t *pc = NULL;
   destroy_cell_t *dc = NULL;
 
-  scheduler_init();
+  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
 
   (void) arg;
 
-  cmux = circuitmux_alloc();
-  tt_assert(cmux);
   ch = new_fake_channel();
-  circuitmux_set_policy(cmux, &ewma_policy);
-  ch->has_queued_writes = has_queued_writes;
+  ch->has_queued_writes = mock_has_queued_writes_true;
   ch->wide_circ_ids = 1;
+  cmux = ch->cmux;
 
   circ = circuitmux_get_first_active_circuit(cmux, &cq);
   tt_ptr_op(circ, OP_EQ, NULL);
@@ -78,10 +70,11 @@ test_cmux_destroy_cell_queue(void *arg)
   tt_int_op(circuitmux_num_cells(cmux), OP_EQ, 2);
 
  done:
-  circuitmux_free(cmux);
-  channel_free(ch);
+  free_fake_channel(ch);
   packed_cell_free(pc);
   tor_free(dc);
+
+  UNMOCK(scheduler_release_channel);
 }
 
 static void
@@ -125,9 +118,363 @@ test_cmux_compute_ticks(void *arg)
   ;
 }
 
+static void
+test_cmux_allocate(void *arg)
+{
+  circuitmux_t *cmux = NULL;
+
+  (void) arg;
+
+  cmux = circuitmux_alloc();
+  tt_assert(cmux);
+  tt_assert(cmux->chanid_circid_map);
+  tt_int_op(HT_SIZE(cmux->chanid_circid_map), OP_EQ, 0);
+  tt_uint_op(cmux->n_circuits, OP_EQ, 0);
+  tt_uint_op(cmux->n_active_circuits, OP_EQ, 0);
+  tt_uint_op(cmux->n_cells, OP_EQ, 0);
+  tt_uint_op(cmux->last_cell_was_destroy, OP_EQ, 0);
+  tt_i64_op(cmux->destroy_ctr, OP_EQ, 0);
+  tt_ptr_op(cmux->policy, OP_EQ, NULL);
+  tt_ptr_op(cmux->policy_data, OP_EQ, NULL);
+
+  tt_assert(TOR_SIMPLEQ_EMPTY(&cmux->destroy_cell_queue.head));
+
+ done:
+  circuitmux_free(cmux);
+}
+
+static void
+test_cmux_attach_circuit(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+  cell_direction_t cdir;
+  unsigned int n_cells;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* While assigning a new circuit IDs, the circuitmux_attach_circuit() is
+   * called for a new channel on the circuit. This means, we should now have
+   * the created circuit attached on both the pchan and nchan cmux. */
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 1);
+
+  /* There should be _no_ active circuit due to no queued cells. */
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_active_circuits(nchan->cmux), OP_EQ, 0);
+
+  /* Circuit should not be active on the cmux. */
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(nchan->cmux, circ), OP_EQ, 0);
+
+  /* Not active so no cells. */
+  n_cells = circuitmux_num_cells_for_circuit(pchan->cmux, circ);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  n_cells = circuitmux_num_cells(pchan->cmux);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  n_cells = circuitmux_num_cells_for_circuit(nchan->cmux, circ);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  n_cells = circuitmux_num_cells(nchan->cmux);
+  tt_uint_op(n_cells, OP_EQ, 0);
+
+  /* So it should be attached :) */
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 1);
+
+  /* Query the chanid<->circid map in the cmux subsytem with what we just
+   * created and validate the cell direction. */
+  cdir = circuitmux_attached_circuit_direction(pchan->cmux, circ);
+  tt_int_op(cdir, OP_EQ, CELL_DIRECTION_IN);
+  cdir = circuitmux_attached_circuit_direction(nchan->cmux, circ);
+  tt_int_op(cdir, OP_EQ, CELL_DIRECTION_OUT);
+
+  /*
+   * We'll activate->deactivate->activate to test all code paths of
+   * circuitmux_set_num_cells().
+   */
+
+  /* Activate circuit. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+
+  /* Deactivate. */
+  circuitmux_clear_num_cells(pchan->cmux, circ);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 0);
+
+  /* Re-activate. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+
+  /* Once re-attached, it should become inactive because the circuit has no
+   * cells while the chanid<->circid object has some. The attach code will
+   * reset the count on the cmux for that circuit:
+   *
+   * if (chanid_circid_muxinfo_t->muxinfo.cell_count > 0 && cell_count == 0) {
+   */
+  circuitmux_attach_circuit(pchan->cmux, circ, CELL_DIRECTION_IN);
+  n_cells = circuitmux_num_cells_for_circuit(pchan->cmux, circ);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 0);
+
+  /* Lets queue a cell on the circuit now so it becomes active when
+   * re-attaching:
+   *
+   * else if (chanid_circid_muxinfo_t->muxinfo.cell_count == 0 &&
+   *          cell_count > 0) {
+   */
+  orcirc->p_chan_cells.n = 1;
+  circuitmux_attach_circuit(pchan->cmux, circ, CELL_DIRECTION_IN);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_detach_circuit(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* While assigning a new circuit IDs, the circuitmux_attach_circuit() is
+   * called for a new channel on the circuit. This means, we should now have
+   * the created circuit attached on both the pchan and nchan cmux. */
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 1);
+
+  /* Now, detach the circuit from pchan and then nchan. */
+  circuitmux_detach_circuit(pchan->cmux, circ);
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 0);
+  circuitmux_detach_circuit(nchan->cmux, circ);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_detach_all_circuits(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+  smartlist_t *detached_out = smartlist_new();
+
+  (void) arg;
+
+  /* Channels need to be registered in order for the detach all circuit
+   * function to find them. */
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  channel_register(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+  channel_register(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* Just make sure it is attached. */
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 1);
+
+  /* Queue some cells so we can test if the circuit becomes inactive on the
+   * cmux after the mass detach. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  circuitmux_set_num_cells(nchan->cmux, circ, 4);
+
+  /* Detach all on pchan and then nchan. */
+  circuitmux_detach_all_circuits(pchan->cmux, detached_out);
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(smartlist_len(detached_out), OP_EQ, 1);
+  circuitmux_detach_all_circuits(nchan->cmux, NULL);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(nchan->cmux, circ), OP_EQ, 0);
+
+ done:
+  smartlist_free(detached_out);
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_policy(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  channel_register(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+  channel_register(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* Confirm we have the EWMA policy by default for new channels. */
+  tt_ptr_op(circuitmux_get_policy(pchan->cmux), OP_EQ, &ewma_policy);
+  tt_ptr_op(circuitmux_get_policy(nchan->cmux), OP_EQ, &ewma_policy);
+
+  /* Putting cell on the cmux means will make the notify policy code path to
+   * trigger. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+
+  /* Clear it out. */
+  circuitmux_clear_policy(pchan->cmux);
+
+  /* Set back the EWMA policy. */
+  circuitmux_set_policy(pchan->cmux, &ewma_policy);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_xmit_cell(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* Queue 4 cells on the circuit. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 4);
+  tt_uint_op(circuitmux_num_cells(pchan->cmux), OP_EQ, 4);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 1);
+
+  /* Emit the first cell. Circuit should still be active. */
+  circuitmux_notify_xmit_cells(pchan->cmux, circ, 1);
+  tt_uint_op(circuitmux_num_cells(pchan->cmux), OP_EQ, 3);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 3);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 1);
+
+  /* Emit the last 3 cells. Circuit should become inactive. */
+  circuitmux_notify_xmit_cells(pchan->cmux, circ, 3);
+  tt_uint_op(circuitmux_num_cells(pchan->cmux), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 0);
+
+  /* Queue a DESTROY cell. */
+  pchan->has_queued_writes = mock_has_queued_writes_true;
+  circuitmux_append_destroy_cell(pchan, pchan->cmux, orcirc->p_circ_id, 0);
+  tt_i64_op(pchan->cmux->destroy_ctr, OP_EQ, 1);
+  tt_int_op(pchan->cmux->destroy_cell_queue.n, OP_EQ, 1);
+  tt_i64_op(circuitmux_count_queued_destroy_cells(pchan, pchan->cmux),
+            OP_EQ, 1);
+
+  /* Emit the DESTROY cell. */
+  circuitmux_notify_xmit_destroy(pchan->cmux);
+  tt_i64_op(pchan->cmux->destroy_ctr, OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void *
+cmux_setup_test(const struct testcase_t *tc)
+{
+  static int whatever;
+
+  (void) tc;
+
+  cell_ewma_initialize_ticks();
+  return &whatever;
+}
+
+static int
+cmux_cleanup_test(const struct testcase_t *tc, void *ptr)
+{
+  (void) tc;
+  (void) ptr;
+
+  circuitmux_ewma_free_all();
+
+  return 1;
+}
+
+static struct testcase_setup_t cmux_test_setup = {
+  .setup_fn = cmux_setup_test,
+  .cleanup_fn = cmux_cleanup_test,
+};
+
+#define TEST_CMUX(name) \
+  { #name, test_cmux_##name, TT_FORK, &cmux_test_setup, NULL }
+
 struct testcase_t circuitmux_tests[] = {
-  { "destroy_cell_queue", test_cmux_destroy_cell_queue, TT_FORK, NULL, NULL },
-  { "compute_ticks", test_cmux_compute_ticks, TT_FORK, NULL, NULL },
+  /* Test circuitmux_t object */
+  TEST_CMUX(allocate),
+  TEST_CMUX(attach_circuit),
+  TEST_CMUX(detach_circuit),
+  TEST_CMUX(detach_all_circuits),
+  TEST_CMUX(policy),
+  TEST_CMUX(xmit_cell),
+
+  /* Misc. */
+  TEST_CMUX(compute_ticks),
+  TEST_CMUX(destroy_cell_queue),
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_circuitmux_ewma.c b/src/test/test_circuitmux_ewma.c
new file mode 100644
index 0000000000..27601e0c7d
--- /dev/null
+++ b/src/test/test_circuitmux_ewma.c
@@ -0,0 +1,228 @@
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITMUX_PRIVATE
+#define CIRCUITMUX_EWMA_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
+
+#include "test/fakechans.h"
+#include "test/fakecircs.h"
+#include "test/test.h"
+
+static void
+test_cmux_ewma_active_circuit(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t *pol_data = NULL;
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  circ_data = ewma_policy.alloc_circ_data(&cmux, pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+
+  /* Get EWMA specific objects. */
+
+  /* Make circuit active. */
+  ewma_policy.notify_circ_active(&cmux, pol_data, &circ, circ_data);
+
+  circuit_t *entry = ewma_policy.pick_active_circuit(&cmux, pol_data);
+  tt_mem_op(entry, OP_EQ, &circ, sizeof(circ));
+
+ done:
+  ewma_policy.free_circ_data(&cmux, pol_data, &circ, circ_data);
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void
+test_cmux_ewma_xmit_cell(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t *pol_data = NULL;
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+  ewma_policy_data_t *ewma_pol_data;
+  ewma_policy_circ_data_t *ewma_data;
+  double old_cell_count;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  circ_data = ewma_policy.alloc_circ_data(&cmux, pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  ewma_data = TO_EWMA_POL_CIRC_DATA(circ_data);
+
+  /* Make circuit active. */
+  ewma_policy.notify_circ_active(&cmux, pol_data, &circ, circ_data);
+
+  /* Move back in time the last time we calibrated so we scale the active
+   * circuit when emitting a cell. */
+  ewma_pol_data->active_circuit_pqueue_last_recalibrated -= 100;
+  ewma_data->cell_ewma.last_adjusted_tick =
+    ewma_pol_data->active_circuit_pqueue_last_recalibrated;
+
+  /* Grab old cell count. */
+  old_cell_count = ewma_data->cell_ewma.cell_count;
+
+  ewma_policy.notify_xmit_cells(&cmux, pol_data, &circ, circ_data, 1);
+
+  /* Our old cell count should be lower to what we have since we just emitted
+   * a cell and thus we scale. */
+  tt_double_op(old_cell_count, OP_LT, ewma_data->cell_ewma.cell_count);
+
+ done:
+  ewma_policy.free_circ_data(&cmux, pol_data, &circ, circ_data);
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void
+test_cmux_ewma_notify_circ(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t *pol_data = NULL;
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+  const ewma_policy_data_t *ewma_pol_data;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  circ_data = ewma_policy.alloc_circ_data(&cmux, pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+
+  /* Currently, notify_circ_active() ignores cmux and circ. They can not be
+   * NULL so it is fine to pass garbage. */
+  ewma_policy.notify_circ_active(&cmux, pol_data, &circ, circ_data);
+
+  /* We should have an active circuit in the queue so its EWMA value can be
+   * tracked. */
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  tt_int_op(smartlist_len(ewma_pol_data->active_circuit_pqueue), OP_EQ, 1);
+  tt_uint_op(ewma_pol_data->active_circuit_pqueue_last_recalibrated, OP_NE, 0);
+
+  ewma_policy.notify_circ_inactive(&cmux, pol_data, &circ, circ_data);
+  /* Should be removed from the active queue. */
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  tt_int_op(smartlist_len(ewma_pol_data->active_circuit_pqueue), OP_EQ, 0);
+  tt_uint_op(ewma_pol_data->active_circuit_pqueue_last_recalibrated, OP_NE, 0);
+
+ done:
+  ewma_policy.free_circ_data(&cmux, pol_data, &circ, circ_data);
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void
+test_cmux_ewma_policy_circ_data(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t pol_data; /* garbage */
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+  const ewma_policy_circ_data_t *ewma_data;
+
+  (void) arg;
+
+  /* Currently, alloc_circ_data() ignores every parameter _except_ the cell
+   * direction so it is OK to pass garbage. They can not be NULL. */
+  circ_data = ewma_policy.alloc_circ_data(&cmux, &pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+  tt_uint_op(circ_data->magic, OP_EQ, EWMA_POL_CIRC_DATA_MAGIC);
+
+  ewma_data = TO_EWMA_POL_CIRC_DATA(circ_data);
+  tt_mem_op(ewma_data->circ, OP_EQ, &circ, sizeof(circuit_t));
+  tt_double_op(ewma_data->cell_ewma.cell_count, OP_LE, 0.0);
+  tt_int_op(ewma_data->cell_ewma.heap_index, OP_EQ, -1);
+  tt_uint_op(ewma_data->cell_ewma.is_for_p_chan, OP_EQ, 0);
+  ewma_policy.free_circ_data(&cmux, &pol_data, &circ, circ_data);
+
+  circ_data = ewma_policy.alloc_circ_data(&cmux, &pol_data, &circ,
+                                          CELL_DIRECTION_IN, 42);
+  tt_assert(circ_data);
+  tt_uint_op(circ_data->magic, OP_EQ, EWMA_POL_CIRC_DATA_MAGIC);
+
+  ewma_data = TO_EWMA_POL_CIRC_DATA(circ_data);
+  tt_mem_op(ewma_data->circ, OP_EQ, &circ, sizeof(circuit_t));
+  tt_double_op(ewma_data->cell_ewma.cell_count, OP_LE, 0.0);
+  tt_int_op(ewma_data->cell_ewma.heap_index, OP_EQ, -1);
+  tt_uint_op(ewma_data->cell_ewma.is_for_p_chan, OP_EQ, 1);
+
+ done:
+  ewma_policy.free_circ_data(&cmux, &pol_data, &circ, circ_data);
+}
+
+static void
+test_cmux_ewma_policy_data(void *arg)
+{
+  circuitmux_t cmux; /* garbage. */
+  circuitmux_policy_data_t *pol_data = NULL;
+  const ewma_policy_data_t *ewma_pol_data;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  tt_uint_op(pol_data->magic, OP_EQ, EWMA_POL_DATA_MAGIC);
+
+  /* Test EWMA object. */
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  tt_assert(ewma_pol_data->active_circuit_pqueue);
+  tt_uint_op(ewma_pol_data->active_circuit_pqueue_last_recalibrated, OP_NE, 0);
+
+ done:
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void *
+cmux_ewma_setup_test(const struct testcase_t *tc)
+{
+  static int whatever;
+
+  (void) tc;
+
+  cell_ewma_initialize_ticks();
+  cmux_ewma_set_options(NULL, NULL);
+
+  return &whatever;
+}
+
+static int
+cmux_ewma_cleanup_test(const struct testcase_t *tc, void *ptr)
+{
+  (void) tc;
+  (void) ptr;
+
+  circuitmux_ewma_free_all();
+
+  return 1;
+}
+
+static struct testcase_setup_t cmux_ewma_test_setup = {
+  .setup_fn = cmux_ewma_setup_test,
+  .cleanup_fn = cmux_ewma_cleanup_test,
+};
+
+#define TEST_CMUX_EWMA(name) \
+  { #name, test_cmux_ewma_##name, TT_FORK, &cmux_ewma_test_setup, NULL }
+
+struct testcase_t circuitmux_ewma_tests[] = {
+  TEST_CMUX_EWMA(active_circuit),
+  TEST_CMUX_EWMA(policy_data),
+  TEST_CMUX_EWMA(policy_circ_data),
+  TEST_CMUX_EWMA(notify_circ),
+  TEST_CMUX_EWMA(xmit_cell),
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c
new file mode 100644
index 0000000000..cfb24c032c
--- /dev/null
+++ b/src/test/test_circuitpadding.c
@@ -0,0 +1,3147 @@
+#define CHANNEL_OBJECT_PRIVATE
+#define TOR_TIMERS_PRIVATE
+#define CIRCUITPADDING_PRIVATE
+#define CIRCUITPADDING_MACHINES_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define CRYPT_PATH_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "lib/testsupport/testsupport.h"
+#include "core/or/connection_or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/crypt_path.h"
+#include <event.h>
+#include "lib/evloop/compat_libevent.h"
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "core/or/relay.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/mainloop/netstatus.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/or/protover.h"
+#include "feature/nodelist/nodelist.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "test/fakecircs.h"
+#include "test/rng_test_helpers.h"
+
+/* Start our monotime mocking at 1 second past whatever monotime_init()
+ * thought the actual wall clock time was, for platforms with bad resolution
+ * and weird timevalues during monotime_init() before mocking. */
+#define MONOTIME_MOCK_START   (monotime_absolute_nsec()+\
+                               TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC)
+
+extern smartlist_t *connection_array;
+void circuit_expire_old_circuits_clientside(void);
+
+circid_t get_unique_circ_id_by_chan(channel_t *chan);
+void helper_create_basic_machine(void);
+static void helper_create_conditional_machines(void);
+
+channel_t *new_fake_channel(void);
+void test_circuitpadding_negotiation(void *arg);
+void test_circuitpadding_wronghop(void *arg);
+void test_circuitpadding_conditions(void *arg);
+
+void test_circuitpadding_serialize(void *arg);
+void test_circuitpadding_rtt(void *arg);
+void test_circuitpadding_tokens(void *arg);
+void test_circuitpadding_state_length(void *arg);
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding);
+void free_fake_origin_circuit(origin_circuit_t *circ);
+
+static int deliver_negotiated = 1;
+static int64_t curr_mocked_time;
+
+static node_t padding_node;
+static node_t non_padding_node;
+
+static channel_t dummy_channel;
+static circpad_machine_spec_t circ_client_machine;
+
+static void
+timers_advance_and_run(int64_t msec_update)
+{
+  curr_mocked_time += msec_update*TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+  timers_run_pending();
+}
+
+static void
+nodes_init(void)
+{
+  padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  padding_node.rs->pv.supports_hs_setup_padding = 1;
+
+  non_padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  non_padding_node.rs->pv.supports_hs_setup_padding = 0;
+}
+
+static void
+nodes_free(void)
+{
+  tor_free(padding_node.rs);
+
+  tor_free(non_padding_node.rs);
+}
+
+static const node_t *
+node_get_by_id_mock(const char *identity_digest)
+{
+  if (identity_digest[0] == 1) {
+    return &padding_node;
+  } else if (identity_digest[0] == 0) {
+    return &non_padding_node;
+  }
+
+  return NULL;
+}
+
+static const node_t *
+circuit_get_nth_node_mock(origin_circuit_t *circ, int hop)
+{
+  (void) circ;
+  (void) hop;
+
+  return &padding_node;
+}
+
+void
+free_fake_origin_circuit(origin_circuit_t *circ)
+{
+  circpad_circuit_free_all_machineinfos(TO_CIRCUIT(circ));
+  circuit_clear_cpath(circ);
+  tor_free(circ);
+}
+
+void dummy_nop_timer(void);
+
+//static int dont_stop_libevent = 0;
+
+static circuit_t *client_side;
+static circuit_t *relay_side;
+
+static int n_client_cells = 0;
+static int n_relay_cells = 0;
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction)
+{
+  (void)cmux;
+  (void)circ;
+  (void)direction;
+
+  return;
+}
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno)
+{
+  (void)cell; (void)on_stream; (void)filename; (void)lineno;
+
+  if (circ == client_side) {
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      // Deliver to relay
+      circpad_handle_padding_negotiate(relay_side, cell);
+    } else {
+
+      int is_target_hop = circpad_padding_is_from_expected_hop(circ,
+                                                             layer_hint);
+      tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_OUT);
+      tt_int_op(is_target_hop, OP_EQ, 1);
+
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(client_side);
+
+      // Receive padding cell at middle
+      circpad_deliver_recognized_relay_cell_events(relay_side,
+              cell->payload[0], NULL);
+    }
+    n_client_cells++;
+  } else if (circ == relay_side) {
+    tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_IN);
+
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATED) {
+      // XXX: blah need right layer_hint..
+      if (deliver_negotiated)
+        circpad_handle_padding_negotiated(client_side, cell,
+                                          TO_ORIGIN_CIRCUIT(client_side)
+                                             ->cpath->next);
+    } else if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      circpad_handle_padding_negotiate(client_side, cell);
+    } else {
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(relay_side);
+
+      // Receive padding cell at client
+      circpad_deliver_recognized_relay_cell_events(client_side,
+              cell->payload[0],
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+    }
+
+    n_relay_cells++;
+  }
+
+ done:
+  timers_advance_and_run(1);
+  return 0;
+}
+
+// Test reading and writing padding to strings (or options_t + consensus)
+void
+test_circuitpadding_serialize(void *arg)
+{
+  (void)arg;
+}
+
+static signed_error_t
+circpad_send_command_to_hop_mock(origin_circuit_t *circ, uint8_t hopnum,
+                                 uint8_t relay_command, const uint8_t *payload,
+                                 ssize_t payload_len)
+{
+  (void) circ;
+  (void) hopnum;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  return 0;
+}
+
+void
+test_circuitpadding_rtt(void *arg)
+{
+  /* Test Plan:
+   *
+   * 1. Test RTT measurement server side
+   *    a. test usage of measured RTT
+   * 2. Test termination of RTT measurement
+   *    a. test non-update of RTT
+   * 3. Test client side circuit and non-application of RTT..
+   */
+  circpad_delay_t rtt_estimate;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+  helper_create_basic_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0);
+
+  /* Test 1: Test measuring RTT */
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+
+  timers_advance_and_run(20);
+
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 2: Termination of RTT measurement (from the previous test) */
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec;
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  timers_advance_and_run(4);
+  circpad_cell_event_nonpadding_sent(relay_side);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ,
+            rtt_estimate);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 3: Make sure client side machine properly ignores RTT */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_NE, client_side->padding_info[0]->rtt_estimate_usec);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_EQ,
+            circpad_machine_current_state(
+                client_side->padding_info[0])->histogram_edges[0]);
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+
+  return;
+}
+
+void
+helper_create_basic_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.name = "basic";
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_START].use_rtt_estimate = 1;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 2500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 5000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[4] = 20000;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+
+  return;
+}
+
+#define BIG_HISTOGRAM_LEN 10
+
+/** Setup a machine with a big histogram */
+static void
+helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy)
+{
+  const int tokens_per_bin = 2;
+
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circpad_state_t *burst_state =
+    &circ_client_machine.states[CIRCPAD_STATE_BURST];
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_RECV] =CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_SENT] =CIRCPAD_STATE_CANCEL;
+
+  burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  burst_state->histogram_len = BIG_HISTOGRAM_LEN;
+
+  int n_tokens = 0;
+  int i;
+  for (i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    burst_state->histogram[i] = tokens_per_bin;
+    n_tokens += tokens_per_bin;
+  }
+
+  burst_state->histogram_edges[0] = 0;
+  burst_state->histogram_edges[1] = 1;
+  burst_state->histogram_edges[2] = 7;
+  burst_state->histogram_edges[3] = 15;
+  burst_state->histogram_edges[4] = 31;
+  burst_state->histogram_edges[5] = 62;
+  burst_state->histogram_edges[6] = 125;
+  burst_state->histogram_edges[7] = 250;
+  burst_state->histogram_edges[8] = 500;
+  burst_state->histogram_edges[9] = 1000;
+
+  burst_state->histogram_total_tokens = n_tokens;
+  burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM;
+  burst_state->length_dist.param1 = n_tokens;
+  burst_state->length_dist.param2 = n_tokens;
+  burst_state->max_length = n_tokens;
+  burst_state->length_includes_nonpadding = 1;
+  burst_state->use_rtt_estimate = 0;
+  burst_state->token_removal = removal_strategy;
+}
+
+static circpad_decision_t
+circpad_machine_schedule_padding_mock(circpad_machine_runtime_t *mi)
+{
+  (void)mi;
+  return 0;
+}
+
+static uint64_t
+mock_monotime_absolute_usec(void)
+{
+  return 100;
+}
+
+/** Test higher token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_higher(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_HIGHER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Test right boundaries of each histogram bin: */
+  const circpad_delay_t bin_right_bounds[] =
+    {0,    6,  14,  30,  61,  124, 249, 499, 999, CIRCPAD_DELAY_INFINITE-1};
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_right_bounds[i], OP_EQ,
+               histogram_get_bin_upper_bound(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 5, 6, 7, 8};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all lower bins are not touched */
+  for (i=0; i < 4 ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  mi->padding_scheduled_at_usec = current_time;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test lower token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_lower(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_LOWER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 2, 1, 0};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all higher bins are untouched */
+  for (i = 5; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy by bin  */
+static void
+test_circuitpadding_closest_token_removal(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 6, 1, 7, 0, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_closest_token_removal_usec(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* XXX we want to test remove_token_exact and
+     circpad_machine_remove_closest_token() with usec */
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 1, 0, 6, 7, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_token_removal_exact(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_EXACT);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /**********************************************************************/
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Ensure that we will clear out bin #4 with this usec */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_cell_event_nonpadding_sent(client_side);
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 1);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[4], OP_EQ, 0);
+
+  /* Ensure that we will not remove any other tokens even tho we try to, since
+   * this is what the exact strategy dictates */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  circpad_cell_event_nonpadding_sent(client_side);
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    if (i != 4) {
+      tt_int_op(mi->histogram[i], OP_EQ, 2);
+    }
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+#undef BIG_HISTOGRAM_LEN
+
+void
+test_circuitpadding_tokens(void *arg)
+{
+  const circpad_state_t *state;
+  circpad_machine_runtime_t *mi;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  testing_enable_reproducible_rng();
+
+  /** Test plan:
+   *
+   * 1. Test symmetry between bin_to_usec and usec_to_bin
+   *    a. Test conversion
+   *    b. Test edge transitions (lower, upper)
+   * 2. Test remove higher on an empty bin
+   *    a. Normal bin
+   *    b. Infinity bin
+   *    c. Bin 0
+   *    d. No higher
+   * 3. Test remove lower
+   *    a. Normal bin
+   *    b. Bin 0
+   *    c. No lower
+   * 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  mi = client_side->padding_info[0];
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_sent(client_side);
+  /* We have to save the infinity bin because one inf delay
+   * could have been chosen when we transition to burst */
+  circpad_hist_token_t inf_bin = mi->histogram[4];
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  state = circpad_machine_current_state(client_side->padding_info[0]);
+
+  // Test 0: convert bin->usec->bin
+  // Bin 0+1 have different semantics
+  for (int bin = 0; bin < 2; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+  }
+  for (int bin = 2; bin < state->histogram_len-1; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    /* Verify we round down */
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec+3);
+    tt_int_op(bin, OP_EQ, bin2);
+
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec-1);
+    tt_int_op(bin, OP_EQ, bin2+1);
+  }
+
+  // Test 1: converting usec->bin->usec->bin
+  // Bin 0+1 have different semantics.
+  for (circpad_delay_t i = 0; i <= state->histogram_edges[0]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_LE, usec);
+  }
+  for (circpad_delay_t i = state->histogram_edges[0]+1;
+       i <= state->histogram_edges[0] +
+         state->histogram_edges[state->histogram_len-2]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_GE, usec);
+  }
+
+  /* 2.a. Normal higher bin */
+  {
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+  }
+
+  /* 2.b. Higher Infinity bin */
+  {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+
+    /* Test past the infinity bin */
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 5)+1000000);
+
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+  }
+
+  /* 2.c. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+    mi->histogram[0] = 1;
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* Drain the infinity bin and cause a refill */
+  while (inf_bin != 0) {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_cell_event_nonpadding_received(client_side);
+    inf_bin--;
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+
+  // We should have refilled here.
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 3.a. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 1);
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* 3.b. Test remove lower normal bin */
+  {
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    /* 3.c. No lower */
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+  }
+
+  /* 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 5. Test remove closest usec
+   *    a. Closest 0
+   *    b. Closest lower (below midpoint)
+   *    c. Closest higher (above midpoint)
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  // XXX: Need more coverage of the actual usec branches
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_wronghop(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Padding sent from hop 1 and 3 to client
+   * 2. Send negotiated from hop 1 and 3 to client
+   * 3. Garbled negotiated cell
+   * 4. Padding negotiate sent to client
+   * 5. Send negotiate stop command for unknown machine
+   * 6. Send negotiated to relay
+   * 7. Garbled padding negotiate cell
+   */
+  (void)arg;
+  uint32_t read_bw = 0, overhead_bw = 0;
+  cell_t cell;
+  signed_error_t ret;
+  origin_circuit_t *orig_client;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  /* Mock this function so that our cell counting tests don't get confused by
+   * padding that gets sent by scheduled timers. */
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  orig_client = TO_ORIGIN_CIRCUIT(client_side);
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build three hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+
+  /* 1. Test padding from first and third hop */
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_LT,
+            orig_client->n_overhead_read_circ_bw);
+
+  /* 2. Test padding negotiated not handled from hops 1,3 */
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 3. Garbled negotiated cell */
+  memset(&cell, 255, sizeof(cell));
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 4. Test that negotiate is dropped at origin */
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+  relay_send_command_from_edge(0, relay_side,
+                               RELAY_COMMAND_PADDING_NEGOTIATE,
+                               (void*)cell.payload,
+                               (size_t)3, NULL);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 5. Test that asking to stop the wrong machine does nothing */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            255, 2, CIRCPAD_COMMAND_STOP);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* 6. Sending negotiated command to relay does nothing */
+  ret = circpad_handle_padding_negotiated(relay_side, &cell, NULL);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 7. Test garbled negotated cell (bad command 255) */
+  memset(&cell, 0, sizeof(cell));
+  ret = circpad_handle_padding_negotiate(relay_side, &cell);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Spoof padding negotiated on circuit with no padding */
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_OK);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_ERR);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_negotiation(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Test circuit where padding is supported by middle
+   *    a. Make sure padding negotiation is sent
+   *    b. Test padding negotiation delivery and parsing
+   * 2. Test circuit where padding is unsupported by middle
+   *    a. Make sure padding negotiation is not sent
+   * 3. Test failure to negotiate a machine due to desync.
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Force negotiate padding. */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            CIRCPAD_MACHINE_CIRC_SETUP,
+                            2, CIRCPAD_COMMAND_START);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 3. Test failure to negotiate a machine due to desync */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  SMARTLIST_FOREACH(relay_padding_machines,
+          circpad_machine_spec_t *,
+          m, tor_free(m->states); tor_free(m));
+  smartlist_free(relay_padding_machines);
+  relay_padding_machines = smartlist_new();
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify echo was sent */
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+}
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding)
+{
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+  char digest[DIGEST_LEN];
+  tor_addr_t addr;
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_sent(client);
+
+  // Receive extend cell at middle
+  circpad_cell_event_nonpadding_received(mid_relay);
+
+  // Advance time a tiny bit so we can calculate an RTT
+  curr_mocked_time += 10 * TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+
+  // Receive extended cell at middle
+  circpad_cell_event_nonpadding_sent(mid_relay);
+
+  // Receive extended cell at first hop
+  circpad_cell_event_nonpadding_received(client);
+
+  // Add a hop to cpath
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+  cpath_extend_linked_list(&TO_ORIGIN_CIRCUIT(client)->cpath, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_OPEN;
+
+  // add an extend info to indicate if this node supports padding or not.
+  // (set the first byte of the digest for our mocked node_get_by_id)
+  digest[0] = padding;
+
+  hop->extend_info = extend_info_new(
+          padding ? "padding" : "non-padding",
+          digest, NULL, NULL, NULL,
+          &addr, padding);
+
+  cpath_init_circuit_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  // Signal that the hop was added
+  circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client));
+}
+
+static circpad_machine_spec_t *
+helper_create_length_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  /* No token removal.. end via state_length only */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Let's have this one end after 12 packets */
+  ret->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param1 = 12;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param2 = 13;
+  ret->states[CIRCPAD_STATE_BURST].max_length = 12;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 4;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 6;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 0;
+
+  return ret;
+}
+
+static circpad_machine_spec_t *
+helper_create_conditional_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Use EXACT removal strategy, otherwise setup_tokens() does not work */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 3;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 0;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  return ret;
+}
+
+static void
+helper_create_conditional_machines(void)
+{
+  circpad_machine_spec_t *add = helper_create_conditional_machine();
+
+  if (!origin_padding_machines)
+    origin_padding_machines = smartlist_new();
+  if (!relay_padding_machines)
+    relay_padding_machines = smartlist_new();
+
+  add->machine_num = 2;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 0;
+  add->conditions.min_hops = 2;
+  add->conditions.state_mask = CIRCPAD_CIRC_BUILDING|
+           CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 1;
+  add->conditions.min_hops = 3;
+  add->conditions.state_mask = CIRCPAD_CIRC_OPENED|
+           CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 2;
+  circpad_register_padding_machine(add, relay_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  circpad_register_padding_machine(add, relay_padding_machines);
+}
+
+void
+test_circuitpadding_state_length(void *arg)
+{
+  /**
+   * Test plan:
+   *  * Explicitly test that with no token removal enabled, we hit
+   *    the state length limit due to either padding, or non-padding.
+   *  * Repeat test with an arbitrary token removal strategy, and
+   *    verify that if we run out of tokens due to padding before we
+   *    hit the state length, we still go to state end (all our
+   *    token removal tests only test nonpadding token removal).
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  circpad_machine_spec_t *client_machine =
+      helper_create_length_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  client_side->padding_machine[0] = client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  circpad_machine_runtime_t *mi = client_side->padding_info[0];
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_EQ, NULL);
+
+  /* Verify that non-padding does not change our state length */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* verify that sending padding changes our state length */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+  circpad_send_padding_cell_for_callback(mi);
+
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+
+  /* Now, count nonpadding as part of the state length */
+  client_machine->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* Verify that non-padding does change our state length now */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_cell_event_nonpadding_sent(client_side);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Now, just test token removal when we send padding */
+  client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_NE, NULL);
+  tt_int_op(mi->chosen_bin, OP_EQ, 2);
+
+  /* verify that sending padding changes our state length and
+   * our histogram now */
+  for (uint32_t i = mi->histogram[2]-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_int_op(mi->chosen_bin, OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, i);
+  }
+
+  tt_i64_op(mi->state_length, OP_EQ, 7);
+  tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+  circpad_send_padding_cell_for_callback(mi);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+ done:
+  tor_free(client_machine->states);
+  tor_free(client_machine);
+
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(node_get_by_id);
+
+  return;
+}
+
+void
+test_circuitpadding_conditions(void *arg)
+{
+  /**
+   * Test plan:
+   *  0. Make a few origin and client machines with diff conditions
+   *     * vanguards, purposes, has_opened circs, no relay early
+   *     * Client side should_negotiate_end
+   *     * Length limits
+   *  1. Test STATE_END transitions
+   *  2. Test new machine after end with same conditions
+   *  3. Test new machine due to changed conditions
+   *     * Esp: built event, no relay early, no streams
+   * XXX: Diff test:
+   *  1. Test STATE_END with pending timers
+   *  2. Test marking a circuit before padding callback fires
+   *  3. Test freeing a circuit before padding callback fires
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  helper_create_conditional_machines();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Send another event.. verify machine gets re-added properly
+   * (test race with shutdown) */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = 0;
+  circpad_machine_event_circ_has_no_streams(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Now make the circuit opened and send built event */
+  TO_ORIGIN_CIRCUIT(client_side)->has_opened = 1;
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->remaining_relay_early_cells = 0;
+  circpad_machine_event_circ_has_no_relay_early(
+          TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  get_options_mutable()->HSLayer2Nodes = (void*)1;
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = (void*)1;
+  circpad_machine_event_circ_has_streams(TO_ORIGIN_CIRCUIT(client_side));
+
+  /* Verify different machine is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  /* Hold off on negotiated */
+  deliver_negotiated = 0;
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client
+   * but not the response for the padding machine */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify machine is gone from the relay (but negotiated not back yet */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Add another hop and verify it's back */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+ done:
+  /* XXX: Free everything */
+  testing_disable_reproducible_rng();
+  return;
+}
+
+/** Disabled unstable test until #29298 is implemented (see #29122) */
+#if 0
+void
+test_circuitpadding_circuitsetup_machine(void *arg)
+{
+  int64_t actual_mocked_monotime_start;
+  /**
+   * Test case plan:
+   *
+   * 1. Simulate a normal circuit setup pattern
+   *    a. Application traffic
+   *
+   * FIXME: This should focus more on exercising the machine
+   * features rather than actual traffic patterns. For example,
+   * test cancellation and bins empty/refill
+   */
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  dummy_channel.cmux = circuitmux_alloc();
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Test case #1: Build a 3 hop circuit, then wait and let pad */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(n_client_cells, OP_EQ, 1);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_BURST);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 6);
+
+  tt_int_op(client_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_END);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_GAP);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+
+  /* Verify we can't schedule padding in END state */
+  circpad_decision_t ret =
+      circpad_machine_schedule_padding(client_side->padding_info[0]);
+  tt_int_op(ret, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Simulate application traffic */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 7);
+
+  // Test timer cancellation
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 8);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Test timer cancel due to state rules */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  circpad_cell_event_padding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Simulate application traffic to cancel timer */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* No cells sent, except negotiate end from relay */
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 9);
+
+  /* Test mark for close and free */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE);
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  timers_advance_and_run(5000);
+
+  /* No cells sent */
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+
+  return;
+}
+#endif /* 0 */
+
+/** Helper function: Initializes a padding machine where every state uses the
+ *  uniform probability distribution.  */
+static void
+helper_circpad_circ_distribution_machine_setup(int min, int max)
+{
+  circpad_machine_states_init(&circ_client_machine, 7);
+
+  circpad_state_t *zero_st = &circ_client_machine.states[0];
+  zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1;
+  zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM;
+  /* param2 is upper bound, param1 is lower */
+  zero_st->iat_dist.param1 = min;
+  zero_st->iat_dist.param2 = max;
+  zero_st->dist_added_shift_usec = min;
+  zero_st->dist_max_sample_usec = max;
+
+  circpad_state_t *first_st = &circ_client_machine.states[1];
+  first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2;
+  first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC;
+  /* param1 is Mu, param2 is sigma. */
+  first_st->iat_dist.param1 = 9;
+  first_st->iat_dist.param2 = 3;
+  first_st->dist_added_shift_usec = min;
+  first_st->dist_max_sample_usec = max;
+
+  circpad_state_t *second_st = &circ_client_machine.states[2];
+  second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3;
+  second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC;
+  /* param1 is Alpha, param2 is 1.0/Beta */
+  second_st->iat_dist.param1 = 1;
+  second_st->iat_dist.param2 = 0.5;
+  second_st->dist_added_shift_usec = min;
+  second_st->dist_max_sample_usec = max;
+
+  circpad_state_t *third_st = &circ_client_machine.states[3];
+  third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4;
+  third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC;
+  /* param1 is 'p' (success probability) */
+  third_st->iat_dist.param1 = 0.2;
+  third_st->dist_added_shift_usec = min;
+  third_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fourth_st = &circ_client_machine.states[4];
+  fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5;
+  fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL;
+  /* param1 is k, param2 is Lambda */
+  fourth_st->iat_dist.param1 = 1.5;
+  fourth_st->iat_dist.param2 = 1;
+  fourth_st->dist_added_shift_usec = min;
+  fourth_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fifth_st = &circ_client_machine.states[5];
+  fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6;
+  fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO;
+  /* param1 is sigma, param2 is xi */
+  fifth_st->iat_dist.param1 = 1;
+  fifth_st->iat_dist.param2 = 5;
+  fifth_st->dist_added_shift_usec = min;
+  fifth_st->dist_max_sample_usec = max;
+}
+
+/** Simple test that the padding delays sampled from a uniform distribution
+ *  actually faill within the uniform distribution range. */
+static void
+test_circuitpadding_sample_distribution(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  int n_samples;
+  int n_states;
+
+  (void) arg;
+
+  /* mock this function so that we dont actually schedule any padding */
+  MOCK(circpad_machine_schedule_padding,
+       circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Initialize a machine with multiple probability distributions */
+  circpad_machines_init();
+  helper_circpad_circ_distribution_machine_setup(0, 10);
+
+  /* Initialize machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* For every state, sample a bunch of values from the distribution and ensure
+   * they fall within range. */
+  for (n_states = 0 ; n_states < 6; n_states++) {
+    /* Make sure we in the right state */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, n_states);
+
+    for (n_samples = 0; n_samples < 100; n_samples++) {
+      circpad_delay_t delay = circpad_machine_sample_delay(mi);
+      tt_int_op(delay, OP_GE, 0);
+      tt_int_op(delay, OP_LE, 10);
+    }
+
+    /* send a non-padding cell to move to the next machine state */
+    circpad_cell_event_nonpadding_received(client_side);
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  UNMOCK(circpad_machine_schedule_padding);
+  testing_disable_reproducible_rng();
+}
+
+static circpad_decision_t
+circpad_machine_spec_transition_mock(circpad_machine_runtime_t *mi,
+                                circpad_event_t event)
+{
+  (void) mi;
+  (void) event;
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/* Test per-machine padding rate limits */
+static void
+test_circuitpadding_machine_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  /* First we are going to test the per-machine rate limits */
+  circ_client_machine.max_padding_percent = 50;
+  circ_client_machine.allowed_padding_count = 100;
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Cover wrap */
+  for (;i<UINT16_MAX;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/2+1);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, mi);
+  for (i=0;i<UINT16_MAX;i++) {
+    circpad_cell_event_nonpadding_sent(client_side);
+  }
+
+  tt_int_op(mi->nonpadding_sent, OP_EQ, UINT16_MAX/2);
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/4+1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  testing_disable_reproducible_rng();
+}
+
+/* Test global padding rate limits */
+static void
+test_circuitpadding_global_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+  int64_t actual_mocked_monotime_start;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  testing_enable_reproducible_rng();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  dummy_channel.cmux = circuitmux_alloc();
+
+  /* Setup machine and circuits */
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  helper_create_basic_machine();
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(relay_side, 0);
+  mi = relay_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Now test the global limits by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_global_allowed_cells=100 circpad_global_max_padding_pct=50",
+                         NULL, 0, 0);
+  /* Register global limits with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Now send 92 non-padding cells to get near the
+   * circpad_global_max_padding_pct=50 limit; in particular with 96 non-padding
+   * cells, the padding traffic is still 51% of total traffic so limit should
+   * trigger */
+  for (i=0;i<92;i++) {
+    circpad_cell_event_nonpadding_sent(relay_side);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Send another non-padding cell to bring the padding traffic to 50% of total
+   * traffic and get past the limit */
+  circpad_cell_event_nonpadding_sent(relay_side);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  testing_disable_reproducible_rng();
+}
+
+/* Test reduced and disabled padding */
+static void
+test_circuitpadding_reduce_disable(void *arg)
+{
+  (void) arg;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel);
+  client_side = (circuit_t *)origin_circuit_new();
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  circpad_machines_init();
+  helper_create_conditional_machines();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Now test the reduced padding machine by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_reduced=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  /* Test that machines get torn down when padding is disabled */
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_disabled=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = NULL;
+  circpad_new_consensus_params(&vote1);
+
+  get_options_mutable()->ReducedCircuitPadding = 1;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  get_options_mutable()->CircuitPadding = 0;
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  testing_disable_reproducible_rng();
+}
+
+/** Just a basic machine whose whole purpose is to reach the END state */
+static void
+helper_create_ender_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_END;
+
+  circ_client_machine.conditions.state_mask = CIRCPAD_STATE_ALL;
+  circ_client_machine.conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+}
+
+static time_t mocked_timeofday;
+/** Set timeval to a mock date and time. This is necessary
+ * to make tor_gettimeofday() mockable. */
+static void
+mock_tor_gettimeofday(struct timeval *timeval)
+{
+  timeval->tv_sec = mocked_timeofday;
+  timeval->tv_usec = 0;
+}
+
+/** Test manual managing of circuit lifetimes by the circuitpadding
+ *  subsystem. In particular this test goes through all the cases of the
+ *  circpad_marked_circuit_for_padding() function, via
+ *  circuit_mark_for_close() as well as
+ *  circuit_expire_old_circuits_clientside(). */
+static void
+test_circuitpadding_manage_circuit_lifetime(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+  MOCK(tor_gettimeofday, mock_tor_gettimeofday);
+  mocked_timeofday = 23;
+
+  helper_create_ender_machine();
+
+  /* Enable manual circuit lifetime manage for this test */
+  circ_client_machine.manage_circ_lifetime = 1;
+
+  /* Test setup */
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Check that the circuit is not marked for close */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+
+  /* Mark this circuit for close due to a remote reason */
+  circuit_mark_for_close(client_side,
+                         END_CIRC_REASON_FLAG_REMOTE|END_CIRC_REASON_NONE);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close due to a protocol issue */
+  circuit_mark_for_close(client_side, END_CIRC_REASON_TORPROTOCOL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark a measurement circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  client_side->marked_for_close = 0;
+
+  /* Mark a general circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+
+  /* Check that this circuit is still not marked for close since we are
+   * managing the lifetime manually, but the circuit was tagged as such by the
+   * circpadding subsystem */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* We just tested case (1) from the comments of
+   * circpad_circuit_should_be_marked_for_close() */
+
+  /* Transition the machine to the END state but did not delete its machine */
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* We just tested case (3) from the comments of
+   * circpad_circuit_should_be_marked_for_close().
+   * Now let's go for case (2). */
+
+  /* Reset the close mark */
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close */
+  circuit_mark_for_close(client_side, 0);
+
+  /* See that the circ got closed since we are already in END state */
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* We just tested case (2). Now let's see that case (4) is unreachable as
+     that comment claims */
+
+  /* First, reset all close marks and tags */
+  client_side->marked_for_close = 0;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  /* Now re-create the ender machine so that we can transition to END again */
+  /* Free up some stuff first */
+  circpad_circuit_free_all_machineinfos(client_side);
+  tor_free(circ_client_machine.states);
+  helper_create_ender_machine();
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* Check we are in START. */
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Test that we don't expire this circuit yet */
+  client_side->timestamp_dirty = 0;
+  client_side->state = CIRCUIT_STATE_OPEN;
+  tor_gettimeofday(&client_side->timestamp_began);
+  TO_ORIGIN_CIRCUIT(client_side)->circuit_idle_timeout = 23;
+  mocked_timeofday += 24;
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->timestamp_dirty, OP_NE, 0);
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Runaway circpad test: if the machine does not transition to end,
+   * test that after CIRCPAD_DELAY_MAX_SECS, we get marked anyway */
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  client_side->padding_info[0]->last_cell_time_sec =
+      approx_time()-(CIRCPAD_DELAY_MAX_SECS+10);
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* Test back to normal: if we had activity, we won't close */
+  client_side->padding_info[0]->last_cell_time_sec = approx_time();
+  client_side->marked_for_close = 0;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Transition to END, but before we're past the dirty timer */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Verify that the circuit was not closed. */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Now that we are in END state, we can be closed by expiry, but via
+   * the timestamp_dirty path, not the idle path. So first test not dirty
+   * enough. */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  tor_free(circ_client_machine.states);
+  monotime_disable_test_mocking();
+  UNMOCK(tor_gettimeofday);
+}
+
+/** Helper for the test_circuitpadding_hs_machines test:
+ *
+ *  - Create a client and relay circuit.
+ *  - Setup right circuit purpose and attach a machine to the client circuit.
+ *  - Verify that state transitions work as intended and state length gets
+ *    enforced.
+ *
+ *  This function is able to do this test both for intro and rend circuits
+ *  depending on the value of <b>test_intro_circs</b>.
+ */
+static void
+helper_test_hs_machines(bool test_intro_circs)
+{
+  /* Setup the circuits */
+  origin_circuit_t *origin_client_side = origin_circuit_new();
+  client_side = TO_CIRCUIT(origin_client_side);
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+
+  /* extend the client circ to two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* machines only apply on opened circuits */
+  origin_client_side->has_opened = 1;
+
+  /************************************/
+
+  /* Attaching the client machine now won't work here because of a wrong
+   * purpose */
+  tt_assert(!client_side->padding_machine[0]);
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_assert(!client_side->padding_machine[0]);
+
+  /* Change the purpose, see the machine getting attached */
+  client_side->purpose = test_intro_circs ?
+    CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT : CIRCUIT_PURPOSE_C_REND_JOINED;
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify that the right machine is attached */
+  tt_str_op(client_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "client_ip_circ" : "client_rp_circ");
+  tt_str_op(relay_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "relay_ip_circ": "relay_rp_circ");
+
+  /***********************************/
+
+  /* Intro machines are at START state, but rend machines have already skipped
+   * to OBFUSCATE_CIRC_SETUP because of the sent PADDING_NEGOTIATE. */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /*Send non-padding to move the machines from START to OBFUSCATE_CIRC_SETUP */
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /* Check that the state lengths have been sampled and are within range */
+  circpad_machine_runtime_t *client_machine_runtime =
+    client_side->padding_info[0];
+  circpad_machine_runtime_t *relay_machine_runtime =
+    relay_side->padding_info[0];
+
+  if (test_intro_circs) {
+    /* on the client side, we don't send any padding so
+     * state length is not set */
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, -1);
+    /* relay side has state limits. check them */
+    tt_i64_op(relay_machine_runtime->state_length, OP_GE,
+              INTRO_MACHINE_MINIMUM_PADDING);
+    tt_i64_op(relay_machine_runtime->state_length, OP_LT,
+              INTRO_MACHINE_MAXIMUM_PADDING);
+  } else {
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, 1);
+    tt_i64_op(relay_machine_runtime->state_length, OP_EQ, 1);
+  }
+
+  if (test_intro_circs) {
+    int i;
+    /* Send state_length worth of padding from the relay and see that the
+     * client state goes to END */
+    for (i = (int) relay_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(relay_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted (the padding info should now be null on both sides) */
+    tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+    tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  } else {
+    int i;
+    /* Send state_length worth of padding and see that the state goes to END */
+    for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(client_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted. */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_END);
+  }
+
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+}
+
+/** Test that the HS circuit padding machines work as intended. */
+static void
+test_circuitpadding_hs_machines(void *arg)
+{
+  (void)arg;
+
+  /* Test logic:
+   *
+   * 1) Register the HS machines, which aim to hide the presense of
+   *    onion service traffic on the client-side
+   *
+   * 2) Call helper_test_hs_machines() to perform tests for the intro circuit
+   *    machines and for the rend circuit machines.
+  */
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock);
+  MOCK(circuit_get_nth_node, circuit_get_nth_node_mock);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  curr_mocked_time = 1*TOR_NSEC_PER_USEC;
+
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  /************************************/
+
+  /* Register the HS machines */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  /***********************************/
+
+  /* Do the tests for the intro circuit machines */
+  helper_test_hs_machines(true);
+  /* Do the tests for the rend circuit machines */
+  helper_test_hs_machines(false);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+
+  SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  smartlist_free(origin_padding_machines);
+  smartlist_free(relay_padding_machines);
+
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuit_get_nth_node);
+  UNMOCK(circpad_machine_schedule_padding);
+}
+
+/** Test that we effectively ignore non-padding cells in padding circuits. */
+static void
+test_circuitpadding_ignore_non_padding_cells(void *arg)
+{
+  int retval;
+  relay_header_t rh;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_CIRCUIT_PADDING;
+
+  rh.command = RELAY_COMMAND_BEGIN;
+
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = handle_relay_cell_command(NULL, client_side, NULL, NULL, &rh, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  expect_log_msg_containing("Ignored cell");
+
+ done:
+  ;
+}
+
+#define TEST_CIRCUITPADDING(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t circuitpadding_tests[] = {
+  TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_state_length, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK),
+  /** Disabled unstable test until #29298 is implemented (see #29122) */
+  //  TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_reduce_disable, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_manage_circuit_lifetime, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_hs_machines, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_ignore_non_padding_cells, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c
index 1cbcb14f2b..e15dec5a01 100644
--- a/src/test/test_circuitstats.c
+++ b/src/test/test_circuitstats.c
@@ -1,10 +1,10 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITSTATS_PRIVATE
 #define CIRCUITLIST_PRIVATE
-#define CHANNEL_PRIVATE_
+#define CHANNEL_FILE_PRIVATE
 
 #include "core/or/or.h"
 #include "test/test.h"
@@ -28,7 +28,7 @@ origin_circuit_t *subtest_fourhop_circuit(struct timeval, int);
 origin_circuit_t *add_opened_threehop(void);
 origin_circuit_t *build_unopened_fourhop(struct timeval);
 
-int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 
 static int marked_for_close;
 /* Mock function because we are not trying to test the close circuit that does
@@ -57,9 +57,9 @@ add_opened_threehop(void)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = DEFAULT_ROUTE_LEN;
 
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
 
   or_circ->has_opened = 1;
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -82,10 +82,10 @@ build_unopened_fourhop(struct timeval circ_start_time)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = 4;
 
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
 
   tor_free(fakehop);
 
@@ -197,7 +197,7 @@ test_circuitstats_hoplen(void *arg)
 }
 
 #define TEST_CIRCUITSTATS(name, flags) \
-    { #name, test_##name, (flags), NULL, NULL }
+    { #name, test_##name, (flags), &helper_pubsub_setup, NULL }
 
 struct testcase_t circuitstats_tests[] = {
   TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK),
diff --git a/src/test/test_circuituse.c b/src/test/test_circuituse.c
index 3acfc12044..49438d9d3b 100644
--- a/src/test/test_circuituse.c
+++ b/src/test/test_circuituse.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITLIST_PRIVATE
diff --git a/src/test/test_cmdline.sh b/src/test/test_cmdline.sh
new file mode 100755
index 0000000000..ded58af63d
--- /dev/null
+++ b/src/test/test_cmdline.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+umask 077
+set -e
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+die() { echo "$1" >&2 ; exit 5; }
+
+echo "A"
+
+DATA_DIR=$(mktemp -d -t tor_cmdline_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# 1. Test list-torrc-options.
+OUT="${DATA_DIR}/output"
+
+echo "B"
+"${TOR_BINARY}" --list-torrc-options > "$OUT"
+
+echo "C"
+
+# regular options are given.
+grep -i "SocksPort" "$OUT" >/dev/null || die "Did not find SocksPort"
+
+
+echo "D"
+
+# unlisted options are given, since they do not have the NOSET flag.
+grep -i "__SocksPort" "$OUT" > /dev/null || die "Did not find __SocksPort"
+
+echo "E"
+
+# unsettable options are not given.
+if grep -i "DisableIOCP" "$OUT"  /dev/null; then
+    die "Found DisableIOCP"
+fi
+if grep -i "HiddenServiceOptions" "$OUT" /dev/null ; then
+    die "Found HiddenServiceOptions"
+fi
+echo "OK"
diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c
index 2f8646e897..5376e08fb3 100644
--- a/src/test/test_compat_libevent.c
+++ b/src/test/test_compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define COMPAT_LIBEVENT_PRIVATE
@@ -13,8 +13,6 @@
 
 #include "test/log_test_helpers.h"
 
-#define NS_MODULE compat_libevent
-
 static void
 test_compat_libevent_logging_callback(void *ignored)
 {
@@ -151,8 +149,6 @@ test_compat_libevent_postloop_events(void *arg)
   mainloop_event_t *a = NULL, *b = NULL;
   periodic_timer_t *timed = NULL;
 
-  tor_libevent_postfork();
-
   /* If postloop events don't work, then these events will activate one
    * another ad infinitum and, and the periodic event will never occur. */
   b = mainloop_event_postloop_new(activate_event_cb, &a);
@@ -187,4 +183,3 @@ struct testcase_t compat_libevent_tests[] = {
     TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_config.c b/src/test/test_config.c
index d648666f6e..71beb93f67 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -1,11 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CONFIG_PRIVATE
+#define RELAY_CONFIG_PRIVATE
+#define RELAY_TRANSPORT_CONFIG_PRIVATE
 #define PT_PRIVATE
 #define ROUTERSET_PRIVATE
 #include "core/or/or.h"
@@ -16,7 +18,10 @@
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "app/config/resolve_addr.h"
+#include "feature/relay/relay_config.h"
+#include "feature/relay/transport_config.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "test/test.h"
@@ -24,6 +29,7 @@
 #include "feature/control/control.h"
 #include "core/mainloop/cpuworker.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/relay/dns.h"
 #include "feature/client/entrynodes.h"
@@ -37,6 +43,7 @@
 #include "core/or/policies.h"
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
+#include "feature/relay/relay_find_addr.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
 #include "feature/nodelist/dirlist.h"
@@ -45,6 +52,7 @@
 #include "app/config/statefile.h"
 
 #include "test/test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "feature/dirclient/dir_server_st.h"
 #include "core/or/port_cfg_st.h"
@@ -54,6 +62,7 @@
 #include "lib/meminfo/meminfo.h"
 #include "lib/net/gethostname.h"
 #include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -670,6 +679,54 @@ transport_is_needed_mock(const char *transport_name)
   return transport_is_needed_mock_return;
 }
 
+static void
+test_config_parse_tcp_proxy_line(void *arg)
+{
+  (void)arg;
+
+  int ret;
+  char *msg = NULL;
+  or_options_t *options = get_options_mutable();
+
+  /* Bad TCPProxy line - too short. */
+  ret = parse_tcp_proxy_line("haproxy", options, &msg);
+  /* Return error. */
+  tt_int_op(ret, OP_EQ, -1);
+  /* Correct error message. */
+  tt_str_op(msg, OP_EQ, "TCPProxy has no address/port. Please fix.");
+  /* Free error message. */
+  tor_free(msg);
+
+  /* Bad TCPProxy line - unsupported protocol. */
+  ret = parse_tcp_proxy_line("unsupported 95.216.163.36:443", options, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "TCPProxy protocol is not supported. Currently the "
+                        "only supported protocol is 'haproxy'. Please fix.");
+  tor_free(msg);
+
+  /* Bad TCPProxy line - unparsable address/port. */
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  ret = parse_tcp_proxy_line("haproxy bogus_address!/300", options, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "TCPProxy address/port failed to parse or resolve. "
+                        "Please fix.");
+  tor_free(msg);
+  UNMOCK(tor_addr_lookup);
+
+  /* Good TCPProxy line - ipv4. */
+  ret = parse_tcp_proxy_line("haproxy 95.216.163.36:443", options, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(options->TCPProxyProtocol, OP_EQ, TCP_PROXY_PROTOCOL_HAPROXY);
+  /* Correct the address. */
+  tt_assert(tor_addr_eq_ipv4h(&options->TCPProxyAddr, 0x5fd8a324));
+  tt_int_op(options->TCPProxyPort, OP_EQ, 443);
+  tor_free(msg);
+
+ done:
+  UNMOCK(tor_addr_lookup);
+}
+
 /**
  * Test parsing for the ClientTransportPlugin and ServerTransportPlugin config
  * options.
@@ -687,84 +744,84 @@ test_config_parse_transport_plugin_line(void *arg)
   int old_transport_is_needed_mock_call_count;
 
   /* Bad transport lines - too short */
-  r = parse_transport_line(options, "bad", 1, 0);
+  r = pt_parse_transport_line(options, "bad", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options, "bad", 1, 1);
+  r = pt_parse_transport_line(options, "bad", 1, 1);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options, "bad bad", 1, 0);
+  r = pt_parse_transport_line(options, "bad bad", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options, "bad bad", 1, 1);
+  r = pt_parse_transport_line(options, "bad bad", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* Test transport list parsing */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
    "transport_1 exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_EQ, 0);
   /* Bad transport identifiers */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* Check SOCKS cases for client transport */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks4 1.2.3.4:567", 1, 0);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 1, 0);
   tt_int_op(r, OP_EQ, 0);
   /* Proxy case for server transport */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 1, 1);
   tt_int_op(r, OP_EQ, 0);
   /* Multiple-transport error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 socks5 1.2.3.4:567", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 proxy 1.2.3.4:567", 1, 1);
   tt_int_op(r, OP_LT, 0);
   /* No port error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3.4", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
      "transport_1 proxy 1.2.3.4", 1, 1);
   tt_int_op(r, OP_LT, 0);
   /* Unparsable address error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3:6x7", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 proxy 1.2.3:6x7", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* "Strange {Client|Server}TransportPlugin field" error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 foo bar", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 foo bar", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* No sandbox mode error exit */
   tmp = options->Sandbox;
   options->Sandbox = 1;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_LT, 0);
   options->Sandbox = tmp;
@@ -776,7 +833,7 @@ test_config_parse_transport_plugin_line(void *arg)
   MOCK(pt_kickstart_proxy, pt_kickstart_proxy_mock);
   old_pt_kickstart_proxy_mock_call_count =
     pt_kickstart_proxy_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 1);
   tt_int_op(r, OP_EQ, 0);
   tt_assert(pt_kickstart_proxy_mock_call_count ==
@@ -784,7 +841,7 @@ test_config_parse_transport_plugin_line(void *arg)
   UNMOCK(pt_kickstart_proxy);
 
   /* This one hits a log line in the !validate_only case only */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 0, 1);
   tt_int_op(r, OP_EQ, 0);
 
@@ -801,7 +858,7 @@ test_config_parse_transport_plugin_line(void *arg)
     transport_add_from_config_mock_call_count;
   old_transport_is_needed_mock_call_count =
     transport_is_needed_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
   tt_int_op(r, OP_EQ, 0);
@@ -825,7 +882,7 @@ test_config_parse_transport_plugin_line(void *arg)
     transport_add_from_config_mock_call_count;
   old_transport_is_needed_mock_call_count =
     transport_is_needed_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
   tt_int_op(r, OP_EQ, 0);
@@ -849,7 +906,7 @@ test_config_parse_transport_plugin_line(void *arg)
     transport_add_from_config_mock_call_count;
   old_transport_is_needed_mock_call_count =
     transport_is_needed_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 0, 0);
   /* Should have succeeded */
   tt_int_op(r, OP_EQ, 0);
@@ -904,14 +961,12 @@ test_config_fix_my_family(void *arg)
   family3->next = NULL;
 
   or_options_t* options = options_new();
-  or_options_t* defaults = options_new();
   (void) arg;
 
   options_init(options);
-  options_init(defaults);
   options->MyFamily_lines = family;
 
-  options_validate(NULL, options, defaults, 0, &err) ;
+  options_validate(NULL, options, &err) ;
 
   if (err != NULL) {
     TT_FAIL(("options_validate failed: %s", err));
@@ -933,7 +988,6 @@ test_config_fix_my_family(void *arg)
  done:
   tor_free(err);
   or_options_free(options);
-  or_options_free(defaults);
 }
 
 static int n_hostname_01010101 = 0;
@@ -1754,6 +1808,18 @@ add_default_fallback_dir_servers_known_default(void)
   n_add_default_fallback_dir_servers_known_default++;
 }
 
+/* Helper for test_config_adding_dir_servers(), which should be
+ * refactored: clear the fields in the options which the options object
+ * does not really own. */
+static void
+ads_clear_helper(or_options_t *options)
+{
+  options->DirAuthorities = NULL;
+  options->AlternateBridgeAuthority = NULL;
+  options->AlternateDirAuthority = NULL;
+  options->FallbackDir = NULL;
+}
+
 /* Test all the different combinations of adding dir servers */
 static void
 test_config_adding_dir_servers(void *arg)
@@ -1761,7 +1827,7 @@ test_config_adding_dir_servers(void *arg)
   (void)arg;
 
   /* allocate options */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
 
   /* Allocate and populate configuration lines:
    *
@@ -1884,7 +1950,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -1966,7 +2034,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2107,7 +2177,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2248,7 +2320,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2390,7 +2464,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2542,7 +2618,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2696,7 +2774,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2859,7 +2939,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3016,7 +3098,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3182,7 +3266,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3345,7 +3431,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3514,10 +3602,7 @@ test_config_adding_dir_servers(void *arg)
   tor_free(test_fallback_directory->value);
   tor_free(test_fallback_directory);
 
-  options->DirAuthorities = NULL;
-  options->AlternateBridgeAuthority = NULL;
-  options->AlternateDirAuthority = NULL;
-  options->FallbackDir = NULL;
+  ads_clear_helper(options);
   or_options_free(options);
 
   UNMOCK(add_default_fallback_dir_servers);
@@ -3532,7 +3617,7 @@ test_config_default_dir_servers(void *arg)
   int fallback_count = 0;
 
   /* new set of options should stop fallback parsing */
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 0;
   /* set old_options to NULL to force dir update */
   consider_adding_dir_servers(opts, NULL);
@@ -3546,7 +3631,7 @@ test_config_default_dir_servers(void *arg)
   /* if we disable the default fallbacks, there must not be any extra */
   tt_assert(fallback_count == trusted_count);
 
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 1;
   consider_adding_dir_servers(opts, opts);
   trusted_count = smartlist_len(router_get_trusted_dir_servers());
@@ -3606,7 +3691,7 @@ test_config_directory_fetch(void *arg)
   (void)arg;
 
   /* Test Setup */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
   routerinfo_t routerinfo;
   memset(&routerinfo, 0, sizeof(routerinfo));
   mock_router_pick_published_address_result = -1;
@@ -3618,70 +3703,76 @@ test_config_directory_fetch(void *arg)
        mock_router_my_exit_policy_is_reject_star);
   MOCK(advertised_server_mode, mock_advertised_server_mode);
   MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo);
+  or_options_free(options);
+  options = options_new();
 
   /* Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
   options->ClientOnly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* Bridge Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->UseBridges = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* Bridge Relays (Bridges) must act like clients, and use multiple
    * directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->BridgeRelay = 1;
   options->ORPort_set = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* Clients set to FetchDirInfoEarly must fetch it from the authorities,
    * but can use multiple authorities for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->FetchDirInfoEarly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* OR servers only fetch the consensus from the authorities when they don't
    * know their own address, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
 
   mock_router_pick_published_address_result = -1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
   /* Exit OR servers only fetch the consensus from the authorities when they
    * refuse unknown exits, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
   options->ExitRelay = 1;
   mock_router_pick_published_address_result = 0;
@@ -3694,7 +3785,7 @@ test_config_directory_fetch(void *arg)
   options->RefuseUnknownExits = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3702,7 +3793,7 @@ test_config_directory_fetch(void *arg)
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3711,7 +3802,8 @@ test_config_directory_fetch(void *arg)
    * advertising their dirport, and never use multiple directories for
    * bootstrap. This only applies if they are also OR servers.
    * (We don't care much about the behaviour of non-OR directory servers.) */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->DirPort_set = 1;
   options->ORPort_set = 1;
   options->DirCache = 1;
@@ -3723,7 +3815,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3732,7 +3824,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3740,7 +3832,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = NULL;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3750,7 +3842,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3760,12 +3852,12 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
  done:
-  tor_free(options);
+  or_options_free(options);
   UNMOCK(router_pick_published_address);
   UNMOCK(router_get_my_routerinfo);
   UNMOCK(advertised_server_mode);
@@ -3776,7 +3868,9 @@ static void
 test_config_default_fallback_dirs(void *arg)
 {
   const char *fallback[] = {
+#ifndef COCCI
 #include "app/config/fallback_dirs.inc"
+#endif
     NULL
   };
 
@@ -3958,40 +4052,40 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   slout = smartlist_new();
 
   // Test no defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 0, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 42, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and with out
-  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and out, adds a new port cfg
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
@@ -4002,7 +4096,7 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   // for a unix address
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, "DNS", 0, "/foo/bar/unixdomain",
+  ret = port_parse_config(slout, NULL, "DNS", 0, "/foo/bar/unixdomain",
                           42, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4029,30 +4123,32 @@ test_config_parse_port_config__ports__ports_given(void *data)
 
   slout = smartlist_new();
 
+  mock_hostname_resolver();
+
   // Test error when encounters an invalid Port specification
   config_port_invalid = mock_config_line("DNSPort", "");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters an empty unix domain specification
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "unix:");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters a unix domain specification but the listener
   // doesn't support domain sockets
   config_port_valid = mock_config_line("DNSPort", "unix:/tmp/foo/bar");
-  ret = parse_port_config(NULL, config_port_valid, "DNS",
+  ret = port_parse_config(NULL, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test valid unix domain
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0, 0);
 #ifdef _WIN32
   tt_int_op(ret, OP_EQ, -1);
@@ -4063,7 +4159,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->port, OP_EQ, 0);
   tt_int_op(port_cfg->is_unix_addr, OP_EQ, 1);
   tt_str_op(port_cfg->unix_addr, OP_EQ, "/tmp/foo/bar");
-  /* Test entry port defaults as initialised in parse_port_config */
+  /* Test entry port defaults as initialised in port_parse_config */
   tt_int_op(port_cfg->entry_cfg.dns_request, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
@@ -4077,7 +4173,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
                                          "unix:/tmp/foo/bar NoIPv4Traffic "
                                          "NoIPv6Traffic "
                                          "NoOnionTraffic");
-  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
+  ret = port_parse_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4086,7 +4182,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.1:80 NoDNSRequest");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS",
+  ret = port_parse_config(NULL, config_port_invalid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4099,7 +4195,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("DNSPort", "127.0.0.1:80 "
                                        "NoIPv6Traffic "
                                        "NoIPv4Traffic NoOnionTraffic");
-  ret = parse_port_config(slout, config_port_valid, "DNS",
+  ret = port_parse_config(slout, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4115,7 +4211,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_invalid = mock_config_line("SOCKSPort",
                                          "NoIPv6Traffic "
                                          "unix:/tmp/foo/bar NoIPv4Traffic");
-  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
+  ret = port_parse_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4128,7 +4224,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4150,7 +4246,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar\" "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4172,7 +4268,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4184,7 +4280,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"\" "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4195,7 +4291,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "OnionTrafficOnly");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4216,7 +4312,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "NoIPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4235,7 +4331,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "IPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4251,28 +4347,28 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure if we specify world writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 WorldWritable");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 GroupWritable");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 RelaxDirModeCheck");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with only a port (this will fail without a default address)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42");
-  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4281,7 +4377,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestPort");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4294,7 +4390,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateDestPorts");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4307,7 +4403,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestAddr");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4320,7 +4416,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4333,7 +4429,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientProtocol");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4346,7 +4442,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientAddr");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4357,7 +4453,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with ignored unknown options
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42 ThisOptionDoesntExist");
-  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4366,7 +4462,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4379,7 +4475,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort",
                                        "42 IPv6Traffic PreferIPv6");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.42", 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4392,7 +4488,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4405,7 +4501,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv6DNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4418,7 +4514,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoCacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4431,7 +4527,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheDNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4444,7 +4540,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv4Cache");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4457,7 +4553,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv6Cache");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4470,7 +4566,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseDNSCache");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4483,7 +4579,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoPreferIPv6Automap");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4495,7 +4591,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 PreferSOCKSNoAuth");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4510,14 +4606,14 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_invalid = mock_config_line("DNSPort", "0");
   config_port_valid = mock_config_line("DNSPort", "42");
   config_port_invalid->next = config_port_valid;
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with warn non-local control
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "Control",
+  ret = port_parse_config(slout, config_port_valid, "Control",
                           CONN_TYPE_CONTROL_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4525,7 +4621,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local listener
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "ExtOR",
+  ret = port_parse_config(slout, config_port_valid, "ExtOR",
                           CONN_TYPE_EXT_OR_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4533,12 +4629,12 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local other
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test success with warn non-local other without out
-  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4549,7 +4645,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IPv4Traffic "
                                        "IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0,
                           CL_PORT_TAKES_HOSTNAMES |
                           CL_PORT_NO_STREAM_OPTIONS);
@@ -4564,7 +4660,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=invalid");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4574,7 +4670,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4584,7 +4680,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123 "
                                          "SessionGroup=321");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4593,7 +4689,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 SessionGroup=1111122");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4605,7 +4701,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "0");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
@@ -4615,7 +4711,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "something");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4628,48 +4724,48 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "auto");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
   tor_addr_parse(&addr, "127.0.0.46");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test success with a port of auto in mixed case
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "AuTo");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
   tor_addr_parse(&addr, "127.0.0.46");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test success with parsing both an address and an auto port
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.122:auto");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
   tor_addr_parse(&addr, "127.0.0.122");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test failure when asked to parse an invalid address followed by auto
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "invalidstuff!!:auto");
   MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
@@ -4679,21 +4775,21 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.123:656");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, 656);
   tor_addr_parse(&addr, "127.0.0.123");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test failure if we can't parse anything at all
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "something wrong");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4702,7 +4798,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.1.0:123:auto");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4712,7 +4808,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/somewhere");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.46", 0,
                           CL_PORT_DFLT_GROUP_WRITABLE);
 #ifdef _WIN32
@@ -4725,6 +4821,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
 #endif /* defined(_WIN32) */
 
  done:
+  unmock_hostname_resolver();
   if (slout)
     SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_free(slout);
@@ -4747,7 +4844,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort",
                                        "127.0.0.124:656 NoAdvertise");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4760,7 +4857,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4774,7 +4871,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen "
                                          "NoAdvertise");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4783,7 +4880,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4796,7 +4893,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "[::1]:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4810,7 +4907,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 IPv6Only "
                                          "IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4819,7 +4916,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 unknown");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4830,7 +4927,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.124:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4839,7 +4936,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "[::1]:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4848,7 +4945,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("ORPort", "unix:\"\"");
-  ret = parse_port_config(slout, config_port_invalid, "ORPort", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "ORPort", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -5128,7 +5225,7 @@ test_config_include_no_permission(void *data)
     chmod(dir, 0700);
   tor_free(dir);
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 static void
 test_config_include_recursion_before_after(void *data)
@@ -5662,7 +5759,6 @@ test_config_check_bridge_distribution_setting_not_a_bridge(void *arg)
 {
   or_options_t* options = get_options_mutable();
   or_options_t* old_options = options;
-  or_options_t* default_options = options;
   char* message = NULL;
   int ret;
 
@@ -5671,7 +5767,7 @@ test_config_check_bridge_distribution_setting_not_a_bridge(void *arg)
   options->BridgeRelay = 0;
   options->BridgeDistribution = (char*)("https");
 
-  ret = options_validate(old_options, options, default_options, 0, &message);
+  ret = options_validate(old_options, options, &message);
 
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(message, OP_EQ, "You set BridgeDistribution, but you "
@@ -5843,7 +5939,7 @@ test_config_compute_max_mem_in_queues(void *data)
 #else
   /* We are on a 32-bit system. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1));
-#endif
+#endif /* SIZEOF_VOID_P >= 8 */
 
   /* We are able to detect the amount of RAM on the system. */
   total_system_memory_return = 0;
@@ -5884,7 +5980,7 @@ test_config_compute_max_mem_in_queues(void *data)
   /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
             MAX_DEFAULT_MEMORY_QUEUE_SIZE);
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
  done:
   UNMOCK(get_total_system_memory);
@@ -5928,6 +6024,7 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "is back here");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
   lp = lp->next;
+  tt_assert(!lp);
   config_free_lines(lines);
 
   /* Try with the "extended" flag enabled. */
@@ -5954,11 +6051,198 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR);
   lp = lp->next;
+  tt_assert(!lp);
 
  done:
   config_free_lines(lines);
 }
 
+static void
+test_config_kvline_parse(void *arg)
+{
+  (void)arg;
+
+  config_line_t *lines = NULL;
+  char *enc = NULL;
+
+  lines = kvline_parse("A=B CD=EF", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "A");
+  tt_str_op(lines->value, OP_EQ, "B");
+  tt_str_op(lines->next->key, OP_EQ, "CD");
+  tt_str_op(lines->next->value, OP_EQ, "EF");
+  enc = kvline_encode(lines, 0);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB CDE=F", 0);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "");
+  tt_str_op(lines->value, OP_EQ, "AB");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F");
+  tt_assert(lines);
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB CDE=F");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\"", 0);
+  tt_assert(!lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "C");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F G");
+  tt_str_op(lines->next->next->key, OP_EQ, "");
+  tt_str_op(lines->next->next->value, OP_EQ, "GHI");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB=", KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse(" AB ", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD DE FGH=I", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD DE FGH=I");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=\"CD E\" DE FGH=\"I\"", KV_OMIT_VALS|KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD E");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS|KV_QUOTED);
+  tt_str_op(enc, OP_EQ, "AB=\"CD E\" DE FGH=I");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD \"EF=GH\"", KV_OMIT_KEYS|KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  tt_str_op(lines->next->key, OP_EQ, "");
+  tt_str_op(lines->next->value, OP_EQ, "EF=GH");
+  enc = kvline_encode(lines, KV_OMIT_KEYS);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_OMIT_KEYS|KV_QUOTED);
+  tt_assert(enc);
+  tt_str_op(enc, OP_EQ, "AB=CD \"EF=GH\"");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = tor_malloc_zero(sizeof(*lines));
+  lines->key = tor_strdup("A=B");
+  lines->value = tor_strdup("CD");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  config_free_lines(lines);
+
+  config_line_append(&lines, "A", "B C");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_RAW);
+  tt_assert(enc);
+  tt_str_op(enc, OP_EQ, "A=B C");
+
+ done:
+  config_free_lines(lines);
+  tor_free(enc);
+}
+
+static void
+test_config_getinfo_config_names(void *arg)
+{
+  (void)arg;
+  char *answer = NULL;
+  const char *error = NULL;
+  int rv;
+
+  rv = getinfo_helper_config(NULL, "config/names", &answer, &error);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(error, OP_EQ, NULL);
+
+  // ContactInfo should be listed.
+  tt_assert(strstr(answer, "\nContactInfo String\n"));
+
+  // V1AuthoritativeDirectory should not be listed, since it is obsolete.
+  tt_assert(! strstr(answer, "V1AuthoritativeDirectory"));
+
+  // ___UsingTestNetworkDefaults should not be listed, since it is invisible.
+  tt_assert(! strstr(answer, "UsingTestNetworkDefaults"));
+
+ done:
+  tor_free(answer);
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
@@ -5977,6 +6261,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(parse_bridge_line, 0),
   CONFIG_TEST(parse_transport_options_line, 0),
   CONFIG_TEST(parse_transport_plugin_line, TT_FORK),
+  CONFIG_TEST(parse_tcp_proxy_line, TT_FORK),
   CONFIG_TEST(check_or_create_data_subdir, TT_FORK),
   CONFIG_TEST(write_to_data_subdir, TT_FORK),
   CONFIG_TEST(fix_my_family, 0),
@@ -6012,5 +6297,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(include_opened_file_list, 0),
   CONFIG_TEST(compute_max_mem_in_queues, 0),
   CONFIG_TEST(extended_fmt, 0),
+  CONFIG_TEST(kvline_parse, 0),
+  CONFIG_TEST(getinfo_config_names, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_confmgr.c b/src/test/test_confmgr.c
new file mode 100644
index 0000000000..a647b92e0a
--- /dev/null
+++ b/src/test/test_confmgr.c
@@ -0,0 +1,499 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confmgt.c's features that support multiple configuration
+ * formats and configuration objects.
+ */
+
+#define CONFMGT_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confmgt.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+/*
+ * Set up a few objects: a pasture_cfg is toplevel; it has a llama_cfg and an
+ * alpaca_cfg.
+ */
+
+typedef struct {
+  uint32_t magic;
+  char *address;
+  int opentopublic;
+  config_suite_t *subobjs;
+} pasture_cfg_t;
+
+typedef struct {
+  char *llamaname;
+  int cuteness;
+  uint32_t magic;
+  int eats_meat; /* deprecated; llamas are never carnivorous. */
+
+  char *description; // derived from other fields.
+} llama_cfg_t;
+
+typedef struct {
+  uint32_t magic;
+  int fuzziness;
+  char *alpacaname;
+  int n_wings; /* deprecated; alpacas don't have wings. */
+
+  int square_fuzziness; /* Derived from fuzziness. */
+} alpaca_cfg_t;
+
+/*
+ * Make the above into configuration objects.
+ */
+
+static pasture_cfg_t pasture_cfg_t_dummy;
+static llama_cfg_t llama_cfg_t_dummy;
+static alpaca_cfg_t alpaca_cfg_t_dummy;
+
+#define PV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(pasture_cfg_t, #name, type, name, 0, dflt)
+#define LV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(llama_cfg_t, #name, type, name, 0, dflt)
+#define AV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(alpaca_cfg_t, #name, type, name, 0, dflt)
+static const config_var_t pasture_vars[] = {
+  PV(address, STRING, NULL),
+  PV(opentopublic, BOOL, "1"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t llama_vars[] =
+{
+  LV(llamaname, STRING, NULL),
+  LV(eats_meat, BOOL, NULL),
+  LV(cuteness, POSINT, "100"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t alpaca_vars[] =
+{
+  AV(alpacaname, STRING, NULL),
+  AV(fuzziness, POSINT, "50"),
+  AV(n_wings, POSINT, "0"),
+  END_OF_CONFIG_VARS
+};
+
+static config_deprecation_t llama_deprecations[] = {
+  { "eats_meat", "Llamas are herbivores." },
+  {NULL,NULL}
+};
+
+static config_deprecation_t alpaca_deprecations[] = {
+  { "n_wings", "Alpacas are quadrupeds." },
+  {NULL,NULL}
+};
+
+static int clear_llama_cfg_called = 0;
+static void
+clear_llama_cfg(const config_mgr_t *mgr, void *llamacfg)
+{
+  (void)mgr;
+  llama_cfg_t *lc = llamacfg;
+  tor_free(lc->description);
+  ++clear_llama_cfg_called;
+}
+
+static config_abbrev_t llama_abbrevs[] = {
+  { "gracia", "cuteness", 0, 0 },
+  { "gentillesse", "cuteness", 0, 0 },
+  { NULL, NULL, 0, 0 },
+};
+
+static int
+legacy_validate_pasture(const void *old_, void *obj, char **msg_out)
+{
+  const pasture_cfg_t *old = old_;
+  pasture_cfg_t *p = obj;
+
+  // llamas can't find their way home if the letters are lowercase.
+  if (p->address)
+    tor_strupper(p->address);
+
+  if (old && old->address &&
+      (!p->address || strcmp(old->address, p->address))) {
+    *msg_out = tor_strdup("You can't move a pasture.");
+    return -1;
+  }
+
+  return 0;
+}
+
+static int
+validate_llama(const void *obj, char **msg_out)
+{
+  const llama_cfg_t *llama = obj;
+  tor_assert(llama->magic == 0x11aa11);
+
+  if (! llama->llamaname || strlen(llama->llamaname) == 0) {
+    *msg_out = tor_strdup("A llama has no name!?");
+    return -1;
+  }
+
+  if (strspn(llama->llamaname, "0123456789") == strlen(llama->llamaname)) {
+    *msg_out = tor_strdup("It is not a number; it is a free llama!");
+    return -1;
+  }
+
+  return 0;
+}
+
+static int
+check_transition_alpaca(const void *old_, const void *new_, char **msg_out)
+{
+  const alpaca_cfg_t *old_alpaca = old_;
+  const alpaca_cfg_t *new_alpaca = new_;
+
+  tor_assert(old_alpaca && new_alpaca);
+  tor_assert(old_alpaca->magic == 0xa15aca);
+  tor_assert(new_alpaca->magic == 0xa15aca);
+
+  if (old_alpaca->fuzziness > new_alpaca->fuzziness) {
+    *msg_out = tor_strdup("An alpaca only becomes more fuzzy over time.");
+    return -1;
+  }
+
+  return 0;
+}
+
+static int
+post_normalize_llama(void *obj, char **msg_out)
+{
+  (void)msg_out;
+  llama_cfg_t *llama = obj;
+  tor_assert(llama->magic == 0x11aa11);
+  tor_assert(llama->llamaname); // we have already checked for a NULL name.
+  tor_free(llama->description);
+  tor_asprintf(&llama->description, "A llama called %s.", llama->llamaname);
+  return 0;
+}
+
+static int
+pre_normalize_alpaca(void *obj, char **msg_out)
+{
+  (void)msg_out;
+  alpaca_cfg_t *alpaca = obj;
+  tor_assert(alpaca->magic == 0xa15aca);
+  alpaca->square_fuzziness = alpaca->fuzziness * alpaca->fuzziness;
+  return 0;
+}
+
+static const config_format_t pasture_fmt = {
+  sizeof(pasture_cfg_t),
+  {
+    "pasture_cfg_t",
+    8989,
+    offsetof(pasture_cfg_t, magic)
+  },
+  .vars = pasture_vars,
+  .has_config_suite = true,
+  .config_suite_offset = offsetof(pasture_cfg_t, subobjs),
+  .legacy_validate_fn = legacy_validate_pasture,
+};
+
+static const config_format_t llama_fmt = {
+  sizeof(llama_cfg_t),
+  {
+    "llama_cfg_t",
+    0x11aa11,
+    offsetof(llama_cfg_t, magic)
+  },
+  .vars = llama_vars,
+  .deprecations = llama_deprecations,
+  .abbrevs = llama_abbrevs,
+  .clear_fn = clear_llama_cfg,
+  .validate_fn = validate_llama,
+  .post_normalize_fn = post_normalize_llama,
+};
+
+static const config_format_t alpaca_fmt = {
+  sizeof(alpaca_cfg_t),
+  {
+    "alpaca_cfg_t",
+    0xa15aca,
+    offsetof(alpaca_cfg_t, magic)
+  },
+  .vars = alpaca_vars,
+  .deprecations = alpaca_deprecations,
+  .pre_normalize_fn = pre_normalize_alpaca,
+  .check_transition_fn = check_transition_alpaca,
+};
+
+#define LLAMA_IDX 0
+#define ALPACA_IDX 1
+
+static config_mgr_t *
+get_mgr(bool freeze)
+{
+  config_mgr_t *mgr = config_mgr_new(&pasture_fmt);
+  tt_int_op(LLAMA_IDX, OP_EQ, config_mgr_add_format(mgr, &llama_fmt));
+  tt_int_op(ALPACA_IDX, OP_EQ, config_mgr_add_format(mgr, &alpaca_fmt));
+  if (freeze)
+    config_mgr_freeze(mgr);
+  return mgr;
+
+ done:
+  config_mgr_free(mgr);
+  return NULL;
+}
+
+static void
+test_confmgr_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  smartlist_t *vars = NULL;
+  tt_ptr_op(mgr, OP_NE, NULL);
+
+  vars = config_mgr_list_vars(mgr);
+  tt_int_op(smartlist_len(vars), OP_EQ, 8); // 8 vars total.
+
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "CUTENESS"));
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "GRACIA"));
+  smartlist_free(vars);
+
+  vars = config_mgr_list_deprecated_vars(mgr); // 2 deprecated vars.
+  tt_int_op(smartlist_len(vars), OP_EQ, 2);
+  tt_assert(smartlist_contains_string(vars, "eats_meat"));
+  tt_assert(smartlist_contains_string(vars, "n_wings"));
+
+  tt_str_op("Llamas are herbivores.", OP_EQ,
+            config_find_deprecation(mgr, "EATS_MEAT"));
+  tt_str_op("Alpacas are quadrupeds.", OP_EQ,
+            config_find_deprecation(mgr, "N_WINGS"));
+
+ done:
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confmgr_magic(void *args)
+{
+  (void)args;
+  // Every time we build a manager, it is supposed to get a different magic
+  // number.  Let's test that.
+  config_mgr_t *mgr1 = get_mgr(true);
+  config_mgr_t *mgr2 = get_mgr(true);
+  config_mgr_t *mgr3 = get_mgr(true);
+
+  pasture_cfg_t *p1 = NULL, *p2 = NULL, *p3 = NULL;
+
+  tt_assert(mgr1);
+  tt_assert(mgr2);
+  tt_assert(mgr3);
+
+  p1 = config_new(mgr1);
+  p2 = config_new(mgr2);
+  p3 = config_new(mgr3);
+
+  tt_assert(p1);
+  tt_assert(p2);
+  tt_assert(p3);
+
+  // By chance, two managers get the same magic with P=2^-32.  Let's
+  // make sure that at least two of them are different, so that our
+  // odds of a false positive are 1/2^-64.
+  tt_assert((p1->magic != p2->magic) || (p2->magic != p3->magic));
+
+ done:
+  config_free(mgr1, p1);
+  config_free(mgr2, p2);
+  config_free(mgr3, p3);
+
+  config_mgr_free(mgr1);
+  config_mgr_free(mgr2);
+  config_mgr_free(mgr3);
+}
+
+static const char *simple_pasture =
+  "LLamaname hugo\n"
+  "Alpacaname daphne\n"
+  "gentillesse 42\n"
+  "address 123 Camelid ave\n";
+
+static void
+test_confmgr_parse(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, p); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_int_op(p->opentopublic, OP_EQ, 1);
+  tt_str_op(p->address, OP_EQ, "123 Camelid ave");
+
+  // We are using this API directly; modules outside confparse will, in the
+  // future, not.
+  const alpaca_cfg_t *ac = config_mgr_get_obj(mgr, p, ALPACA_IDX);
+  const llama_cfg_t *lc = config_mgr_get_obj(mgr, p, LLAMA_IDX);
+  tt_str_op(lc->llamaname, OP_EQ, "hugo");
+  tt_str_op(ac->alpacaname, OP_EQ, "daphne");
+  tt_int_op(lc->cuteness, OP_EQ, 42);
+  tt_int_op(ac->fuzziness, OP_EQ, 50);
+
+  // We set the description for the llama here, so that the clear function
+  // can clear it.  (Later we can do this in a verification function.)
+  clear_llama_cfg_called = 0;
+  llama_cfg_t *mut_lc = config_mgr_get_obj_mutable(mgr, p, LLAMA_IDX);
+  mut_lc->description = tor_strdup("A llama named Hugo.");
+  config_free(mgr, p);
+  tt_int_op(clear_llama_cfg_called, OP_EQ, 1);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_mgr_free(mgr);
+  tor_free(msg);
+}
+
+static void
+test_confmgr_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  pasture_cfg_t *defaults = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+  char *s = NULL;
+
+  config_init(mgr, p); // set defaults.
+  config_init(mgr, defaults); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  s = config_dump(mgr, defaults, p, 1, 0);
+  tt_str_op("address 123 Camelid ave\n"
+            "alpacaname daphne\n"
+            "cuteness 42\n"
+            "llamaname hugo\n", OP_EQ, s);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_free(mgr, defaults);
+  config_mgr_free(mgr);
+
+  tor_free(msg);
+  tor_free(s);
+}
+
+static pasture_cfg_t *
+parse_and_validate(config_mgr_t *mgr,
+                   const char *inp, const pasture_cfg_t *old, char **msg_out)
+{
+  pasture_cfg_t *p = config_new(mgr);
+  pasture_cfg_t *result = NULL;
+  config_line_t *lines = NULL;
+
+  config_init(mgr, p); // set defaults.
+  int r = config_get_lines(inp, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, msg_out);
+  tt_int_op(r, OP_EQ, 0);
+  tor_free(*msg_out); // sets it to NULL
+  r = config_validate(mgr, old, p, msg_out);
+  if (r < 0)
+    goto done;
+
+  tt_ptr_op(*msg_out, OP_EQ, NULL);
+  result = p;
+  p = NULL; // prevent free
+ done:
+  config_free(mgr, p);
+  config_free_lines(lines);
+  return result;
+}
+
+static void
+test_confmgr_validate(void *arg)
+{
+  (void)arg;
+  char *msg = NULL;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p_orig, *p=NULL;
+
+  p_orig = parse_and_validate(mgr, "Llamaname Quest\n"
+                                   "Address 99 camelid way\n"
+                                   "Fuzziness 8\n", NULL, &msg);
+  tt_assert(p_orig);
+
+  // Make sure normalization code was run.
+  const alpaca_cfg_t *ac0 = config_mgr_get_obj(mgr, p_orig, ALPACA_IDX);
+  const llama_cfg_t *lc0 = config_mgr_get_obj(mgr, p_orig, LLAMA_IDX);
+  tt_int_op(ac0->fuzziness, OP_EQ, 8);
+  tt_int_op(ac0->square_fuzziness, OP_EQ, 64);
+  tt_str_op(lc0->description, OP_EQ, "A llama called Quest.");
+  tt_str_op(p_orig->address, OP_EQ, "99 CAMELID WAY");
+
+  // try a bad llamaname.
+  p = parse_and_validate(mgr, "llamaname 123", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "It is not a number; it is a free llama!");
+  tor_free(msg);
+
+  // try a llamaname that would crash the post_normalize step, if it ran.
+  p = parse_and_validate(mgr, "", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "A llama has no name!?");
+  tor_free(msg);
+
+  // Verify that a transition to a less fuzzy alpaca fails.
+  p = parse_and_validate(mgr, "Llamaname Quest\n"
+                              "Address 99 camelid way\n"
+                              "Fuzziness 4\n", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "An alpaca only becomes more fuzzy over time.");
+  tor_free(msg);
+
+  // Try a transition to a more fuzzy alpaca; it should work fine.
+  p = parse_and_validate(mgr, "Llamaname Mercutio\n"
+                              // the default fuzziness is 50
+                              "Address 99 camelid way\n", p_orig, &msg);
+  tt_assert(p);
+  config_free(mgr, p);
+
+  // Verify that we can't move the pasture.
+  p = parse_and_validate(mgr, "Llamaname Montague\n"
+                              // the default fuzziness is 50
+                              "Address 99 ungulate st\n", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "You can't move a pasture.");
+
+ done:
+  config_free(mgr, p);
+  config_free(mgr, p_orig);
+  config_mgr_free(mgr);
+  tor_free(msg);
+}
+
+#define CONFMGR_TEST(name, flags)                       \
+  { #name, test_confmgr_ ## name, flags, NULL, NULL }
+
+struct testcase_t confmgr_tests[] = {
+  CONFMGR_TEST(init, 0),
+  CONFMGR_TEST(magic, 0),
+  CONFMGR_TEST(parse, 0),
+  CONFMGR_TEST(dump, 0),
+  CONFMGR_TEST(validate, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c
new file mode 100644
index 0000000000..21301ce75e
--- /dev/null
+++ b/src/test/test_confparse.c
@@ -0,0 +1,1091 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confmgt.c module that we use to parse various
+ * configuration/state file types.
+ */
+
+#define CONFMGT_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "feature/nodelist/routerset.h"
+#include "lib/confmgt/confmgt.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "lib/confmgt/unitparse.h"
+
+typedef struct test_struct_t {
+  uint32_t magic;
+  char *s;
+  char *fn;
+  int pos;
+  int i;
+  int deprecated_int;
+  uint64_t u64;
+  int interval;
+  int msec_interval;
+  uint64_t mem;
+  double dbl;
+  int boolean;
+  int autobool;
+  time_t time;
+  smartlist_t *csv;
+  int csv_interval;
+  config_line_t *lines;
+  config_line_t *mixed_lines;
+  routerset_t *routerset;
+  int hidden_int;
+  config_line_t *mixed_hidden_lines;
+
+  config_line_t *extra_lines;
+} test_struct_t;
+
+static test_struct_t test_struct_t_dummy;
+
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
+  VAR(#member, conftype, member, initvalue)
+#define OBSOLETE(varname)                       \
+  CONFIG_VAR_OBSOLETE(varname)
+
+static const config_var_t test_vars[] = {
+  V(s, STRING, "hello"),
+  V(fn, FILENAME, NULL),
+  V(pos, POSINT, NULL),
+  V(i, INT, "-10"),
+  V(deprecated_int, INT, "3"),
+  V(u64, UINT64, NULL),
+  V(interval, INTERVAL, "10 seconds"),
+  V(msec_interval, MSEC_INTERVAL, "150 msec"),
+  V(mem, MEMUNIT, "10 MB"),
+  V(dbl, DOUBLE, NULL),
+  V(boolean, BOOL, "0"),
+  V(autobool, AUTOBOOL, "auto"),
+  V(time, ISOTIME, NULL),
+  V(csv, CSV, NULL),
+  V(csv_interval, CSV_INTERVAL, "5 seconds"),
+  V(lines, LINELIST, NULL),
+  VAR("MixedLines", LINELIST_V, mixed_lines, NULL),
+  VAR("LineTypeA", LINELIST_S, mixed_lines, NULL),
+  VAR("LineTypeB", LINELIST_S, mixed_lines, NULL),
+  OBSOLETE("obsolete"),
+  {
+   .member = { .name = "routerset",
+               .type = CONFIG_TYPE_EXTENDED,
+               .type_def = &ROUTERSET_type_defn,
+               .offset = offsetof(test_struct_t, routerset),
+             },
+  },
+  VAR("__HiddenInt", POSINT, hidden_int, "0"),
+  VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL),
+  VAR("__HiddenLineA", LINELIST_S, mixed_hidden_lines, NULL),
+  VAR("VisibleLineB", LINELIST_S, mixed_hidden_lines, NULL),
+
+  END_OF_CONFIG_VARS,
+};
+
+static config_abbrev_t test_abbrevs[] = {
+  { "uint", "pos", 0, 0 },
+  { "float", "dbl", 0, 1 },
+  { NULL, NULL, 0, 0 }
+};
+
+static config_deprecation_t test_deprecation_notes[] = {
+  { "deprecated_int", "This integer is deprecated." },
+  { NULL, NULL }
+};
+
+static int
+test_validate_cb(const void *old_options, void *options, char **msg)
+{
+  (void)old_options;
+  (void)msg;
+  test_struct_t *ts = options;
+
+  if (ts->i == 0xbad) {
+    *msg = tor_strdup("bad value for i");
+    return -1;
+  }
+  return 0;
+}
+
+#define TEST_MAGIC 0x1337
+
+static const config_format_t test_fmt = {
+  .size = sizeof(test_struct_t),
+  .magic = {
+   "test_struct_t",
+   TEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  .abbrevs = test_abbrevs,
+  .deprecations = test_deprecation_notes,
+  .vars = test_vars,
+  .legacy_validate_fn = test_validate_cb,
+};
+
+/* Make sure that config_init sets everything to the right defaults. */
+static void
+test_confparse_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_init(mgr, tst);
+
+  // Make sure that options are initialized right. */
+  tt_str_op(tst->s, OP_EQ, "hello");
+  tt_ptr_op(tst->fn, OP_EQ, NULL);
+  tt_int_op(tst->pos, OP_EQ, 0);
+  tt_int_op(tst->i, OP_EQ, -10);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, 0);
+  tt_int_op(tst->interval, OP_EQ, 10);
+  tt_int_op(tst->msec_interval, OP_EQ, 150);
+  tt_u64_op(tst->mem, OP_EQ, 10 * 1024 * 1024);
+  tt_double_op(tst->dbl, OP_LT, .0000000001);
+  tt_double_op(tst->dbl, OP_GT, -0.0000000001);
+  tt_int_op(tst->boolean, OP_EQ, 0);
+  tt_int_op(tst->autobool, OP_EQ, -1);
+  tt_i64_op(tst->time, OP_EQ, 0);
+  tt_ptr_op(tst->csv, OP_EQ, NULL);
+  tt_int_op(tst->csv_interval, OP_EQ, 5);
+  tt_ptr_op(tst->lines, OP_EQ, NULL);
+  tt_ptr_op(tst->mixed_lines, OP_EQ, NULL);
+  tt_int_op(tst->hidden_int, OP_EQ, 0);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static const char simple_settings[] =
+      "s this is a \n"
+      "fn /simple/test of the\n"
+      "uint 77\n" // this is an abbrev
+      "i 3\n"
+      "u64   1000000000000  \n"
+      "interval 5 minutes \n"
+      "msec_interval 5 minutes \n"
+      "mem 10\n"
+      "dbl 6.060842\n"
+      "BOOLEAN 1\n"
+      "aUtObOOl 0\n"
+      "time 2019-06-14 13:58:51\n"
+      "csv configuration, parsing  , system  \n"
+      "csv_interval 10 seconds, 5 seconds, 10 hours\n"
+      "lines hello\n"
+      "LINES world\n"
+      "linetypea i d\n"
+      "linetypeb i c\n"
+      "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+      "__hiddenint 11\n"
+      "__hiddenlineA XYZ\n"
+      "visiblelineB ABC\n";
+
+/* Return a configuration object set up from simple_settings above. */
+static test_struct_t *
+get_simple_config(const config_mgr_t *mgr)
+{
+  test_struct_t *result = NULL;
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(simple_settings, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  result = tst;
+  tst = NULL; // prevent free
+ done:
+  tor_free(msg);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  return result;
+}
+
+/* Make sure that config_assign can parse things. */
+static void
+test_confparse_assign_simple(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  tt_str_op(tst->s, OP_EQ, "this is a");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the");
+  tt_int_op(tst->pos, OP_EQ, 77);
+  tt_int_op(tst->i, OP_EQ, 3);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, UINT64_C(1000000000000));
+  tt_int_op(tst->interval, OP_EQ, 5 * 60);
+  tt_int_op(tst->msec_interval, OP_EQ, 5 * 60 * 1000);
+  tt_u64_op(tst->mem, OP_EQ, 10);
+  tt_double_op(tst->dbl, OP_LT, 6.060843);
+  tt_double_op(tst->dbl, OP_GT, 6.060841);
+  tt_int_op(tst->boolean, OP_EQ, 1);
+  tt_int_op(tst->autobool, OP_EQ, 0);
+  tt_i64_op(tst->time, OP_EQ, 1560520731);
+  tt_ptr_op(tst->csv, OP_NE, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 3);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "configuration");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "parsing");
+  tt_str_op(smartlist_get(tst->csv, 2), OP_EQ, "system");
+  tt_int_op(tst->csv_interval, OP_EQ, 10);
+  tt_int_op(tst->hidden_int, OP_EQ, 11);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(!tst->lines->next->next);
+
+  tt_assert(tst->mixed_lines);
+  tt_str_op(tst->mixed_lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(tst->mixed_lines->value, OP_EQ, "i d");
+  tt_assert(tst->mixed_lines->next);
+  tt_str_op(tst->mixed_lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(tst->mixed_lines->next->value, OP_EQ, "i c");
+  tt_assert(!tst->mixed_lines->next->next);
+
+  tt_assert(tst->mixed_hidden_lines);
+  tt_str_op(tst->mixed_hidden_lines->key, OP_EQ, "__HiddenLineA");
+  tt_str_op(tst->mixed_hidden_lines->value, OP_EQ, "XYZ");
+  tt_assert(tst->mixed_hidden_lines->next);
+  tt_str_op(tst->mixed_hidden_lines->next->key, OP_EQ, "VisibleLineB");
+  tt_str_op(tst->mixed_hidden_lines->next->value, OP_EQ, "ABC");
+  tt_assert(!tst->mixed_hidden_lines->next->next);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an obsolete option, and make sure we get a warning. */
+static void
+test_confparse_assign_obsolete(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("obsolete option here",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("Skipping obsolete configuration option");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an deprecated option, and make sure we get a warning
+ * but the assignment works anyway. */
+static void
+test_confparse_assign_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("deprecated_int 7",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("This integer is deprecated.");
+
+  tt_int_op(tst->deprecated_int, OP_EQ, 7);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to re-assign an option name that has been depreacted in favor of
+ * another. */
+static void
+test_confparse_assign_replaced(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("float 1000\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("use 'dbl' instead.");
+
+  tt_double_op(tst->dbl, OP_GT, 999.999);
+  tt_double_op(tst->dbl, OP_LT, 1000.001);
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a linelist value with no option. */
+static void
+test_confparse_assign_emptystring(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("lines\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("has no value");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a the same option twice; make sure we get a warning. */
+static void
+test_confparse_assign_twice(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("pos 10\n"
+                           "pos 99\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("used more than once");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+typedef struct badval_test_t {
+  const char *cfg;
+  const char *expect_msg;
+} badval_test_t;
+
+/* Try to set an option and make sure that we get a failure and an expected
+ * warning. */
+static void
+test_confparse_assign_badval(void *arg)
+{
+  const badval_test_t *bt = arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(bt->cfg, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  if (! strstr(msg, bt->expect_msg)) {
+    TT_DIE(("'%s' did not contain '%s'" , msg, bt->expect_msg));
+  }
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Various arguments for badval test.
+ *
+ * Note that the expected warnings here are _very_ truncated, since we
+ * are writing these tests before a refactoring that we expect will
+ * change them.
+ */
+static const badval_test_t bv_notint = { "pos X\n", "malformed" };
+static const badval_test_t bv_negint = { "pos -10\n", "out of bounds" };
+static const badval_test_t bv_badu64 = { "u64 u64\n", "malformed" };
+static const badval_test_t bv_dbl1 = { "dbl xxx\n", "Could not convert" };
+static const badval_test_t bv_dbl2 = { "dbl 1.0 xx\n", "Could not convert" };
+static const badval_test_t bv_dbl3 = {
+   "dbl 1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl4 = {
+   "dbl 1e1000\n", "too large to express" };
+static const badval_test_t bv_dbl5 = {
+   "dbl -1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl6 = {
+   "dbl -1e1000\n", "too large to express" };
+static const badval_test_t bv_badcsvi1 =
+  { "csv_interval 10 wl\n", "malformed" };
+static const badval_test_t bv_badcsvi2 =
+  { "csv_interval cl,10\n", "malformed" };
+static const badval_test_t bv_nonoption = { "fnord 10\n", "Unknown option" };
+static const badval_test_t bv_badmem = { "mem 3 trits\n", "malformed" };
+static const badval_test_t bv_badbool = { "boolean 7\n", "Unrecognized value"};
+static const badval_test_t bv_badabool =
+  { "autobool 7\n", "Unrecognized value" };
+static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" };
+static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" };
+static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" };
+static const badval_test_t bv_big_interval =
+  { "interval 1000 months", "too large" };
+
+/* Try config_dump(), and make sure it behaves correctly */
+static void
+test_confparse_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  char *dumped = NULL;
+
+  /* Minimal version. */
+  dumped = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  /* commented */
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 1);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "# deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+ done:
+  config_free(mgr, tst);
+  tor_free(dumped);
+  config_mgr_free(mgr);
+}
+
+/* Try confparse_reset_line(), and make sure it behaves correctly */
+static void
+test_confparse_reset(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  config_reset_line(mgr, tst, "interval", 0);
+  tt_int_op(tst->interval, OP_EQ, 0);
+
+  config_reset_line(mgr, tst, "interval", 1);
+  tt_int_op(tst->interval, OP_EQ, 10);
+
+  tt_ptr_op(tst->routerset, OP_NE, NULL);
+  config_reset_line(mgr, tst, "routerset", 0);
+  tt_ptr_op(tst->routerset, OP_EQ, NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, and make sure
+ * it behaves correctly. */
+static void
+test_confparse_reassign(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *rs = NULL;
+
+  int r = config_get_lines(
+         "s eleven\n"
+         "i 12\n"
+         "lines 13\n"
+         "csv 14,15\n"
+         "routerset 127.0.0.1\n",
+         &lines, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); // unchanged
+  tt_int_op(tst->pos, OP_EQ, 77); // unchanged
+  tt_int_op(tst->i, OP_EQ, 12);
+  tt_ptr_op(tst->lines, OP_NE, NULL);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "13");
+  tt_ptr_op(tst->lines->next, OP_EQ, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 2);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "14");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "15");
+
+  rs = routerset_to_string(tst->routerset);
+  tt_str_op(rs, OP_EQ, "127.0.0.1");
+
+  // Try again with the CLEAR_FIRST and USE_DEFAULTS flags
+  r = config_assign(mgr, tst, lines,
+                    CAL_CLEAR_FIRST|CAL_USE_DEFAULTS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  // tt_ptr_op(tst->fn, OP_EQ, NULL); //XXXX why is this not cleared?
+  // tt_int_op(tst->pos, OP_EQ, 0); //XXXX why is this not cleared?
+  tt_int_op(tst->i, OP_EQ, 12);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  tor_free(rs);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, using the +foo
+ * linelist-extending syntax. */
+static void
+test_confparse_reassign_extend(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  int r = config_get_lines(
+         "+lines 13\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(tst->lines->next->next);
+  tt_str_op(tst->lines->next->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->next->value, OP_EQ, "13");
+  tt_assert(tst->lines->next->next->next == NULL);
+  config_free_lines(lines);
+
+  r = config_get_lines(
+         "/lines\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+  config_free_lines(lines);
+
+  config_free(mgr, tst);
+  tst = get_simple_config(mgr);
+  r = config_get_lines(
+         "/lines away!\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Test out confparse_get_assigned(). */
+static void
+test_confparse_get_assigned(void *arg)
+{
+  (void)arg;
+
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+
+  lines = config_get_assigned_option(mgr, tst, "I", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "i");
+  tt_str_op(lines->value, OP_EQ, "3");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "this is a");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "obsolete", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "nonesuch", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "mixedlines", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(lines->value, OP_EQ, "i d");
+  tt_assert(lines->next);
+  tt_str_op(lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->next->value, OP_EQ, "i c");
+  tt_assert(lines->next->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "linetypeb", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->value, OP_EQ, "i c");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  tor_free(tst->s);
+  tst->s = tor_strdup("Hello\nWorld");
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "\"Hello\\nWorld\"");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  config_mgr_free(mgr);
+}
+
+/* Another variant, which accepts and stores unrecognized lines.*/
+#define ETEST_MAGIC 13371337
+
+static struct_member_t extra = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(test_struct_t, extra_lines),
+};
+
+static config_format_t etest_fmt = {
+  .size = sizeof(test_struct_t),
+  .magic = {
+   "test_struct_t (with extra lines)",
+   ETEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  .abbrevs = test_abbrevs,
+  .deprecations = test_deprecation_notes,
+  .vars = test_vars,
+  .legacy_validate_fn = test_validate_cb,
+  .extra = &extra,
+};
+
+/* Try out the feature where we can store unrecognized lines and dump them
+ * again.  (State files use this.) */
+static void
+test_confparse_extra_lines(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&etest_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *dump = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(
+      "unknotty addita\n"
+      "pos 99\n"
+      "wombat knish\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->extra_lines);
+
+  dump = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dump, OP_EQ,
+      "pos 99\n"
+      "unknotty addita\n"
+      "wombat knish\n");
+
+ done:
+  tor_free(msg);
+  tor_free(dump);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_unitparse(void *args)
+{
+  (void)args;
+  /* spot-check a few memunit values. */
+  int ok = 3;
+  tt_u64_op(config_parse_memunit("100 MB", &ok), OP_EQ, 100<<20);
+  tt_assert(ok);
+  tt_u64_op(config_parse_memunit("100 TB", &ok), OP_EQ, UINT64_C(100)<<40);
+  tt_assert(ok);
+  // This is a floating-point value, but note that 1.5 can be represented
+  // precisely.
+  tt_u64_op(config_parse_memunit("1.5 MB", &ok), OP_EQ, 3<<19);
+  tt_assert(ok);
+
+  /* Try some good intervals and msec intervals */
+  tt_int_op(config_parse_interval("2 days", &ok), OP_EQ, 48*3600);
+  tt_assert(ok);
+  tt_int_op(config_parse_interval("1.5 hour", &ok), OP_EQ, 5400);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("1 minute", &ok), OP_EQ, 60);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("2 days", &ok), OP_EQ, 48*3600*1000);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("10 msec", &ok), OP_EQ, 10);
+  tt_assert(ok);
+
+  /* Try a couple of unitless values. */
+  tt_int_op(config_parse_interval("10", &ok), OP_EQ, 10);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("15.0", &ok), OP_EQ, 15);
+  tt_assert(ok);
+
+  /* u64 overflow */
+  tt_u64_op(config_parse_memunit("20000000 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // This test fails the double check as the float representing 15000000.5 TB
+  // is greater than (double) INT64_MAX
+  tt_u64_op(config_parse_memunit("15000000.5 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // 8388608.1 TB passes double check because it falls in the same float
+  // value as (double)INT64_MAX (which is 2^63) due to precision.
+  // But will fail the int check because the unsigned representation of
+  // the float, which is 2^63, is strictly greater than INT64_MAX (2^63-1)
+  tt_u64_op(config_parse_memunit("8388608.1 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+  /* negative float */
+  tt_u64_op(config_parse_memunit("-1.5 GB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+  /* i32 overflow */
+  tt_int_op(config_parse_interval("1000 months", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("4 weeks", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+
+  /* bad units */
+  tt_u64_op(config_parse_memunit("7 nybbles", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // XXXX these next two should return -1 according to the documentation.
+  tt_int_op(config_parse_interval("7 cowznofski", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("1 kalpa", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+ done:
+  ;
+}
+
+static void
+test_confparse_check_ok_fail(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  tst->pos = -10;
+  tt_assert(! config_check_ok(mgr, tst, LOG_INFO));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_vars(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_vars(mgr);
+  smartlist_t *varnames = smartlist_new();
+  char *joined = NULL;
+
+  tt_assert(vars);
+  SMARTLIST_FOREACH(vars, config_var_t *, cv,
+                    smartlist_add(varnames, (void*)cv->member.name));
+  smartlist_sort_strings(varnames);
+  joined = smartlist_join_strings(varnames, "::", 0, NULL);
+  tt_str_op(joined, OP_EQ,
+            "LineTypeA::"
+            "LineTypeB::"
+            "MixedHiddenLines::"
+            "MixedLines::"
+            "VisibleLineB::"
+            "__HiddenInt::"
+            "__HiddenLineA::"
+            "autobool::"
+            "boolean::"
+            "csv::"
+            "csv_interval::"
+            "dbl::"
+            "deprecated_int::"
+            "fn::"
+            "i::"
+            "interval::"
+            "lines::"
+            "mem::"
+            "msec_interval::"
+            "obsolete::"
+            "pos::"
+            "routerset::"
+            "s::"
+            "time::"
+            "u64");
+
+ done:
+  tor_free(joined);
+  smartlist_free(varnames);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_deprecated_vars(mgr);
+  char *joined = NULL;
+
+  tt_assert(vars);
+  smartlist_sort_strings(vars);
+  joined = smartlist_join_strings(vars, "::", 0, NULL);
+
+  tt_str_op(joined, OP_EQ, "deprecated_int");
+
+ done:
+  tor_free(joined);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_find_option_name(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+
+  // exact match
+  tt_str_op(config_find_option_name(mgr, "u64"), OP_EQ, "u64");
+  // case-insensitive match
+  tt_str_op(config_find_option_name(mgr, "S"), OP_EQ, "s");
+  tt_str_op(config_find_option_name(mgr, "linetypea"), OP_EQ, "LineTypeA");
+  // prefix match
+  tt_str_op(config_find_option_name(mgr, "deprec"), OP_EQ, "deprecated_int");
+  // explicit abbreviation
+  tt_str_op(config_find_option_name(mgr, "uint"), OP_EQ, "pos");
+  tt_str_op(config_find_option_name(mgr, "UINT"), OP_EQ, "pos");
+  // no match
+  tt_ptr_op(config_find_option_name(mgr, "absent"), OP_EQ, NULL);
+
+ done:
+  config_mgr_free(mgr);
+}
+
+#ifndef COCCI
+#define CONFPARSE_TEST(name, flags)                          \
+  { #name, test_confparse_ ## name, flags, NULL, NULL }
+
+#define BADVAL_TEST(name)                               \
+  { "badval_" #name, test_confparse_assign_badval, 0,   \
+      &passthrough_setup, (void*)&bv_ ## name }
+#endif /* !defined(COCCI) */
+
+struct testcase_t confparse_tests[] = {
+  CONFPARSE_TEST(init, 0),
+  CONFPARSE_TEST(assign_simple, 0),
+  CONFPARSE_TEST(assign_obsolete, 0),
+  CONFPARSE_TEST(assign_deprecated, 0),
+  CONFPARSE_TEST(assign_replaced, 0),
+  CONFPARSE_TEST(assign_emptystring, 0),
+  CONFPARSE_TEST(assign_twice, 0),
+  BADVAL_TEST(notint),
+  BADVAL_TEST(negint),
+  BADVAL_TEST(badu64),
+  BADVAL_TEST(dbl1),
+  BADVAL_TEST(dbl2),
+  BADVAL_TEST(dbl3),
+  BADVAL_TEST(dbl4),
+  BADVAL_TEST(dbl5),
+  BADVAL_TEST(dbl6),
+  BADVAL_TEST(badcsvi1),
+  BADVAL_TEST(badcsvi2),
+  BADVAL_TEST(nonoption),
+  BADVAL_TEST(badmem),
+  BADVAL_TEST(badbool),
+  BADVAL_TEST(badabool),
+  BADVAL_TEST(badtime),
+  BADVAL_TEST(virt),
+  BADVAL_TEST(rs),
+  BADVAL_TEST(big_interval),
+  CONFPARSE_TEST(dump, 0),
+  CONFPARSE_TEST(reset, 0),
+  CONFPARSE_TEST(reassign, 0),
+  CONFPARSE_TEST(reassign_extend, 0),
+  CONFPARSE_TEST(get_assigned, 0),
+  CONFPARSE_TEST(extra_lines, 0),
+  CONFPARSE_TEST(unitparse, 0),
+  CONFPARSE_TEST(check_ok_fail, 0),
+  CONFPARSE_TEST(list_vars, 0),
+  CONFPARSE_TEST(list_deprecated, 0),
+  CONFPARSE_TEST(find_option_name, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_connection.c b/src/test/test_connection.c
index 7234d01627..b1252864f5 100644
--- a/src/test/test_connection.c
+++ b/src/test/test_connection.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -10,6 +10,7 @@
 #include "core/or/or.h"
 #include "test/test.h"
 
+#include "app/config/or_options_st.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "feature/hs/hs_common.h"
@@ -312,6 +313,25 @@ test_conn_download_status_teardown(const struct testcase_t *tc, void *arg)
   return rv;
 }
 
+static void *
+test_conn_proxy_connect_setup(const struct testcase_t *tc)
+{
+  return test_conn_get_proxy_or_connection(*(unsigned int *)tc->setup_data);
+}
+
+static int
+test_conn_proxy_connect_teardown(const struct testcase_t *tc, void *arg)
+{
+  (void)tc;
+  or_connection_t *conn = arg;
+
+  tt_assert(conn);
+  assert_connection_ok(&conn->base_, time(NULL));
+
+ done:
+  return 1;
+}
+
 /* Like connection_ap_make_link(), but does much less */
 static connection_t *
 test_conn_get_linked_connection(connection_t *l_conn, uint8_t state)
@@ -360,6 +380,10 @@ static struct testcase_setup_t test_conn_download_status_st = {
   test_conn_download_status_setup, test_conn_download_status_teardown
 };
 
+static struct testcase_setup_t test_conn_proxy_connect_st = {
+  test_conn_proxy_connect_setup, test_conn_proxy_connect_teardown
+};
+
 static void
 test_conn_get_basic(void *arg)
 {
@@ -789,6 +813,64 @@ test_conn_download_status(void *arg)
   /* the teardown function removes all the connections in the global list*/;
 }
 
+static void
+test_conn_https_proxy_connect(void *arg)
+{
+  size_t sz;
+  char *buf = NULL;
+  or_connection_t *conn = arg;
+
+  MOCK(connection_or_change_state, mock_connection_or_change_state);
+
+  tt_int_op(conn->base_.proxy_state, OP_EQ, PROXY_HTTPS_WANT_CONNECT_OK);
+
+  buf = buf_get_contents(conn->base_.outbuf, &sz);
+  tt_str_op(buf, OP_EQ, "CONNECT 127.0.0.1:12345 HTTP/1.0\r\n\r\n");
+
+ done:
+  UNMOCK(connection_or_change_state);
+  tor_free(buf);
+}
+
+static int handshake_start_called = 0;
+
+static int
+handshake_start(or_connection_t *conn, int receiving)
+{
+  (void)receiving;
+
+  tor_assert(conn);
+
+  handshake_start_called = 1;
+  return 0;
+}
+
+static void
+test_conn_haproxy_proxy_connect(void *arg)
+{
+  size_t sz;
+  char *buf = NULL;
+  or_connection_t *conn = arg;
+
+  MOCK(connection_or_change_state, mock_connection_or_change_state);
+  MOCK(connection_tls_start_handshake, handshake_start);
+
+  tt_int_op(conn->base_.proxy_state, OP_EQ, PROXY_HAPROXY_WAIT_FOR_FLUSH);
+
+  buf = buf_get_contents(conn->base_.outbuf, &sz);
+  tt_str_op(buf, OP_EQ, "PROXY TCP4 0.0.0.0 127.0.0.1 0 12345\r\n");
+
+  connection_or_finished_flushing(conn);
+
+  tt_int_op(conn->base_.proxy_state, OP_EQ, PROXY_CONNECTED);
+  tt_int_op(handshake_start_called, OP_EQ, 1);
+
+ done:
+  UNMOCK(connection_or_change_state);
+  UNMOCK(connection_tls_start_handshake);
+  tor_free(buf);
+}
+
 static node_t test_node;
 
 static node_t *
@@ -882,22 +964,39 @@ test_failed_orconn_tracker(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define CONNECTION_TESTCASE(name, fork, setup)                           \
   { #name, test_conn_##name, fork, &setup, NULL }
 
-/* where arg is a string. */
-#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg)                  \
-  { #name "_" arg, test_conn_##name, fork, &setup, (void *)arg }
+#define STR(x) #x
+/* where arg is an expression (constant, variable, compound expression) */
+#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg)                 \
+  { #name "_" STR(x),                                                   \
+      test_conn_##name,                                                 \
+      fork,                                                             \
+      &setup,                                                           \
+      (void *)arg }
+#endif /* !defined(COCCI) */
+
+static const unsigned int PROXY_CONNECT_ARG = PROXY_CONNECT;
+static const unsigned int PROXY_HAPROXY_ARG = PROXY_HAPROXY;
 
 struct testcase_t connection_tests[] = {
   CONNECTION_TESTCASE(get_basic, TT_FORK, test_conn_get_basic_st),
   CONNECTION_TESTCASE(get_rend,  TT_FORK, test_conn_get_rend_st),
   CONNECTION_TESTCASE(get_rsrc,  TT_FORK, test_conn_get_rsrc_st),
+
   CONNECTION_TESTCASE_ARG(download_status,  TT_FORK,
                           test_conn_download_status_st, "microdesc"),
   CONNECTION_TESTCASE_ARG(download_status,  TT_FORK,
                           test_conn_download_status_st, "ns"),
-//CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair),
+
+  CONNECTION_TESTCASE_ARG(https_proxy_connect,   TT_FORK,
+                          test_conn_proxy_connect_st, &PROXY_CONNECT_ARG),
+  CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, TT_FORK,
+                          test_conn_proxy_connect_st, &PROXY_HAPROXY_ARG),
+
+  //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair),
   { "failed_orconn_tracker", test_failed_orconn_tracker, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_connection.h b/src/test/test_connection.h
index 47a5599e5f..bf327c0a3d 100644
--- a/src/test/test_connection.h
+++ b/src/test/test_connection.h
@@ -1,13 +1,18 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_CONNECTION_H
+#define TOR_TEST_CONNECTION_H
+
 /** Some constants used by test_connection and helpers */
 #define TEST_CONN_FAMILY        (AF_INET)
 #define TEST_CONN_ADDRESS       "127.0.0.1"
+#define TEST_CONN_ADDRESS_2     "127.0.0.2"
 #define TEST_CONN_PORT          (12345)
 #define TEST_CONN_ADDRESS_PORT  "127.0.0.1:12345"
-#define TEST_CONN_FD_INIT 50
+#define TEST_CONN_FD_INIT       0x10000
 
 void test_conn_lookup_addr_helper(const char *address,
                                   int family, tor_addr_t *addr);
 
+#endif /* !defined(TOR_TEST_CONNECTION_H) */
diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c
index 095ff09350..c805774fa3 100644
--- a/src/test/test_conscache.c
+++ b/src/test/test_conscache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c
index 682ba5b970..242e2f7818 100644
--- a/src/test/test_consdiff.c
+++ b/src/test/test_consdiff.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2014, Daniel Martí
- * Copyright (c) 2014-2019, The Tor Project, Inc. */
+ * Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFF_PRIVATE
@@ -14,6 +14,39 @@
 #define tt_str_eq_line(a,b) \
   tt_assert(line_str_eq((b),(a)))
 
+static int
+consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area)
+{
+  size_t len = strlen(s);
+  return consensus_split_lines(out, s, len, area);
+}
+
+static int
+consensus_compute_digest_(const char *cons,
+                          consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
+static int
+consensus_compute_digest_as_signed_(const char *cons,
+                                    consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest_as_signed(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
 static void
 test_consdiff_smartlist_slice(void *arg)
 {
@@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg)
 
   /* Create a regular smartlist. */
   (void)arg;
-  consensus_split_lines(sl, "a\nd\nc\na\nb\n", area);
+  consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area);
 
   /* See that smartlist_slice_string_pos respects the bounds of the slice. */
   sls = smartlist_slice(sl, 2, 5);
@@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg)
   int e_lengths2[] = { 0, 1, 1, 2, 3, 4 };
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area);
-  consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area);
+  consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area);
+  consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area);
-  consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area);
-  consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area);
-  consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area);
+  consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area);
+  consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area);
+  consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area);
+  consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area);
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
   sls3 = smartlist_slice(sl3, 0, -1);
@@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\nb\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   /* Length of sls1 is 0. */
   sls1 = smartlist_slice(sl1, 0, 0);
@@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\na\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\na\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg)
   tt_assert(!bitarray_is_set(changed2, 3));
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "a\nb\na\nb\n", area);
+  consensus_split_lines_(sl2, "a\nb\na\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg)
   bitarray_clear(changed1, 3);
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "b\nb\nb\nb\n", area);
+  consensus_split_lines_(sl2, "b\nb\nb\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg)
   /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
-  consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(7, OP_EQ, smartlist_len(diff));
@@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg)
 
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "B\n", area);
-  consensus_split_lines(cons2, "A\nB\n", area);
+  consensus_split_lines_(cons1, "B\n", area);
+  consensus_split_lines_(cons2, "A\nB\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(3, OP_EQ, smartlist_len(diff));
@@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg)
   diff = smartlist_new();
   setup_capture_of_logs(LOG_WARN);
 
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
 
   /* Command without range. */
   smartlist_add_linecpy(diff, area, "a");
@@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_clear(diff);
 
   /* Test appending text, 'a'. */
-  consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
+  consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(8, OP_EQ, smartlist_len(cons2));
@@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test deleting text, 'd'. */
-  consensus_split_lines(diff, "4d\n1,2d\n", area);
+  consensus_split_lines_(diff, "4d\n1,2d\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(2, OP_EQ, smartlist_len(cons2));
@@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test changing text, 'c'. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(5, OP_EQ, smartlist_len(cons2));
@@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test 'a', 'd' and 'c' together. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(6, OP_EQ, smartlist_len(cons2));
@@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg)
       );
 
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons2_str, &digests2));
+      consensus_compute_digest_(cons2_str, &digests2));
 
-  consensus_split_lines(cons1, cons1_str, area);
-  consensus_split_lines(cons2, cons2_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
+  consensus_split_lines_(cons2, cons2_str, area);
 
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_EQ, diff);
@@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   smartlist_clear(cons1);
-  consensus_split_lines(cons1, cons1_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(11, OP_EQ, smartlist_len(diff));
@@ -991,13 +1024,13 @@ test_consdiff_apply_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons1_str, &digests1));
-  consensus_split_lines(cons1, cons1_str, area);
+      consensus_compute_digest_(cons1_str, &digests1));
+  consensus_split_lines_(cons1, cons1_str, area);
 
   /* diff doesn't have enough lines. */
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("too short")
+  expect_single_log_msg_containing("too short");
 
   /* first line doesn't match format-version string. */
   smartlist_add_linecpy(diff, area, "foo-bar");
@@ -1005,7 +1038,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("format is not known")
+  expect_single_log_msg_containing("format is not known");
 
   /* The first word of the second header line is not "hash". */
   smartlist_clear(diff);
@@ -1015,7 +1048,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("does not include the necessary digests")
+  expect_single_log_msg_containing("does not include the necessary digests");
 
   /* Wrong number of words after "hash". */
   smartlist_clear(diff);
@@ -1024,7 +1057,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("does not include the necessary digests")
+  expect_single_log_msg_containing("does not include the necessary digests");
 
   /* base16 digests do not have the expected length. */
   smartlist_clear(diff);
@@ -1034,7 +1067,7 @@ test_consdiff_apply_diff(void *arg)
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
   expect_single_log_msg_containing("includes base16-encoded digests of "
-                                   "incorrect size")
+                                   "incorrect size");
 
   /* base16 digests contain non-base16 characters. */
   smartlist_clear(diff);
@@ -1045,7 +1078,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("includes malformed digests")
+  expect_single_log_msg_containing("includes malformed digests");
 
   /* Invalid ed diff.
    * As tested in apply_ed_diff, but check that apply_diff does return NULL if
@@ -1062,7 +1095,7 @@ test_consdiff_apply_diff(void *arg)
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
   expect_single_log_msg_containing("because an ed command was missing a line "
-                                   "number")
+                                   "number");
 
   /* Base consensus doesn't match its digest as found in the diff. */
   smartlist_clear(diff);
@@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = {
   CONSDIFF_LEGACY(apply_diff),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c
index 254a5ba5d0..4bec6baeb0 100644
--- a/src/test/test_consdiffmgr.c
+++ b/src/test/test_consdiffmgr.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFFMGR_PRIVATE
@@ -21,6 +21,23 @@
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
+static char *
+consensus_diff_apply_(const char *c, const char *d)
+{
+  size_t c_len = strlen(c);
+  size_t d_len = strlen(d);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  char *c_tmp = tor_memdup(c, c_len);
+  char *d_tmp = tor_memdup(d, d_len);
+  char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len);
+  tor_free(c_tmp);
+  tor_free(d_tmp);
+  return result;
+}
+
 // ============================== Setup/teardown the consdiffmgr
 // These functions get run before/after each test in this module
 
@@ -102,7 +119,7 @@ typedef struct fake_work_queue_ent_t {
   void (*reply_fn)(void *);
   void *arg;
 } fake_work_queue_ent_t;
-static struct workqueue_entry_s *
+static struct workqueue_entry_t *
 mock_cpuworker_queue_work(workqueue_priority_t prio,
                           enum workqueue_reply_t (*fn)(void *, void *),
                           void (*reply_fn)(void *),
@@ -118,7 +135,7 @@ mock_cpuworker_queue_work(workqueue_priority_t prio,
   ent->reply_fn = reply_fn;
   ent->arg = arg;
   smartlist_add(fake_cpuworker_queue, ent);
-  return (struct workqueue_entry_s *)ent;
+  return (struct workqueue_entry_t *)ent;
 }
 static int
 mock_cpuworker_run_work(void)
@@ -153,7 +170,8 @@ lookup_diff_from(consensus_cache_entry_t **out,
                  const char *str1)
 {
   uint8_t digest[DIGEST256_LEN];
-  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+  if (router_get_networkstatus_v3_sha3_as_signed(digest,
+                                                 str1, strlen(str1))<0) {
     TT_FAIL(("Unable to compute sha3-as-signed"));
     return CONSDIFF_NOT_FOUND;
   }
@@ -175,14 +193,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav,
 
   consensus_cache_entry_incref(ent);
   size_t size;
-  char *diff_string = NULL;
-  int r = uncompress_or_copy(&diff_string, &size, ent);
+  const char *diff_string = NULL;
+  char *diff_owned = NULL;
+  int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent);
   consensus_cache_entry_decref(ent);
   if (diff_string == NULL || r < 0)
     return -1;
 
-  char *applied = consensus_diff_apply(str1, diff_string);
-  tor_free(diff_string);
+  char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size);
+  tor_free(diff_owned);
   if (applied == NULL)
     return -1;
 
@@ -282,7 +301,8 @@ test_consdiffmgr_add(void *arg)
   (void) arg;
   time_t now = approx_time();
 
-  char *body = NULL;
+  const char *body = NULL;
+  char *body_owned = NULL;
 
   consensus_cache_entry_t *ent = NULL;
   networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
@@ -324,7 +344,7 @@ test_consdiffmgr_add(void *arg)
   tt_assert(ent);
   consensus_cache_entry_incref(ent);
   size_t s;
-  r = uncompress_or_copy(&body, &s, ent);
+  r = uncompress_or_set_ptr(&body, &s, &body_owned, ent);
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(s, OP_EQ, 4);
   tt_mem_op(body, OP_EQ, "quux", 4);
@@ -337,7 +357,7 @@ test_consdiffmgr_add(void *arg)
   networkstatus_vote_free(ns_tmp);
   teardown_capture_of_logs();
   consensus_cache_entry_decref(ent);
-  tor_free(body);
+  tor_free(body_owned);
 }
 
 static void
@@ -370,7 +390,8 @@ test_consdiffmgr_make_diffs(void *arg)
   ns = fake_ns_new(FLAV_MICRODESC, now-3600);
   md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
   r = consdiffmgr_add_consensus(md_ns_body, ns);
-  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body,
+                                             strlen(md_ns_body));
   networkstatus_vote_free(ns);
   tt_int_op(r, OP_EQ, 0);
 
@@ -414,7 +435,7 @@ test_consdiffmgr_make_diffs(void *arg)
   r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
   tt_int_op(r, OP_EQ, 0);
   diff_text = tor_memdup_nulterm(diff_body, diff_size);
-  applied = consensus_diff_apply(md_ns_body, diff_text);
+  applied = consensus_diff_apply_(md_ns_body, diff_text);
   tt_assert(applied);
   tt_str_op(applied, OP_EQ, md_ns_body_2);
 
diff --git a/src/test/test_containers.c b/src/test/test_containers.c
index aedd2f7a89..6072148d1b 100644
--- a/src/test/test_containers.c
+++ b/src/test/test_containers.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -96,6 +96,30 @@ test_container_smartlist_basic(void *arg)
   tor_free(v555);
 }
 
+/** Test SMARTLIST_FOREACH_REVERSE_BEGIN loop macro */
+static void
+test_container_smartlist_foreach_reverse(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  int i;
+
+  (void) arg;
+
+  /* Add integers to smartlist in increasing order */
+  for (i=0;i<100;i++) {
+    smartlist_add(sl, (void*)(uintptr_t)i);
+  }
+
+  /* Pop them out in reverse and test their value */
+  SMARTLIST_FOREACH_REVERSE_BEGIN(sl, void*, k) {
+    i--;
+    tt_ptr_op(k, OP_EQ, (void*)(uintptr_t)i);
+  } SMARTLIST_FOREACH_END(k);
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for smartlist-of-strings functionality. */
 static void
 test_container_smartlist_strings(void *arg)
@@ -582,6 +606,66 @@ test_container_smartlist_ints_eq(void *arg)
   smartlist_free(sl2);
 }
 
+static void
+test_container_smartlist_grow(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  int i;
+  const char *s[] = { "first", "2nd", "3rd" };
+
+  /* case 1: starting from empty. */
+  smartlist_grow(sl, 10);
+  tt_int_op(10, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 10; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 2: starting with a few elements, probably not reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 5);
+  tt_int_op(5, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  tt_ptr_op(smartlist_get(sl, 3), OP_EQ, NULL);
+  tt_ptr_op(smartlist_get(sl, 4), OP_EQ, NULL);
+
+  /* case 3: starting with a few elements, but reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 100);
+  tt_int_op(100, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  for (i = 3; i < 100; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 4: shrinking doesn't happen. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 1);
+  tt_int_op(3, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for bitarray code */
 static void
 test_container_bitarray(void *arg)
@@ -922,6 +1006,10 @@ test_container_smartlist_remove(void *arg)
   tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]);
   tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]);
   tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]);
+  /* Ordinary code should never look at this pointer; we're doing it here
+   * to make sure that we really cleared the pointer we removed.
+   */
+  tt_ptr_op(sl->list[4], OP_EQ, NULL);
 
  done:
   smartlist_free(sl);
@@ -1281,12 +1369,14 @@ test_container_smartlist_strings_eq(void *arg)
 struct testcase_t container_tests[] = {
   CONTAINER_LEGACY(smartlist_basic),
   CONTAINER_LEGACY(smartlist_strings),
+  CONTAINER_LEGACY(smartlist_foreach_reverse),
   CONTAINER_LEGACY(smartlist_overlap),
   CONTAINER_LEGACY(smartlist_digests),
   CONTAINER_LEGACY(smartlist_join),
   CONTAINER_LEGACY(smartlist_pos),
   CONTAINER(smartlist_remove, 0),
   CONTAINER(smartlist_ints_eq, 0),
+  CONTAINER(smartlist_grow, 0),
   CONTAINER_LEGACY(bitarray),
   CONTAINER_LEGACY(digestset),
   CONTAINER_LEGACY(strmap),
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index 5b406e159b..a69ec17db8 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -1,12 +1,19 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define CONTROL_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
 #include "core/or/or.h"
+#include "app/config/config.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "feature/client/bridges.h"
 #include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/entrynodes.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dircache/dirserv.h"
 #include "feature/hs/hs_common.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/rend/rendservice.h"
@@ -15,48 +22,257 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "lib/net/resolve.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
 #include "feature/dirclient/download_status_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/node_st.h"
 
+typedef struct {
+  const char *input;
+  const char *expected_parse;
+  const char *expected_error;
+} parser_testcase_t;
+
+typedef struct {
+  const control_cmd_syntax_t *syntax;
+  size_t n_testcases;
+  const parser_testcase_t *testcases;
+} parse_test_params_t;
+
+static char *
+control_cmd_dump_args(const control_cmd_args_t *result)
+{
+  buf_t *buf = buf_new();
+  buf_add_string(buf, "{ args=[");
+  if (result->args) {
+    if (smartlist_len(result->args)) {
+        buf_add_string(buf, " ");
+    }
+    SMARTLIST_FOREACH_BEGIN(result->args, const char *, s) {
+      const bool last = (s_sl_idx == smartlist_len(result->args)-1);
+      buf_add_printf(buf, "%s%s ",
+                     escaped(s),
+                     last ? "" : ",");
+    } SMARTLIST_FOREACH_END(s);
+  }
+  buf_add_string(buf, "]");
+  if (result->cmddata) {
+    buf_add_string(buf, ", obj=");
+    buf_add_string(buf, escaped(result->cmddata));
+  }
+  if (result->kwargs) {
+    buf_add_string(buf, ", { ");
+    const config_line_t *line;
+    for (line = result->kwargs; line; line = line->next) {
+      const bool last = (line->next == NULL);
+      buf_add_printf(buf, "%s=%s%s ", line->key, escaped(line->value),
+                     last ? "" : ",");
+    }
+    buf_add_string(buf, "}");
+  }
+  buf_add_string(buf, " }");
+
+  char *encoded = buf_extract(buf, NULL);
+  buf_free(buf);
+  return encoded;
+}
+
+static void
+test_controller_parse_cmd(void *arg)
+{
+  const parse_test_params_t *params = arg;
+  control_cmd_args_t *result = NULL;
+  char *error = NULL;
+  char *encoded = NULL;
+
+  for (size_t i = 0; i < params->n_testcases; ++i) {
+    const parser_testcase_t *t = &params->testcases[i];
+    result = control_cmd_parse_args("EXAMPLE",
+                                    params->syntax,
+                                    strlen(t->input),
+                                    t->input,
+                                    &error);
+    // A valid test should expect exactly one parse or error.
+    tt_int_op((t->expected_parse == NULL), OP_NE,
+              (t->expected_error == NULL));
+    // We get a result or an error, not both.
+    tt_int_op((result == NULL), OP_EQ, (error != NULL));
+    // We got the one we expected.
+    tt_int_op((result == NULL), OP_EQ, (t->expected_parse == NULL));
+
+    if (result) {
+      encoded = control_cmd_dump_args(result);
+      tt_str_op(encoded, OP_EQ, t->expected_parse);
+    } else {
+      tt_str_op(error, OP_EQ, t->expected_error);
+    }
+
+    tor_free(error);
+    tor_free(encoded);
+    control_cmd_args_free(result);
+  }
+
+ done:
+  tor_free(error);
+  tor_free(encoded);
+  control_cmd_args_free(result);
+}
+
+#ifndef COCCI
+#define OK(inp, out) \
+  { inp "\r\n", out, NULL }
+#define ERR(inp, err) \
+  { inp "\r\n", NULL, err }
+
+#define TESTPARAMS(syntax, array)                \
+  { &syntax,                                     \
+      ARRAY_LENGTH(array),                       \
+      array }
+#endif /* !defined(COCCI) */
+
+static const parser_testcase_t one_to_three_tests[] = {
+   ERR("", "Need at least 1 argument(s)"),
+   ERR("   \t", "Need at least 1 argument(s)"),
+   OK("hello", "{ args=[ \"hello\" ] }"),
+   OK("hello world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world      ", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello there world", "{ args=[ \"hello\", \"there\", \"world\" ] }"),
+   ERR("why hello there world", "Cannot accept more than 3 argument(s)"),
+   ERR("hello\r\nworld.\r\n.", "Unexpected body"),
+};
+
+static const control_cmd_syntax_t one_to_three_syntax = {
+   .min_args=1, .max_args=3
+};
+
+static const parse_test_params_t parse_one_to_three_params =
+  TESTPARAMS( one_to_three_syntax, one_to_three_tests );
+
+// =
+static const parser_testcase_t no_args_one_obj_tests[] = {
+  ERR("Hi there!\r\n.", "Cannot accept more than 0 argument(s)"),
+  ERR("", "Empty body"),
+  OK("\r\n", "{ args=[], obj=\"\\n\" }"),
+  OK("\r\nHello world\r\n", "{ args=[], obj=\"Hello world\\n\\n\" }"),
+  OK("\r\nHello\r\nworld\r\n", "{ args=[], obj=\"Hello\\nworld\\n\\n\" }"),
+  OK("\r\nHello\r\n..\r\nworld\r\n",
+     "{ args=[], obj=\"Hello\\n.\\nworld\\n\\n\" }"),
+};
+static const control_cmd_syntax_t no_args_one_obj_syntax = {
+   .min_args=0, .max_args=0,
+   .want_cmddata=true,
+};
+static const parse_test_params_t parse_no_args_one_obj_params =
+  TESTPARAMS( no_args_one_obj_syntax, no_args_one_obj_tests );
+
+static const parser_testcase_t no_args_kwargs_tests[] = {
+  OK("", "{ args=[] }"),
+  OK(" ", "{ args=[] }"),
+  OK("hello there=world", "{ args=[], { hello=\"\", there=\"world\" } }"),
+  OK("hello there=world today",
+     "{ args=[], { hello=\"\", there=\"world\", today=\"\" } }"),
+  ERR("=Foo", "Cannot parse keyword argument(s)"),
+};
+static const control_cmd_syntax_t no_args_kwargs_syntax = {
+   .min_args=0, .max_args=0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_VALS
+};
+static const parse_test_params_t parse_no_args_kwargs_params =
+  TESTPARAMS( no_args_kwargs_syntax, no_args_kwargs_tests );
+
+static const char *one_arg_kwargs_allow_keywords[] = {
+  "Hello", "world", NULL
+};
+static const parser_testcase_t one_arg_kwargs_tests[] = {
+  ERR("", "Need at least 1 argument(s)"),
+  OK("Hi", "{ args=[ \"Hi\" ] }"),
+  ERR("hello there=world", "Unrecognized keyword argument \"there\""),
+  OK("Hi HELLO=foo", "{ args=[ \"Hi\" ], { HELLO=\"foo\" } }"),
+  OK("Hi world=\"bar baz\" hello  ",
+     "{ args=[ \"Hi\" ], { world=\"bar baz\", hello=\"\" } }"),
+};
+static const control_cmd_syntax_t one_arg_kwargs_syntax = {
+   .min_args=1, .max_args=1,
+   .accept_keywords=true,
+   .allowed_keywords=one_arg_kwargs_allow_keywords,
+   .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+static const parse_test_params_t parse_one_arg_kwargs_params =
+  TESTPARAMS( one_arg_kwargs_syntax, one_arg_kwargs_tests );
+
+static char *reply_str = NULL;
+/* Mock for control_write_reply that copies the string for inspection
+ * by tests */
+static void
+mock_control_write_reply(control_connection_t *conn, int code, int c,
+                                const char *s)
+{
+  (void)conn;
+  (void)code;
+  (void)c;
+  tor_free(reply_str);
+  reply_str = tor_strdup(s);
+}
+
 static void
 test_add_onion_helper_keyarg_v3(void *arg)
 {
   int ret, hs_version;
   add_onion_secret_key_t pk;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   const char *key_new_alg = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit ED25519-V3 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
+  /* Test "BEST" key generation (Assumes BEST = ED25519-V3). */
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, NULL);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
+  tt_assert(key_new_blob);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
+
   /* Test discarding the private key. */
+  tor_free(reply_str);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
@@ -76,9 +292,10 @@ test_add_onion_helper_keyarg_v3(void *arg)
 
     tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk);
     tt_assert(key_blob);
+    tor_free(reply_str);
     ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg,
                                   &key_new_blob, &pk, &hs_version,
-                                  &err_msg);
+                                  NULL);
     tor_free(key_blob);
     tt_int_op(ret, OP_EQ, 0);
     tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
@@ -86,7 +303,7 @@ test_add_onion_helper_keyarg_v3(void *arg)
     tt_mem_op(pk.v3, OP_EQ, hex_sk, 64);
     tt_ptr_op(key_new_alg, OP_EQ, NULL);
     tt_ptr_op(key_new_blob, OP_EQ, NULL);
-    tt_ptr_op(err_msg, OP_EQ, NULL);
+    tt_ptr_op(reply_str, OP_EQ, NULL);
     tor_free(pk.v3); pk.v3 = NULL;
     tor_free(key_new_blob);
   }
@@ -94,7 +311,8 @@ test_add_onion_helper_keyarg_v3(void *arg)
  done:
   tor_free(pk.v3);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -105,72 +323,61 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_t *pk1 = NULL;
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   char *encoded = NULL;
   char *arg_str = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit RSA1024 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
-
-  /* Test "BEST" key generation (Assumes BEST = RSA1024). */
-  crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
-  tt_assert(pk.v2);
-  tt_str_op(key_new_alg, OP_EQ, "RSA1024");
-  tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test discarding the private key. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+  ret = add_onion_helper_keyarg("NEW:RSA1024", 1, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test generating a invalid key type. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a RSA1024 key. */
-  tor_free(err_msg);
+  tor_free(reply_str);
   pk1 = pk_generate(0);
   tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded));
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0);
 
   /* Test loading a invalid key type. */
@@ -179,36 +386,37 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_asprintf(&arg_str, "RSA512:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a invalid key. */
   tor_free(arg_str);
   crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(err_msg);
+  tor_free(reply_str);
   encoded[strlen(encoded)/2] = '\0';
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
  done:
   crypto_pk_free(pk1);
   crypto_pk_free(pk.v2);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
   tor_free(encoded);
   tor_free(arg_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -362,49 +570,52 @@ static void
 test_add_onion_helper_clientauth(void *arg)
 {
   rend_authorized_client_t *client = NULL;
-  char *err_msg = NULL;
   int created = 0;
 
   (void)arg;
 
+  MOCK(control_write_reply, mock_control_write_reply);
   /* Test "ClientName" only. */
-  client = add_onion_helper_clientauth("alice", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice", &created, NULL);
   tt_assert(client);
   tt_assert(created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test "ClientName:Blob" */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B",
-                                       &created, &err_msg);
+                                       &created, NULL);
   tt_assert(client);
   tt_assert(!created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test invalid client names */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("no*asterisks*allowed", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid auth cookie */
-  client = add_onion_helper_clientauth("alice:12345", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice:12345", &created, NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid syntax */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
  done:
   rend_authorized_client_free(client);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 /* Mocks and data/variables used for GETINFO download status tests */
@@ -1485,6 +1696,138 @@ test_download_status_bridge(void *arg)
   return;
 }
 
+/** Mock cached consensus */
+static cached_dir_t *mock_ns_consensus_cache;
+static cached_dir_t *mock_microdesc_consensus_cache;
+
+/**  Mock the function that retrieves consensus from cache. These use a
+ * global variable so that they can be cleared from within the test.
+ * The actual code retains the pointer to the consensus data, but
+ * we are doing this here, to prevent memory leaks
+ * from within the tests */
+static cached_dir_t *
+mock_dirserv_get_consensus(const char *flavor_name)
+{
+  if (!strcmp(flavor_name, "ns")) {
+    mock_ns_consensus_cache = tor_malloc_zero(sizeof(cached_dir_t));
+    mock_ns_consensus_cache->dir = tor_strdup("mock_ns_consensus");
+    return mock_ns_consensus_cache;
+  } else {
+    mock_microdesc_consensus_cache = tor_malloc_zero(sizeof(cached_dir_t));
+    mock_microdesc_consensus_cache->dir = tor_strdup(
+                                            "mock_microdesc_consensus");
+    return mock_microdesc_consensus_cache;
+  }
+}
+
+/** Mock the function that retrieves consensuses
+ *  from a files in the directory. */
+static tor_mmap_t *
+mock_tor_mmap_file(const char* filename)
+{
+  tor_mmap_t *res;
+  res = tor_malloc_zero(sizeof(tor_mmap_t));
+  if (strstr(filename, "cached-consensus") != NULL) {
+    res->data = "mock_ns_consensus";
+  } else if (strstr(filename, "cached-microdesc-consensus") != NULL) {
+    res->data = "mock_microdesc_consensus";
+  } else {
+    res->data = ".";
+  }
+  res->size = strlen(res->data);
+  return res;
+}
+
+/** Mock the function that clears file data
+ * loaded into the memory */
+static int
+mock_tor_munmap_file(tor_mmap_t *handle)
+{
+  tor_free(handle);
+  return 0;
+}
+
+static void
+test_getinfo_helper_current_consensus_from_file(void *arg)
+{
+  /* We just need one of these to pass, it doesn't matter what's in it */
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+
+  MOCK(tor_mmap_file, mock_tor_mmap_file);
+  MOCK(tor_munmap_file, mock_tor_munmap_file);
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_ns_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tor_free(answer);
+  errmsg = NULL;
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus-microdesc",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_microdesc_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  errmsg = NULL;
+
+ done:
+  tor_free(answer);
+  UNMOCK(tor_mmap_file);
+  UNMOCK(tor_munmap_file);
+  return;
+}
+
+static void
+test_getinfo_helper_current_consensus_from_cache(void *arg)
+{
+  /* We just need one of these to pass, it doesn't matter what's in it */
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+  or_options_t *options = get_options_mutable();
+  options->FetchUselessDescriptors = 1;
+  MOCK(dirserv_get_consensus, mock_dirserv_get_consensus);
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_ns_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tor_free(answer);
+  tor_free(mock_ns_consensus_cache->dir);
+  tor_free(mock_ns_consensus_cache);
+  errmsg = NULL;
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus-microdesc",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_microdesc_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tor_free(mock_microdesc_consensus_cache->dir);
+  tor_free(answer);
+  errmsg = NULL;
+
+ done:
+  options->FetchUselessDescriptors = 0;
+  tor_free(answer);
+  tor_free(mock_microdesc_consensus_cache);
+  UNMOCK(dirserv_get_consensus);
+  return;
+}
+
 /** Set timeval to a mock date and time. This is necessary
  * to make tor_gettimeofday() mockable. */
 static void
@@ -1543,7 +1886,7 @@ test_current_time(void *arg)
 static size_t n_nodelist_get_list = 0;
 static smartlist_t *nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 mock_nodelist_get_list(void)
 {
   n_nodelist_get_list++;
@@ -1614,7 +1957,172 @@ test_getinfo_md_all(void *arg)
   return;
 }
 
+static smartlist_t *reply_strs;
+
+static void
+mock_control_write_reply_list(control_connection_t *conn, int code, int c,
+                              const char *s)
+{
+  (void)conn;
+  /* To make matching easier, don't append "\r\n" */
+  smartlist_add_asprintf(reply_strs, "%03d%c%s", code, c, s);
+}
+
+static void
+test_control_reply(void *arg)
+{
+  (void)arg;
+  smartlist_t *lines = smartlist_new();
+
+  MOCK(control_write_reply, mock_control_write_reply);
+
+  tor_free(reply_str);
+  control_reply_clear(lines);
+  control_reply_add_str(lines, 250, "FOO");
+  control_write_reply_lines(NULL, lines);
+  tt_str_op(reply_str, OP_EQ, "FOO");
+
+  tor_free(reply_str);
+  control_reply_clear(lines);
+  control_reply_add_done(lines);
+  control_write_reply_lines(NULL, lines);
+  tt_str_op(reply_str, OP_EQ, "OK");
+
+  tor_free(reply_str);
+  control_reply_clear(lines);
+  UNMOCK(control_write_reply);
+  MOCK(control_write_reply, mock_control_write_reply_list);
+  reply_strs = smartlist_new();
+  control_reply_add_one_kv(lines, 250, 0, "A", "B");
+  control_reply_add_one_kv(lines, 250, 0, "C", "D");
+  control_write_reply_lines(NULL, lines);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 2);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250-A=B");
+  tt_str_op((char *)smartlist_get(reply_strs, 1), OP_EQ, "250 C=D");
+
+  control_reply_clear(lines);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+  control_reply_add_printf(lines, 250, "PROTOCOLINFO %d", 1);
+  control_reply_add_one_kv(lines, 250, KV_OMIT_VALS|KV_RAW, "AUTH", "");
+  control_reply_append_kv(lines, "METHODS", "COOKIE");
+  control_reply_append_kv(lines, "COOKIEFILE", escaped("/tmp/cookie"));
+  control_reply_add_done(lines);
+  control_write_reply_lines(NULL, lines);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 3);
+  tt_str_op((char *)smartlist_get(reply_strs, 0),
+            OP_EQ, "250-PROTOCOLINFO 1");
+  tt_str_op((char *)smartlist_get(reply_strs, 1),
+            OP_EQ, "250-AUTH METHODS=COOKIE COOKIEFILE=\"/tmp/cookie\"");
+  tt_str_op((char *)smartlist_get(reply_strs, 2),
+            OP_EQ, "250 OK");
+
+ done:
+  UNMOCK(control_write_reply);
+  tor_free(reply_str);
+  control_reply_free(lines);
+  if (reply_strs)
+    SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_free(reply_strs);
+  return;
+}
+
+static void
+test_control_getconf(void *arg)
+{
+  (void)arg;
+  control_connection_t conn;
+  char *args = NULL;
+  int r = -1;
+
+  memset(&conn, 0, sizeof(conn));
+  conn.current_cmd = tor_strdup("GETCONF");
+
+  MOCK(control_write_reply, mock_control_write_reply_list);
+  reply_strs = smartlist_new();
+
+  args = tor_strdup("");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250 OK");
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+  tor_free(args);
+
+  args = tor_strdup("NoSuch");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ,
+            "552 Unrecognized configuration key \"NoSuch\"");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("NoSuch1 NoSuch2");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 2);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ,
+            "552-Unrecognized configuration key \"NoSuch1\"");
+  tt_str_op((char *)smartlist_get(reply_strs, 1), OP_EQ,
+            "552 Unrecognized configuration key \"NoSuch2\"");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("ControlPort NoSuch");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  /* Valid keys ignored if there are any invalid ones */
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ,
+            "552 Unrecognized configuration key \"NoSuch\"");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("ClientOnly");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  /* According to config.c, this is an exception for the unit tests */
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250 ClientOnly=0");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("BridgeRelay ClientOnly");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 2);
+  /* Change if config.c changes BridgeRelay default (unlikely) */
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250-BridgeRelay=0");
+  tt_str_op((char *)smartlist_get(reply_strs, 1), OP_EQ, "250 ClientOnly=0");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+ done:
+  tor_free(conn.current_cmd);
+  tor_free(args);
+  UNMOCK(control_write_reply);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_free(reply_strs);
+}
+
+#ifndef COCCI
+#define PARSER_TEST(type)                                             \
+  { "parse/" #type, test_controller_parse_cmd, 0, &passthrough_setup, \
+      (void*)&parse_ ## type ## _params }
+#endif
+
 struct testcase_t controller_tests[] = {
+  PARSER_TEST(one_to_three),
+  PARSER_TEST(no_args_one_obj),
+  PARSER_TEST(no_args_kwargs),
+  PARSER_TEST(one_arg_kwargs),
   { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0,
     NULL, NULL },
   { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0,
@@ -1626,11 +2134,17 @@ struct testcase_t controller_tests[] = {
     NULL },
   { "download_status_consensus", test_download_status_consensus, 0, NULL,
     NULL },
+  {"getinfo_helper_current_consensus_from_cache",
+   test_getinfo_helper_current_consensus_from_cache, 0, NULL, NULL },
+  {"getinfo_helper_current_consensus_from_file",
+   test_getinfo_helper_current_consensus_from_file, 0, NULL, NULL },
   { "download_status_cert", test_download_status_cert, 0, NULL,
     NULL },
   { "download_status_desc", test_download_status_desc, 0, NULL, NULL },
   { "download_status_bridge", test_download_status_bridge, 0, NULL, NULL },
   { "current_time", test_current_time, 0, NULL, NULL },
   { "getinfo_md_all", test_getinfo_md_all, 0, NULL, NULL },
+  { "control_reply", test_control_reply, 0, NULL, NULL },
+  { "control_getconf", test_control_getconf, 0, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c
index 70d36e53d4..60dfbd630a 100644
--- a/src/test/test_controller_events.c
+++ b/src/test/test_controller_events.c
@@ -1,19 +1,30 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "app/main/subsysmgr.h"
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitlist.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
 
+#include "core/or/entry_connection_st.h"
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
 
 static void
 add_testing_cell_stats_entry(circuit_t *circ, uint8_t command,
@@ -351,10 +362,10 @@ test_cntev_dirboot_defer_desc(void *arg)
   /* This event should get deferred */
   control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
   assert_bootmsg("45 TAG=requesting_descriptors");
@@ -374,29 +385,343 @@ test_cntev_dirboot_defer_orconn(void *arg)
   control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
   assert_bootmsg("0 TAG=starting");
   /* This event should get deferred */
-  control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+  control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
-  assert_bootmsg("80 TAG=conn_or");
+  assert_bootmsg("75 TAG=enough_dirinfo");
  done:
   tor_free(saved_event_str);
   UNMOCK(queue_control_event_string);
 }
 
-#define TEST(name, flags)                                               \
+static void
+test_cntev_signal(void *arg)
+{
+  (void)arg;
+  int rv;
+
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+
+  /* Nothing is listening for signals, so no event should be queued. */
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+
+  /* Now try with signals included in the event mask. */
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_GOT_SIGNAL));
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL RELOAD\r\n");
+
+  rv = control_event_signal(SIGACTIVE);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL ACTIVE\r\n");
+
+  /* Try a signal that doesn't exist. */
+  setup_full_capture_of_logs(LOG_WARN);
+  tor_free(saved_event_str);
+  rv = control_event_signal(99999);
+  tt_int_op(-1, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+  expect_single_log_msg_containing("Unrecognized signal 99999");
+
+ done:
+  tor_free(saved_event_str);
+ teardown_capture_of_logs();
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+setup_orconn_state(orconn_state_msg_t *msg, uint64_t gid, uint64_t chan,
+                   int proxy_type)
+{
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->proxy_type = proxy_type;
+}
+
+static void
+send_orconn_state(const orconn_state_msg_t *msg_in, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  msg->state = state;
+  orconn_state_publish(msg);
+}
+
+static void
+send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->onehop = onehop;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_cntev_orconn_state(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_NONE);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  send_ocirc_chan(1, 1, true);
+  assert_bootmsg("5 TAG=conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  /* It doesn't know it's an origin circuit yet */
+  assert_bootmsg("15 TAG=handshake_done");
+  send_ocirc_chan(2, 2, false);
+  assert_bootmsg("80 TAG=ap_conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("85 TAG=ap_conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("89 TAG=ap_handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("90 TAG=ap_handshake_done");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_pt(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("1 TAG=conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("2 TAG=conn_done_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("76 TAG=ap_conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("77 TAG=ap_conn_done_pt");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_proxy(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_CONNECT);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("3 TAG=conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("4 TAG=conn_done_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("78 TAG=ap_conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("79 TAG=ap_conn_done_proxy");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_format_stream(void *arg)
+{
+  entry_connection_t *ec = NULL;
+  char *conndesc = NULL;
+  (void)arg;
+
+  ec = entry_connection_new(CONN_TYPE_AP, AF_INET);
+
+  char *username = tor_strdup("jeremy");
+  char *password = tor_strdup("letmein");
+  ec->socks_request->username = username; // steal reference
+  ec->socks_request->usernamelen = strlen(username);
+  ec->socks_request->password = password; // steal reference
+  ec->socks_request->passwordlen = strlen(password);
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "SOCKS_USERNAME=\"jeremy\""));
+  tt_assert(strstr(conndesc, "SOCKS_PASSWORD=\"letmein\""));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_LISTENER;
+  ec->socks_request->socks_version = 4;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=SOCKS4"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_LISTENER;
+  ec->socks_request->socks_version = 5;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=SOCKS5"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_LISTENER;
+  ec->socks_request->socks_version = 6;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=UNKNOWN"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_TRANS_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=TRANS"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_NATD_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=NATD"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_DNS_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=DNS"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=HTTPCONNECT"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_OR;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=UNKNOWN"));
+  tor_free(conndesc);
+
+  ec->nym_epoch = 1337;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "NYM_EPOCH=1337"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.session_group = 4321;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "SESSION_GROUP=4321"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_DESTPORT;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=DESTPORT"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=DESTPORT,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_DESTADDR;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=DESTADDR"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=DESTADDR,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_SOCKSAUTH;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=SOCKS_USERNAME,SOCKS_PASSWORD"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=SOCKS_USERNAME,SOCKS_PASSWORD,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_CLIENTPROTO;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=CLIENT_PROTOCOL"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=CLIENT_PROTOCOL,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_CLIENTADDR;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=CLIENTADDR"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=CLIENTADDR,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_SESSIONGRP;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=SESSION_GROUP"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=SESSION_GROUP,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_NYM_EPOCH;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=NYM_EPOCH"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=NYM_EPOCH,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_DESTPORT | ISO_SOCKSAUTH | ISO_NYM_EPOCH;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc,
+    "ISO_FIELDS=DESTPORT,SOCKS_USERNAME,SOCKS_PASSWORD,NYM_EPOCH"));
+  tt_assert(!strstr(conndesc,
+    "ISO_FIELDS=DESTPORT,SOCKS_USERNAME,SOCKS_PASSWORD,NYM_EPOCH,"));
+
+ done:
+  tor_free(conndesc);
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+}
+
+#define TEST(name, flags)                               \
   { #name, test_cntev_ ## name, flags, 0, NULL }
 
+#define T_PUBSUB(name, setup)                                           \
+  { #name, test_cntev_ ## name, TT_FORK, &helper_pubsub_setup, NULL }
+
 struct testcase_t controller_event_tests[] = {
   TEST(sum_up_cell_stats, TT_FORK),
   TEST(append_cell_stats, TT_FORK),
   TEST(format_cell_stats, TT_FORK),
   TEST(event_mask, TT_FORK),
-  TEST(dirboot_defer_desc, TT_FORK),
-  TEST(dirboot_defer_orconn, TT_FORK),
+  TEST(format_stream, TT_FORK),
+  TEST(signal, TT_FORK),
+  T_PUBSUB(dirboot_defer_desc, TT_FORK),
+  T_PUBSUB(dirboot_defer_orconn, TT_FORK),
+  T_PUBSUB(orconn_state, TT_FORK),
+  T_PUBSUB(orconn_state_pt, TT_FORK),
+  T_PUBSUB(orconn_state_proxy, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 5af0cce130..0d75a212e9 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -29,10 +29,10 @@
 
 #if defined(ENABLE_OPENSSL)
 #include "lib/crypt_ops/compat_openssl.h"
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 #include <openssl/dh.h>
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Run unit tests for Diffie-Hellman functionality. */
 static void
@@ -190,7 +190,7 @@ test_crypto_dh(void *arg)
     DH_get0_key(dh4, &pk, &sk);
 #else
     pk = dh4->pub_key;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
     tt_assert(pk);
     tt_int_op(BN_num_bytes(pk), OP_LE, DH1024_KEY_LEN);
     tt_int_op(BN_num_bytes(pk), OP_GT, 0);
@@ -207,7 +207,7 @@ test_crypto_dh(void *arg)
     tt_int_op(s1len, OP_GT, 0);
     tt_mem_op(s1, OP_EQ, s2, s1len);
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
  done:
   crypto_dh_free(dh1);
@@ -219,7 +219,7 @@ test_crypto_dh(void *arg)
     DH_free(dh4);
   if (pubkey_tmp)
     BN_free(pubkey_tmp);
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 }
 
 static void
@@ -248,174 +248,12 @@ test_crypto_openssl_version(void *arg)
   tt_int_op(a, OP_GE, 0);
   tt_int_op(b, OP_GE, 0);
   tt_int_op(c, OP_GE, 0);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
  done:
   ;
 }
 
-/** Run unit tests for our random number generation function and its wrappers.
- */
-static void
-test_crypto_rng(void *arg)
-{
-  int i, j, allok;
-  char data1[100], data2[100];
-  double d;
-  char *h=NULL;
-
-  /* Try out RNG. */
-  (void)arg;
-  tt_assert(! crypto_seed_rng());
-  crypto_rand(data1, 100);
-  crypto_rand(data2, 100);
-  tt_mem_op(data1,OP_NE, data2,100);
-  allok = 1;
-  for (i = 0; i < 100; ++i) {
-    uint64_t big;
-    char *host;
-    j = crypto_rand_int(100);
-    if (j < 0 || j >= 100)
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(1)<<40);
-    if (big >= (UINT64_C(1)<<40))
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(5));
-    if (big >= 5)
-      allok = 0;
-    d = crypto_rand_double();
-    tt_assert(d >= 0);
-    tt_assert(d < 1.0);
-    host = crypto_random_hostname(3,8,"www.",".onion");
-    if (strcmpstart(host,"www.") ||
-        strcmpend(host,".onion") ||
-        strlen(host) < 13 ||
-        strlen(host) > 18)
-      allok = 0;
-    tor_free(host);
-  }
-
-  /* Make sure crypto_random_hostname clips its inputs properly. */
-  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
-  tt_assert(! strcmpstart(h,"www."));
-  tt_assert(! strcmpend(h,".onion"));
-  tt_int_op(63+4+6, OP_EQ, strlen(h));
-
-  tt_assert(allok);
- done:
-  tor_free(h);
-}
-
-static void
-test_crypto_rng_range(void *arg)
-{
-  int got_smallest = 0, got_largest = 0;
-  int i;
-
-  (void)arg;
-  for (i = 0; i < 1000; ++i) {
-    int x = crypto_rand_int_range(5,9);
-    tt_int_op(x, OP_GE, 5);
-    tt_int_op(x, OP_LT, 9);
-    if (x == 5)
-      got_smallest = 1;
-    if (x == 8)
-      got_largest = 1;
-  }
-  /* These fail with probability 1/10^603. */
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  got_smallest = got_largest = 0;
-  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
-  for (i = 0; i < 1000; ++i) {
-    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
-    tt_u64_op(x, OP_GE, ten_billion);
-    tt_u64_op(x, OP_LT, ten_billion+10);
-    if (x == ten_billion)
-      got_smallest = 1;
-    if (x == ten_billion+9)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  const time_t now = time(NULL);
-  for (i = 0; i < 2000; ++i) {
-    time_t x = crypto_rand_time_range(now, now+60);
-    tt_i64_op(x, OP_GE, now);
-    tt_i64_op(x, OP_LT, now+60);
-    if (x == now)
-      got_smallest = 1;
-    if (x == now+59)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
- done:
-  ;
-}
-
-static void
-test_crypto_rng_strongest(void *arg)
-{
-  const char *how = arg;
-  int broken = 0;
-
-  if (how == NULL) {
-    ;
-  } else if (!strcmp(how, "nosyscall")) {
-    break_strongest_rng_syscall = 1;
-  } else if (!strcmp(how, "nofallback")) {
-    break_strongest_rng_fallback = 1;
-  } else if (!strcmp(how, "broken")) {
-    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
-  }
-
-#define N 128
-  uint8_t combine_and[N];
-  uint8_t combine_or[N];
-  int i, j;
-
-  memset(combine_and, 0xff, N);
-  memset(combine_or, 0, N);
-
-  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
-    uint8_t output[N];
-    memset(output, 0, N);
-    if (how == NULL) {
-      /* this one can't fail. */
-      crypto_strongest_rand(output, sizeof(output));
-    } else {
-      int r = crypto_strongest_rand_raw(output, sizeof(output));
-      if (r == -1) {
-        if (broken) {
-          goto done; /* we're fine. */
-        }
-        /* This function is allowed to break, but only if it always breaks. */
-        tt_int_op(i, OP_EQ, 0);
-        tt_skip();
-      } else {
-        tt_assert(! broken);
-      }
-    }
-    for (j = 0; j < N; ++j) {
-      combine_and[j] &= output[j];
-      combine_or[j] |= output[j];
-    }
-  }
-
-  for (j = 0; j < N; ++j) {
-    tt_int_op(combine_and[j], OP_EQ, 0);
-    tt_int_op(combine_or[j], OP_EQ, 0xff);
-  }
- done:
-  ;
-#undef N
-}
-
 /** Run unit tests for our AES128 functionality */
 static void
 test_crypto_aes128(void *arg)
@@ -551,7 +389,7 @@ test_crypto_aes128(void *arg)
                                    "\xff\xff\xff\xff\xff\xff\xff\xff"
                                    "\xff\xff\xff\xff\xff\xff\xff\xff");
   crypto_cipher_crypt_inplace(env1, data2, 64);
-  tt_assert(tor_mem_is_zero(data2, 64));
+  tt_assert(fast_mem_is_zero(data2, 64));
 
  done:
   tor_free(mem_op_hex_tmp);
@@ -1173,13 +1011,19 @@ test_crypto_sha3_xof(void *arg)
   crypto_xof_free(xof);
   memset(out, 0, sizeof(out));
 
+  /* Test one-function absorb/squeeze. */
+  crypto_xof(out, sizeof(out), msg, sizeof(msg));
+  test_memeq_hex(out, squeezed_hex);
+  memset(out, 0, sizeof(out));
+
   /* Test incremental absorb/squeeze. */
   xof = crypto_xof_new();
   tt_assert(xof);
   for (size_t i = 0; i < sizeof(msg); i++)
     crypto_xof_add_bytes(xof, msg + i, 1);
-  for (size_t i = 0; i < sizeof(out); i++)
+  for (size_t i = 0; i < sizeof(out); i++) {
     crypto_xof_squeeze_bytes(xof, out + i, 1);
+  }
   test_memeq_hex(out, squeezed_hex);
 
  done:
@@ -1903,13 +1747,13 @@ test_crypto_base32_decode(void *arg)
   /* Encode and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Encode, uppercase, and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   tor_strupper(encoded);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Change encoded string and decode. */
   if (encoded[0] == 'A' || encoded[0] == 'a')
@@ -1917,12 +1761,12 @@ test_crypto_base32_decode(void *arg)
   else
     encoded[0] = 'A';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_NE, decoded, 60);
   /* Bad encodings. */
   encoded[0] = '!';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(0, OP_GT, res);
+  tt_int_op(res, OP_LT, 0);
 
  done:
   ;
@@ -2117,7 +1961,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "e0544770bc7de853b38f9100489e3e79";
   const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88"
                                 "bd13861475516bc2cd2b6e005e805064";
-#else /* !(defined(SLOW_CURVE25519_TEST)) */
+#else /* !defined(SLOW_CURVE25519_TEST) */
   const int loop_max=200;
   const char e1_expected[]    = "bc7112cde03f97ef7008cad1bdc56be3"
                                 "c6a1037d74cceb3712e9206871dcf654";
@@ -2269,7 +2113,7 @@ test_crypto_curve25519_encode(void *arg)
 
   curve25519_secret_key_generate(&seckey, 0);
   curve25519_public_key_generate(&key1, &seckey);
-  tt_int_op(0, OP_EQ, curve25519_public_to_base64(buf, &key1));
+  curve25519_public_to_base64(buf, &key1);
   tt_int_op(CURVE25519_BASE64_PADDED_LEN, OP_EQ, strlen(buf));
 
   tt_int_op(0, OP_EQ, curve25519_public_from_base64(&key2, buf));
@@ -2328,7 +2172,7 @@ test_crypto_curve25519_persist(void *arg)
   tt_u64_op((uint64_t)st.st_size, OP_EQ,
             32+CURVE25519_PUBKEY_LEN+CURVE25519_SECKEY_LEN);
   tt_assert(fast_memeq(content, "== c25519v1: testing ==", taglen));
-  tt_assert(tor_mem_is_zero(content+taglen, 32-taglen));
+  tt_assert(fast_mem_is_zero(content+taglen, 32-taglen));
   cp = content + 32;
   tt_mem_op(keypair.seckey.secret_key,OP_EQ,
              cp,
@@ -2649,13 +2493,13 @@ test_crypto_ed25519_encode(void *arg)
 
   /* Test roundtrip. */
   tt_int_op(0, OP_EQ, ed25519_keypair_generate(&kp, 0));
-  tt_int_op(0, OP_EQ, ed25519_public_to_base64(buf, &kp.pubkey));
+  ed25519_public_to_base64(buf, &kp.pubkey);
   tt_int_op(ED25519_BASE64_LEN, OP_EQ, strlen(buf));
   tt_int_op(0, OP_EQ, ed25519_public_from_base64(&pk, buf));
   tt_mem_op(kp.pubkey.pubkey, OP_EQ, pk.pubkey, ED25519_PUBKEY_LEN);
 
   tt_int_op(0, OP_EQ, ed25519_sign(&sig1, (const uint8_t*)"ABC", 3, &kp));
-  tt_int_op(0, OP_EQ, ed25519_signature_to_base64(buf, &sig1));
+  ed25519_signature_to_base64(buf, &sig1);
   tt_int_op(0, OP_EQ, ed25519_signature_from_base64(&sig2, buf));
   tt_mem_op(sig1.sig, OP_EQ, sig2.sig, ED25519_SIG_LEN);
 
@@ -3165,6 +3009,7 @@ test_crypto_failure_modes(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define CRYPTO_LEGACY(name)                                            \
   { #name, test_crypto_ ## name , 0, NULL, NULL }
 
@@ -3175,18 +3020,10 @@ test_crypto_failure_modes(void *arg)
 #define ED25519_TEST(name, fl)                  \
   ED25519_TEST_ONE(name, (fl), "donna"),        \
   ED25519_TEST_ONE(name, (fl), "ref10")
+#endif /* !defined(COCCI) */
 
 struct testcase_t crypto_tests[] = {
   CRYPTO_LEGACY(formats),
-  CRYPTO_LEGACY(rng),
-  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
-  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
-  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nosyscall" },
-  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nofallback" },
-  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"broken" },
   { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL },
   { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" },
   { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" },
diff --git a/src/test/test_crypto_ope.c b/src/test/test_crypto_ope.c
index dc67c02676..119ebc114a 100644
--- a/src/test/test_crypto_ope.c
+++ b/src/test/test_crypto_ope.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c
index 42dc3f6be2..989f4a56ca 100644
--- a/src/test/test_crypto_openssl.c
+++ b/src/test/test_crypto_openssl.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c
new file mode 100644
index 0000000000..b0dc4c117c
--- /dev/null
+++ b/src/test/test_crypto_rng.c
@@ -0,0 +1,332 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#define CRYPTO_RAND_PRIVATE
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+/** Run unit tests for our random number generation function and its wrappers.
+ */
+static void
+test_crypto_rng(void *arg)
+{
+  int i, j, allok;
+  char data1[100], data2[100];
+  double d;
+  char *h=NULL;
+
+  /* Try out RNG. */
+  (void)arg;
+  tt_assert(! crypto_seed_rng());
+  crypto_rand(data1, 100);
+  crypto_rand(data2, 100);
+  tt_mem_op(data1,OP_NE, data2,100);
+  allok = 1;
+  for (i = 0; i < 100; ++i) {
+    uint64_t big;
+    char *host;
+    j = crypto_rand_int(100);
+    if (j < 0 || j >= 100)
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(1)<<40);
+    if (big >= (UINT64_C(1)<<40))
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(5));
+    if (big >= 5)
+      allok = 0;
+    d = crypto_rand_double();
+    tt_assert(d >= 0);
+    tt_assert(d < 1.0);
+    host = crypto_random_hostname(3,8,"www.",".onion");
+    if (strcmpstart(host,"www.") ||
+        strcmpend(host,".onion") ||
+        strlen(host) < 13 ||
+        strlen(host) > 18)
+      allok = 0;
+    tor_free(host);
+  }
+
+  /* Make sure crypto_random_hostname clips its inputs properly. */
+  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
+  tt_assert(! strcmpstart(h,"www."));
+  tt_assert(! strcmpend(h,".onion"));
+  tt_int_op(63+4+6, OP_EQ, strlen(h));
+
+  tt_assert(allok);
+ done:
+  tor_free(h);
+}
+
+static void
+test_crypto_rng_range(void *arg)
+{
+  int got_smallest = 0, got_largest = 0;
+  int i;
+
+  (void)arg;
+  for (i = 0; i < 1000; ++i) {
+    int x = crypto_rand_int_range(5,9);
+    tt_int_op(x, OP_GE, 5);
+    tt_int_op(x, OP_LT, 9);
+    if (x == 5)
+      got_smallest = 1;
+    if (x == 8)
+      got_largest = 1;
+  }
+  /* These fail with probability 1/10^603. */
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  got_smallest = got_largest = 0;
+  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
+  for (i = 0; i < 1000; ++i) {
+    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
+    tt_u64_op(x, OP_GE, ten_billion);
+    tt_u64_op(x, OP_LT, ten_billion+10);
+    if (x == ten_billion)
+      got_smallest = 1;
+    if (x == ten_billion+9)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  const time_t now = time(NULL);
+  for (i = 0; i < 2000; ++i) {
+    time_t x = crypto_rand_time_range(now, now+60);
+    tt_i64_op(x, OP_GE, now);
+    tt_i64_op(x, OP_LT, now+60);
+    if (x == now)
+      got_smallest = 1;
+    if (x == now+59)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+ done:
+  ;
+}
+
+static void
+test_crypto_rng_strongest(void *arg)
+{
+  const char *how = arg;
+  int broken = 0;
+
+  if (how == NULL) {
+    ;
+  } else if (!strcmp(how, "nosyscall")) {
+    break_strongest_rng_syscall = 1;
+  } else if (!strcmp(how, "nofallback")) {
+    break_strongest_rng_fallback = 1;
+  } else if (!strcmp(how, "broken")) {
+    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
+  }
+
+#define N 128
+  uint8_t combine_and[N];
+  uint8_t combine_or[N];
+  int i, j;
+
+  memset(combine_and, 0xff, N);
+  memset(combine_or, 0, N);
+
+  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
+    uint8_t output[N];
+    memset(output, 0, N);
+    if (how == NULL) {
+      /* this one can't fail. */
+      crypto_strongest_rand(output, sizeof(output));
+    } else {
+      int r = crypto_strongest_rand_raw(output, sizeof(output));
+      if (r == -1) {
+        if (broken) {
+          goto done; /* we're fine. */
+        }
+        /* This function is allowed to break, but only if it always breaks. */
+        tt_int_op(i, OP_EQ, 0);
+        tt_skip();
+      } else {
+        tt_assert(! broken);
+      }
+    }
+    for (j = 0; j < N; ++j) {
+      combine_and[j] &= output[j];
+      combine_or[j] |= output[j];
+    }
+  }
+
+  for (j = 0; j < N; ++j) {
+    tt_int_op(combine_and[j], OP_EQ, 0);
+    tt_int_op(combine_or[j], OP_EQ, 0xff);
+  }
+ done:
+  ;
+#undef N
+}
+
+static void
+test_crypto_rng_fast(void *arg)
+{
+  (void)arg;
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  tt_assert(rng);
+
+  /* Rudimentary black-block test to make sure that our prng outputs
+   * have all bits sometimes on and all bits sometimes off. */
+  uint64_t m1 = 0, m2 = ~(uint64_t)0;
+  const int N = 128;
+
+  for (int i=0; i < N; ++i) {
+    uint64_t v;
+    crypto_fast_rng_getbytes(rng, (void*)&v, sizeof(v));
+    m1 |= v;
+    m2 &= v;
+  }
+
+  tt_u64_op(m1, OP_EQ, ~(uint64_t)0);
+  tt_u64_op(m2, OP_EQ, 0);
+
+  /* Check range functions. */
+  int counts[5];
+  memset(counts, 0, sizeof(counts));
+  for (int i=0; i < N; ++i) {
+    unsigned u = crypto_fast_rng_get_uint(rng, 5);
+    tt_int_op(u, OP_GE, 0);
+    tt_int_op(u, OP_LT, 5);
+    counts[u]++;
+
+    uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+
+    double d = crypto_fast_rng_get_double(rng);
+    tt_assert(d >= 0.0);
+    tt_assert(d < 1.0);
+  }
+
+  /* All values should have come up once. */
+  for (int i=0; i<5; ++i) {
+    tt_int_op(counts[i], OP_GT, 0);
+  }
+
+  /* per-thread rand_fast shouldn't crash or leak. */
+  crypto_fast_rng_t *t_rng = get_thread_fast_rng();
+  for (int i = 0; i < N; ++i) {
+    uint64_t u64 = crypto_fast_rng_get_uint64(t_rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+  }
+
+ done:
+  crypto_fast_rng_free(rng);
+}
+
+static void
+test_crypto_rng_fast_whitebox(void *arg)
+{
+  (void)arg;
+  const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream();
+  char *buf = tor_malloc_zero(buflen);
+  char *buf2 = tor_malloc_zero(buflen);
+  char *buf3 = NULL, *buf4 = NULL;
+
+  crypto_cipher_t *cipher = NULL, *cipher2 = NULL;
+  uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed, 0, sizeof(seed));
+
+  /* Start with a prng with zero key and zero IV. */
+  crypto_fast_rng_t *rng = crypto_fast_rng_new_from_seed(seed);
+  tt_assert(rng);
+
+  /* We'll use a stream cipher to keep in sync */
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+
+  /* The first 48 bytes are used for the next seed -- let's make sure we have
+   * them.
+   */
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+
+  /* if we get 128 bytes, they should match the bytes from the aes256-counter
+   * stream, starting at position 48.
+   */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 128);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* Try that again, with an odd number of bytes. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 199);
+  memset(buf2, 0, 199);
+  crypto_cipher_crypt_inplace(cipher, buf2, 199);
+  tt_mem_op(buf, OP_EQ, buf2, 199);
+
+  /* Make sure that refilling works as expected: skip all but the last 5 bytes
+   * of this steam. */
+  size_t skip = buflen - (199+128) - 5;
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, skip);
+  crypto_cipher_crypt_inplace(cipher, buf2, skip);
+
+  /* Now get the next 128 bytes. The first 5 will come from this stream, and
+   * the next 5 will come from the stream keyed by the new value of 'seed'. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 5);
+  crypto_cipher_free(cipher);
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, buf2+5, 128-5);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* And check the next 7 bytes to make sure we didn't discard anything. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 7);
+  memset(buf2, 0, 7);
+  crypto_cipher_crypt_inplace(cipher, buf2, 7);
+  tt_mem_op(buf, OP_EQ, buf2, 7);
+
+  /* Now try the optimization for long outputs. */
+  buf3 = tor_malloc(65536);
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf3, 65536);
+
+  buf4 = tor_malloc_zero(65536);
+  uint8_t seed2[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed2, 0, sizeof(seed2));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed2, sizeof(seed2));
+  cipher2 = crypto_cipher_new_with_iv_and_bits(seed2, seed2+32, 256);
+  crypto_cipher_crypt_inplace(cipher2, buf4, 65536);
+  tt_mem_op(buf3, OP_EQ, buf4, 65536);
+
+ done:
+  crypto_fast_rng_free(rng);
+  crypto_cipher_free(cipher);
+  crypto_cipher_free(cipher2);
+  tor_free(buf);
+  tor_free(buf2);
+  tor_free(buf3);
+  tor_free(buf4);
+}
+
+struct testcase_t crypto_rng_tests[] = {
+  { "rng", test_crypto_rng, 0, NULL, NULL },
+  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
+  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
+  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nosyscall" },
+  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nofallback" },
+  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"broken" },
+  { "fast", test_crypto_rng_fast, 0, NULL, NULL },
+  { "fast_whitebox", test_crypto_rng_fast_whitebox, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c
index e24aee8930..1702427b08 100644
--- a/src/test/test_crypto_slow.c
+++ b/src/test/test_crypto_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -109,7 +109,7 @@ run_s2k_tests(const unsigned flags, const unsigned type,
             secret_to_key_derivekey(buf3, sizeof(buf3), buf, speclen,
                                     pw1, strlen(pw1)));
   tt_mem_op(buf2, OP_EQ, buf3, sizeof(buf3));
-  tt_assert(!tor_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
+  tt_assert(!fast_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
 
  done:
   ;
@@ -342,7 +342,7 @@ test_crypto_scrypt_vectors(void *arg)
 #endif
 
   /* Test vectors from
-     http://tools.ietf.org/html/draft-josefsson-scrypt-kdf-00 section 11.
+     https://tools.ietf.org/html/draft-josefsson-scrypt-kdf-00 section 11.
 
      Note that the names of 'r' and 'N' are switched in that section. Or
      possibly in libscrypt.
@@ -584,6 +584,7 @@ test_crypto_ed25519_fuzz_donna(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define CRYPTO_LEGACY(name)                                            \
   { #name, test_crypto_ ## name , 0, NULL, NULL }
 
@@ -594,6 +595,7 @@ test_crypto_ed25519_fuzz_donna(void *arg)
 #define ED25519_TEST(name, fl)                  \
   ED25519_TEST_ONE(name, (fl), "donna"),        \
   ED25519_TEST_ONE(name, (fl), "ref10")
+#endif /* !defined(COCCI) */
 
 struct testcase_t slow_crypto_tests[] = {
   CRYPTO_LEGACY(s2k_rfc2440),
diff --git a/src/test/test_data.c b/src/test/test_data.c
index fe1190ea77..30c14fcfff 100644
--- a/src/test/test_data.c
+++ b/src/test/test_data.c
@@ -1,6 +1,6 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "test/test.h"
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 0e44c47f3f..3a0b8237cb 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,10 +8,10 @@
 
 #define BWAUTH_PRIVATE
 #define CONFIG_PRIVATE
-#define CONTROL_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
+#define DIRAUTH_SYS_PRIVATE
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
-#define DIRSERV_PRIVATE
 #define DIRVOTE_PRIVATE
 #define DLSTATUS_PRIVATE
 #define HIBERNATE_PRIVATE
@@ -26,14 +26,15 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/connection.h"
 #include "core/or/relay.h"
 #include "core/or/versions.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_getinfo.h"
 #include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/dsigs_parse.h"
 #include "feature/dirauth/process_descs.h"
@@ -46,10 +47,11 @@
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dircommon/fp_pair.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nickname.h"
 #include "feature/nodelist/node_select.h"
@@ -71,13 +73,16 @@
 #include "lib/memarea/memarea.h"
 #include "lib/osinfo/uname.h"
 #include "test/log_test_helpers.h"
+#include "test/opts_test_helpers.h"
 #include "test/test.h"
 #include "test/test_dir_common.h"
 
 #include "core/or/addr_policy_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/document_signature_st.h"
 #include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/networkstatus_voter_info_st.h"
 #include "feature/dirauth/ns_detached_signatures_st.h"
@@ -91,8 +96,26 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
-#define NS_MODULE dir
+static networkstatus_t *
+networkstatus_parse_vote_from_string_(const char *s,
+                                      const char **eos_out,
+                                      enum networkstatus_type_t ns_type)
+{
+  size_t len = strlen(s);
+  // memdup so that it won't be nul-terminated.
+  char *tmp = tor_memdup(s, len);
+  networkstatus_t *result =
+    networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type);
+  if (eos_out && *eos_out) {
+    *eos_out = s + (*eos_out - tmp);
+  }
+  tor_free(tmp);
+  return result;
+}
 
 static void
 test_dir_nicknames(void *arg)
@@ -142,6 +165,269 @@ test_dir_nicknames(void *arg)
   ;
 }
 
+/* Allocate and return a new routerinfo, with the fields set from the
+ * arguments to this function.
+ *
+ * Also sets:
+ *  - random RSA identity and onion keys,
+ *  - the platform field using get_platform_str(), and
+ *  - supports_tunnelled_dir_requests to 1.
+ *
+ * If rsa_onion_keypair_out is not NULL, it is set to the onion keypair.
+ * The caller must free this keypair.
+ */
+static routerinfo_t *
+basic_routerinfo_new(const char *nickname, uint32_t ipv4_addr,
+                     uint16_t or_port, uint16_t dir_port,
+                     uint32_t bandwidthrate, uint32_t bandwidthburst,
+                     uint32_t bandwidthcapacity,
+                     time_t published_on,
+                     crypto_pk_t **rsa_onion_keypair_out)
+{
+  char platform[256];
+
+  tor_assert(nickname);
+
+  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
+  /* These keys are random: idx is ignored. */
+  pk1 = pk_generate(0);
+  pk2 = pk_generate(1);
+
+  tor_assert(pk1);
+  tor_assert(pk2);
+
+  get_platform_str(platform, sizeof(platform));
+
+  routerinfo_t *r1 = tor_malloc_zero(sizeof(routerinfo_t));
+
+  r1->nickname = tor_strdup(nickname);
+  r1->platform = tor_strdup(platform);
+
+  r1->addr = ipv4_addr;
+  r1->or_port = or_port;
+  r1->dir_port = dir_port;
+  r1->supports_tunnelled_dir_requests = 1;
+
+  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
+  r1->identity_pkey = pk2;
+
+  r1->bandwidthrate = bandwidthrate;
+  r1->bandwidthburst = bandwidthburst;
+  r1->bandwidthcapacity = bandwidthcapacity;
+
+  r1->cache_info.published_on = published_on;
+
+  if (rsa_onion_keypair_out) {
+    *rsa_onion_keypair_out = pk1;
+  } else {
+    crypto_pk_free(pk1);
+  }
+
+  return r1;
+}
+
+/* Allocate and return a new string containing a "router" line for r1. */
+static char *
+get_new_router_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "router %s %s %d 0 %d\n",
+               r1->nickname, fmt_addr32(r1->addr),
+               r1->or_port, r1->dir_port);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "platform" line for the
+ * current Tor version and OS. */
+static char *
+get_new_platform_line(void)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "platform Tor %s on %s\n",
+               VERSION, get_uname());
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "published" line for r1.
+ * r1->cache_info.published_on must be between 0 and 59 seconds. */
+static char *
+get_new_published_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_assert(r1->cache_info.published_on >= 0);
+  tor_assert(r1->cache_info.published_on <= 59);
+
+  tor_asprintf(&line,
+               "published 1970-01-01 00:00:%02u\n",
+               (unsigned)r1->cache_info.published_on);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "fingerprint" line for r1. */
+static char *
+get_new_fingerprint_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+  char fingerprint[FINGERPRINT_LEN+1];
+
+  tor_assert(r1);
+
+  tor_assert(!crypto_pk_get_fingerprint(r1->identity_pkey, fingerprint, 1));
+  tor_assert(strlen(fingerprint) > 0);
+
+  tor_asprintf(&line,
+               "fingerprint %s\n",
+               fingerprint);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "uptime" line with uptime t.
+ *
+ * You should pass a hard-coded value to this function, because even if we made
+ * it reflect uptime, that still wouldn't make it right, because the two
+ * descriptors might be made on different seconds.
+ */
+static char *
+get_new_uptime_line(time_t t)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "uptime %u\n",
+               (unsigned)t);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "bandwidth" line for r1.
+ */
+static char *
+get_new_bandwidth_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "bandwidth %u %u %u\n",
+               r1->bandwidthrate,
+               r1->bandwidthburst,
+               r1->bandwidthcapacity);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a key_name block for the
+ * RSA key pk1.
+ */
+static char *
+get_new_rsa_key_block(const char *key_name, crypto_pk_t *pk1)
+{
+  char *block = NULL;
+  char *pk1_str = NULL;
+  size_t pk1_str_len = 0;
+
+  tor_assert(key_name);
+  tor_assert(pk1);
+
+  tor_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
+                                                   &pk1_str_len));
+  tor_assert(pk1_str);
+  tor_assert(pk1_str_len);
+
+  tor_asprintf(&block,
+               "%s\n%s",
+               key_name,
+               pk1_str);
+  tor_free(pk1_str);
+
+  tor_assert(block);
+  return block;
+}
+
+/* Allocate and return a new string containing an "onion-key" block for the
+ * router r1.
+ */
+static char *
+get_new_onion_key_block(const routerinfo_t *r1)
+{
+  char *block = NULL;
+  tor_assert(r1);
+  crypto_pk_t *pk_tmp = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                  r1->onion_pkey_len);
+  block = get_new_rsa_key_block("onion-key", pk_tmp);
+  crypto_pk_free(pk_tmp);
+  return block;
+}
+
+/* Allocate and return a new string containing an "signing-key" block for the
+ * router r1.
+ */
+static char *
+get_new_signing_key_block(const routerinfo_t *r1)
+{
+  tor_assert(r1);
+  return get_new_rsa_key_block("signing-key", r1->identity_pkey);
+}
+
+/* Allocate and return a new string containing an "ntor-onion-key" line for
+ * the curve25519 public key ntor_onion_pubkey.
+ */
+static char *
+get_new_ntor_onion_key_line(const curve25519_public_key_t *ntor_onion_pubkey)
+{
+  char *line = NULL;
+  char cert_buf[256];
+  int rv = 0;
+
+  tor_assert(ntor_onion_pubkey);
+
+  rv = base64_encode(cert_buf, sizeof(cert_buf),
+                     (const char*)ntor_onion_pubkey->public_key, 32,
+                     BASE64_ENCODE_MULTILINE);
+  tor_assert(rv > 0);
+  tor_assert(strlen(cert_buf) > 0);
+
+  tor_asprintf(&line,
+               "ntor-onion-key %s",
+               cert_buf);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "bridge-distribution-request"
+ * line for options.
+ */
+static char *
+get_new_bridge_distribution_request_line(const or_options_t *options)
+{
+  if (options->BridgeRelay) {
+    return tor_strdup("bridge-distribution-request any\n");
+  } else {
+    return tor_strdup("");
+  }
+}
+
 static smartlist_t *mocked_configured_ports = NULL;
 
 /** Returns mocked_configured_ports */
@@ -151,71 +437,509 @@ mock_get_configured_ports(void)
   return mocked_configured_ports;
 }
 
-/** Run unit tests for router descriptor generation logic. */
+static tor_cert_t *
+mock_tor_cert_dup_null(const tor_cert_t *cert)
+{
+  (void)cert;
+  return NULL;
+}
+
+static crypto_pk_t *mocked_server_identitykey = NULL;
+
+/* Returns mocked_server_identitykey with no checks. */
+static crypto_pk_t *
+mock_get_server_identity_key(void)
+{
+  return mocked_server_identitykey;
+}
+
+static crypto_pk_t *mocked_onionkey = NULL;
+
+/* Returns mocked_onionkey with no checks. */
+static crypto_pk_t *
+mock_get_onion_key(void)
+{
+  return mocked_onionkey;
+}
+
+static routerinfo_t *mocked_routerinfo = NULL;
+
+/* Returns 0 and sets ri_out to mocked_routerinfo.
+ * ri_out must not be NULL. There are no other checks. */
+static int
+mock_router_build_fresh_unsigned_routerinfo(routerinfo_t **ri_out)
+{
+  tor_assert(ri_out);
+  *ri_out = mocked_routerinfo;
+  return 0;
+}
+
+static ed25519_keypair_t *mocked_master_signing_key = NULL;
+
+/* Returns mocked_master_signing_key with no checks. */
+static const ed25519_keypair_t *
+mock_get_master_signing_keypair(void)
+{
+  return mocked_master_signing_key;
+}
+
+static struct tor_cert_st *mocked_signing_key_cert = NULL;
+
+/* Returns mocked_signing_key_cert with no checks. */
+static const struct tor_cert_st *
+mock_get_master_signing_key_cert(void)
+{
+  return mocked_signing_key_cert;
+}
+
+static curve25519_keypair_t *mocked_curve25519_onion_key = NULL;
+
+/* Returns mocked_curve25519_onion_key with no checks. */
+static const curve25519_keypair_t *
+mock_get_current_curve25519_keypair(void)
+{
+  return mocked_curve25519_onion_key;
+}
+
+/* Unmock get_configured_ports() and free mocked_configured_ports. */
 static void
-test_dir_formats(void *arg)
+cleanup_mock_configured_ports(void)
+{
+  UNMOCK(get_configured_ports);
+
+  if (mocked_configured_ports) {
+    SMARTLIST_FOREACH(mocked_configured_ports, port_cfg_t *, p, tor_free(p));
+    smartlist_free(mocked_configured_ports);
+  }
+}
+
+/* Mock get_configured_ports() with a list containing or_port and dir_port.
+ * If a port is 0, don't set it.
+ * Only sets the minimal data required for the tests to pass. */
+static void
+setup_mock_configured_ports(uint16_t or_port, uint16_t dir_port)
+{
+  cleanup_mock_configured_ports();
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  MOCK(get_configured_ports, mock_get_configured_ports);
+  mocked_configured_ports = smartlist_new();
+
+  if (or_port) {
+    port_cfg_t *or_port_cfg = tor_malloc_zero(sizeof(*or_port_cfg));
+    or_port_cfg->type = CONN_TYPE_OR_LISTENER;
+    or_port_cfg->addr.family = AF_INET;
+    or_port_cfg->port = or_port;
+    smartlist_add(mocked_configured_ports, or_port_cfg);
+  }
+
+  if (dir_port) {
+    port_cfg_t *dir_port_cfg = tor_malloc_zero(sizeof(*dir_port_cfg));
+    dir_port_cfg->type = CONN_TYPE_DIR_LISTENER;
+    dir_port_cfg->addr.family = AF_INET;
+    dir_port_cfg->port = dir_port;
+    smartlist_add(mocked_configured_ports, dir_port_cfg);
+  }
+}
+
+/* Clean up the data structures and unmock the functions needed for generating
+ * a fresh descriptor. */
+static void
+cleanup_mocks_for_fresh_descriptor(void)
+{
+  tor_free(get_options_mutable()->Nickname);
+
+  mocked_server_identitykey = NULL;
+  UNMOCK(get_server_identity_key);
+
+  crypto_pk_free(mocked_onionkey);
+  UNMOCK(get_onion_key);
+}
+
+/* Mock the data structures and functions needed for generating a fresh
+ * descriptor.
+ *
+ * Sets options->Nickname from r1->nickname.
+ * Mocks get_server_identity_key() with r1->identity_pkey.
+ *
+ * If rsa_onion_keypair is not NULL, it is used to mock get_onion_key().
+ * Otherwise, the public key in r1->onion_pkey is used to mock get_onion_key().
+ */
+static void
+setup_mocks_for_fresh_descriptor(const routerinfo_t *r1,
+                                 crypto_pk_t *rsa_onion_keypair)
+{
+  cleanup_mocks_for_fresh_descriptor();
+
+  tor_assert(r1);
+
+  /* router_build_fresh_signed_extrainfo() requires options->Nickname */
+  get_options_mutable()->Nickname = tor_strdup(r1->nickname);
+
+  /* router_build_fresh_signed_extrainfo() requires get_server_identity_key().
+   * Use the same one as the call to router_dump_router_to_string() above.
+   */
+  mocked_server_identitykey = r1->identity_pkey;
+  MOCK(get_server_identity_key, mock_get_server_identity_key);
+
+  /* router_dump_and_sign_routerinfo_descriptor_body() requires
+   * get_onion_key(). Use the same one as r1.
+   */
+  if (rsa_onion_keypair) {
+    mocked_onionkey = crypto_pk_dup_key(rsa_onion_keypair);
+  } else {
+    mocked_onionkey = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                r1->onion_pkey_len);
+  }
+  MOCK(get_onion_key, mock_get_onion_key);
+}
+
+/* Set options based on arg.
+ *
+ * b: BridgeRelay 1
+ * e: ExtraInfoStatistics 1
+ * s: sets all the individual statistics options to 1
+ *
+ * Always sets AssumeReachable to 1.
+ *
+ * Does not set ServerTransportPlugin, because it's parsed before use.
+ *
+ * Does not set BridgeRecordUsageByCountry, because the tests don't have access
+ * to a GeoIPFile or GeoIPv6File. */
+static void
+setup_dir_formats_options(const char *arg, or_options_t *options)
+{
+  /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */
+  options->AssumeReachable = 1;
+
+  if (strchr(arg, 'b')) {
+    options->BridgeRelay = 1;
+  }
+
+  if (strchr(arg, 'e')) {
+    options->ExtraInfoStatistics = 1;
+  }
+
+  if (strchr(arg, 's')) {
+    options->DirReqStatistics = 1;
+    options->HiddenServiceStatistics = 1;
+    options->EntryStatistics = 1;
+    options->CellStatistics = 1;
+    options->ExitPortStatistics = 1;
+    options->ConnDirectionStatistics = 1;
+    options->PaddingStatistics = 1;
+  }
+}
+
+/* Check that routerinfos r1 and rp1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_ROUTERINFO_CONSISTENCY(r1, rp1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(rp1); \
+  tt_int_op(rp1->addr,OP_EQ, r1->addr); \
+  tt_int_op(rp1->or_port,OP_EQ, r1->or_port); \
+  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); \
+  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); \
+  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); \
+  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); \
+  crypto_pk_t *rp1_onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, \
+                                                      rp1->onion_pkey_len); \
+  crypto_pk_t *r1_onion_pkey = router_get_rsa_onion_pkey(r1->onion_pkey, \
+                                                      r1->onion_pkey_len); \
+  tt_int_op(crypto_pk_cmp_keys(rp1_onion_pkey, r1_onion_pkey), OP_EQ, 0); \
+  crypto_pk_free(rp1_onion_pkey); \
+  crypto_pk_free(r1_onion_pkey); \
+  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, r1->identity_pkey), \
+            OP_EQ, 0); \
+  tt_int_op(rp1->supports_tunnelled_dir_requests, OP_EQ, \
+            r1->supports_tunnelled_dir_requests); \
+STMT_END
+
+/* Check that routerinfo r1 and extrainfo e1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_EXTRAINFO_CONSISTENCY(r1, e1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(e1); \
+\
+  tt_str_op(e1->nickname, OP_EQ, r1->nickname); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA-only
+ * router. Tor versions without ed25519 (0.2.6 and earlier) are no longer
+ * officially supported, but the authorities still accept their descriptors.
+ */
+static void
+test_dir_formats_rsa(void *arg)
 {
   char *buf = NULL;
-  char buf2[8192];
-  char platform[256];
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *pk1_str = NULL, *pk2_str = NULL, *cp;
-  size_t pk1_str_len, pk2_str_len;
-  routerinfo_t *r1=NULL, *r2=NULL;
-  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
-  routerinfo_t *rp1 = NULL, *rp2 = NULL;
-  addr_policy_t *ex1, *ex2;
-  routerlist_t *dir1 = NULL, *dir2 = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
   uint8_t *rsa_cc = NULL;
-  or_options_t *options = get_options_mutable();
-  const addr_policy_t *p;
-  time_t now = time(NULL);
-  port_cfg_t orport, dirport;
-  char cert_buf[256];
 
-  (void)arg;
-  pk1 = pk_generate(0);
-  pk2 = pk_generate(1);
+  routerinfo_t *r1 = NULL;
+  extrainfo_t *e1 = NULL;
+  routerinfo_t *rp1 = NULL;
+  extrainfo_t *ep1 = NULL;
 
-  tt_assert(pk1 && pk2);
+  smartlist_t *chunks = NULL;
+  const char *msg = NULL;
+  int rv = -1;
+
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
 
   hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
 
-  get_platform_str(platform, sizeof(platform));
-  r1 = tor_malloc_zero(sizeof(routerinfo_t));
-  r1->addr = 0xc0a80001u; /* 192.168.0.1 */
-  r1->cache_info.published_on = 0;
-  r1->or_port = 9000;
-  r1->dir_port = 9003;
-  r1->supports_tunnelled_dir_requests = 1;
-  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
-  r1->ipv6_orport = 9999;
-  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
-  /* Fake just enough of an ntor key to get by */
+  /* r1 is a minimal, RSA-only descriptor, with DirPort and IPv6 */
+  r1 = basic_routerinfo_new("Magri", 0xc0a80001u /* 192.168.0.1 */,
+                            9000, 9003,
+                            1000, 5000, 10000,
+                            0,
+                            NULL);
+
+ /* Fake just enough of an ntor key to get by */
   curve25519_keypair_t r1_onion_keypair;
   curve25519_keypair_generate(&r1_onion_keypair, 0);
   r1->onion_curve25519_pkey = tor_memdup(&r1_onion_keypair.pubkey,
                                          sizeof(curve25519_public_key_t));
-  r1->identity_pkey = crypto_pk_dup_key(pk2);
-  r1->bandwidthrate = 1000;
-  r1->bandwidthburst = 5000;
-  r1->bandwidthcapacity = 10000;
+
+  /* Now add IPv6 */
+  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
+  r1->ipv6_orport = 9999;
+
   r1->exit_policy = NULL;
-  r1->nickname = tor_strdup("Magri");
-  r1->platform = tor_strdup(platform);
 
-  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex1->policy_type = ADDR_POLICY_ACCEPT;
-  tor_addr_from_ipv4h(&ex1->addr, 0);
-  ex1->maskbits = 0;
-  ex1->prt_min = ex1->prt_max = 80;
-  ex2->policy_type = ADDR_POLICY_REJECT;
-  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
-  ex2->maskbits = 8;
-  ex2->prt_min = ex2->prt_max = 24;
-  r2 = tor_malloc_zero(sizeof(routerinfo_t));
-  r2->addr = 0x0a030201u; /* 10.3.2.1 */
+  /* XXXX+++ router_dump_to_string should really take this from ri. */
+  options->ContactInfo = tor_strdup("Magri White "
+                                    "<magri@elsewhere.example.com>");
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  tor_free(options->ContactInfo);
+  cleanup_mock_configured_ports();
+
+  /* Synthesise a router descriptor, without the signature */
+  chunks = smartlist_new();
+
+  smartlist_add(chunks, get_new_router_line(r1));
+  smartlist_add_strdup(chunks, "or-address [1:2:3:4::]:9999\n");
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r1));
+  smartlist_add(chunks, get_new_fingerprint_line(r1));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r1));
+
+  smartlist_add(chunks, get_new_onion_key_block(r1));
+  smartlist_add(chunks, get_new_signing_key_block(r1));
+
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
+
+  smartlist_add_strdup(chunks, "contact Magri White "
+                               "<magri@elsewhere.example.com>\n");
+
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r1_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "reject *:*\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
+
+  smartlist_add_strdup(chunks, "router-signature\n");
+
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+                             * twice */
+
+  tt_str_op(buf,OP_EQ, buf2);
+  tor_free(buf);
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  cleanup_mock_configured_ports();
+
+  /* Now, try to parse buf */
+  cp = buf;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  tor_free(buf);
+  routerinfo_free(rp1);
+
+  /* Test extrainfo creation.
+   * We avoid calling router_build_fresh_unsigned_routerinfo(), because it's
+   * too complex. Instead, we re-use the manually-created routerinfos.
+   */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r1, NULL);
+
+  /* router_build_fresh_signed_extrainfo() passes the result of
+   * get_master_signing_key_cert() directly to tor_cert_dup(), which fails on
+   * NULL. But we want a NULL ei->cache_info.signing_key_cert to test the
+   * non-ed key path.
+   */
+  MOCK(tor_cert_dup, mock_tor_cert_dup_null);
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  /* Test some of the low-level static functions. */
+  e1 = router_build_fresh_signed_extrainfo(r1);
+  tt_assert(e1);
+  router_update_routerinfo_from_extrainfo(r1, e1);
+  rv = router_dump_and_sign_routerinfo_descriptor_body(r1);
+  tt_assert(rv == 0);
+  msg = "";
+  rv = routerinfo_incompatible_with_extrainfo(r1->identity_pkey, e1,
+                                              &r1->cache_info, &msg);
+  /* If they are incompatible, fail and show the msg string */
+  tt_str_op(msg, OP_EQ, "");
+  tt_assert(rv == 0);
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  UNMOCK(tor_cert_dup);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, e1);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r1->cache_info.signed_descriptor_body);
+  cp = r1->cache_info.signed_descriptor_body;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  routerinfo_free(rp1);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e1->cache_info.signed_descriptor_body);
+  cp = e1->cache_info.signed_descriptor_body;
+  ep1 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, ep1);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep1);
+
+ done:
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->ContactInfo);
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r1);
+  routerinfo_free(rp1);
+
+  extrainfo_free(e1);
+  extrainfo_free(ep1);
+
+  tor_free(rsa_cc);
+
+  tor_free(buf);
+  tor_free(buf2);
+}
+
+/* Check that the exit policy in rp2 is as expected. */
+#define CHECK_PARSED_EXIT_POLICY(rp2) \
+STMT_BEGIN \
+  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); \
+ \
+  p = smartlist_get(rp2->exit_policy, 0); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); \
+  tt_assert(tor_addr_is_null(&p->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 0); \
+  tt_int_op(p->prt_min,OP_EQ, 80); \
+  tt_int_op(p->prt_max,OP_EQ, 80); \
+ \
+  p = smartlist_get(rp2->exit_policy, 1); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); \
+  tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 8); \
+  tt_int_op(p->prt_min,OP_EQ, 24); \
+  tt_int_op(p->prt_max,OP_EQ, 24); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA + ed25519
+ * router.
+ */
+static void
+test_dir_formats_rsa_ed25519(void *arg)
+{
+  char *buf = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
+  crypto_pk_t *r2_onion_pkey = NULL;
+  char cert_buf[256];
+  uint8_t *rsa_cc = NULL;
+  time_t now = time(NULL);
+
+  routerinfo_t *r2 = NULL;
+  extrainfo_t *e2 = NULL;
+  routerinfo_t *r2_out = NULL;
+  routerinfo_t *rp2 = NULL;
+  extrainfo_t *ep2 = NULL;
+  addr_policy_t *ex1, *ex2;
+  const addr_policy_t *p;
+
+  smartlist_t *chunks = NULL;
+  int rv = -1;
+
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
+
+  hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
+
+  /* r2 is a RSA + ed25519 descriptor, with an exit policy, but no DirPort or
+   * IPv6 */
+  r2 = basic_routerinfo_new("Fred", 0x0a030201u /* 10.3.2.1 */,
+                            9005, 0,
+                            3000, 3000, 3000,
+                            5,
+                            &r2_onion_pkey);
+
+  /* Fake just enough of an ntor key to get by */
+  curve25519_keypair_t r2_onion_keypair;
+  curve25519_keypair_generate(&r2_onion_keypair, 0);
+  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
+                                         sizeof(curve25519_public_key_t));
+
+  /* Now add relay ed25519 keys
+   * We can't use init_mock_ed_keys() here, because the keys are seeded */
   ed25519_keypair_t kp1, kp2;
   ed25519_secret_key_from_seed(&kp1.seckey,
                           (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
@@ -228,157 +952,78 @@ test_dir_formats(void *arg)
                                          &kp2.pubkey,
                                          now, 86400,
                                          CERT_FLAG_INCLUDE_SIGNING_KEY);
-  r2->platform = tor_strdup(platform);
-  r2->cache_info.published_on = 5;
-  r2->or_port = 9005;
-  r2->dir_port = 0;
-  r2->supports_tunnelled_dir_requests = 1;
-  router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len);
-  curve25519_keypair_t r2_onion_keypair;
-  curve25519_keypair_generate(&r2_onion_keypair, 0);
-  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
-                                         sizeof(curve25519_public_key_t));
-  r2->identity_pkey = crypto_pk_dup_key(pk1);
-  r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000;
+
+  /* Now add an exit policy */
+  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex1->policy_type = ADDR_POLICY_ACCEPT;
+  tor_addr_from_ipv4h(&ex1->addr, 0);
+  ex1->maskbits = 0;
+  ex1->prt_min = ex1->prt_max = 80;
+  ex2->policy_type = ADDR_POLICY_REJECT;
+  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
+  ex2->maskbits = 8;
+  ex2->prt_min = ex2->prt_max = 24;
+
   r2->exit_policy = smartlist_new();
   smartlist_add(r2->exit_policy, ex1);
   smartlist_add(r2->exit_policy, ex2);
-  r2->nickname = tor_strdup("Fred");
-
-  tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
-                                                    &pk1_str_len));
-  tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str,
-                                                    &pk2_str_len));
-
-  /* XXXX+++ router_dump_to_string should really take this from ri.*/
-  options->ContactInfo = tor_strdup("Magri White "
-                                    "<magri@elsewhere.example.com>");
-  /* Skip reachability checks for DirPort and tunnelled-dir-server */
-  options->AssumeReachable = 1;
-
-  /* Fake just enough of an ORPort and DirPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
 
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9000;
-  smartlist_add(mocked_configured_ports, &orport);
-
-  memset(&dirport, 0, sizeof(dirport));
-  dirport.type = CONN_TYPE_DIR_LISTENER;
-  dirport.addr.family = AF_INET;
-  dirport.port = 9003;
-  smartlist_add(mocked_configured_ports, &dirport);
-
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  tor_free(options->ContactInfo);
+  buf = router_dump_router_to_string(r2,
+                                     r2->identity_pkey, r2_onion_pkey,
+                                     &r2_onion_keypair, &kp2);
   tt_assert(buf);
 
-  strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n"
-          "or-address [1:2:3:4::]:9999\n"
-          "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:00\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-  /* XXX the "0" above is hard-coded, but even if we made it reflect
-   * uptime, that still wouldn't make it right, because the two
-   * descriptors might be made on different seconds... hm. */
-         "bandwidth 1000 5000 10000\n"
-          "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "contact Magri White <magri@elsewhere.example.com>\n",
-          sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r1_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "reject *:*\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\nrouter-signature\n", sizeof(buf2));
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
-                             * twice */
+  cleanup_mock_configured_ports();
 
-  tt_str_op(buf,OP_EQ, buf2);
-  tor_free(buf);
+  chunks = smartlist_new();
 
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-  tt_assert(buf);
-  cp = buf;
-  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp1);
-  tt_int_op(rp1->addr,OP_EQ, r1->addr);
-  tt_int_op(rp1->or_port,OP_EQ, r1->or_port);
-  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port);
-  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate);
-  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst);
-  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity);
-  crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey,
-                                                      rp1->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0);
-  tt_assert(rp1->supports_tunnelled_dir_requests);
-  //tt_assert(rp1->exit_policy == NULL);
-  tor_free(buf);
+  /* Synthesise a router descriptor, without the signatures */
+  smartlist_add(chunks, get_new_router_line(r2));
 
-  strlcpy(buf2,
-          "router Fred 10.3.2.1 9005 0 0\n"
-          "identity-ed25519\n"
-          "-----BEGIN ED25519 CERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks,
+                       "identity-ed25519\n"
+                       "-----BEGIN ED25519 CERT-----\n");
   base64_encode(cert_buf, sizeof(cert_buf),
                 (const char*)r2->cache_info.signing_key_cert->encoded,
                 r2->cache_info.signing_key_cert->encoded_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END ED25519 CERT-----\n", sizeof(buf2));
-  strlcat(buf2, "master-key-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END ED25519 CERT-----\n");
+
+  smartlist_add_strdup(chunks, "master-key-ed25519 ");
   {
     char k[ED25519_BASE64_LEN+1];
-    tt_int_op(ed25519_public_to_base64(k,
-                          &r2->cache_info.signing_key_cert->signing_key),
-              OP_GE, 0);
-    strlcat(buf2, k, sizeof(buf2));
-    strlcat(buf2, "\n", sizeof(buf2));
+    ed25519_public_to_base64(k, &r2->cache_info.signing_key_cert->signing_key);
+    smartlist_add_strdup(chunks, k);
+    smartlist_add_strdup(chunks, "\n");
   }
-  strlcat(buf2, "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:05\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk1, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-          "bandwidth 3000 3000 3000\n", sizeof(buf2));
-  strlcat(buf2, "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r2));
+  smartlist_add(chunks, get_new_fingerprint_line(r2));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r2));
+
+  smartlist_add(chunks, get_new_onion_key_block(r2));
+  smartlist_add(chunks, get_new_signing_key_block(r2));
+
   int rsa_cc_len;
-  rsa_cc = make_tap_onion_key_crosscert(pk2,
+  rsa_cc = make_tap_onion_key_crosscert(r2_onion_pkey,
                                         &kp1.pubkey,
-                                        pk1,
+                                        r2->identity_pkey,
                                         &rsa_cc_len);
   tt_assert(rsa_cc);
   base64_encode(cert_buf, sizeof(cert_buf), (char*)rsa_cc, rsa_cc_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, "onion-key-crosscert\n"
-          "-----BEGIN CROSSCERT-----\n", sizeof(buf2));
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END CROSSCERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks, "onion-key-crosscert\n"
+                       "-----BEGIN CROSSCERT-----\n");
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END CROSSCERT-----\n");
   int ntor_cc_sign;
   {
     tor_cert_t *ntor_cc = NULL;
@@ -393,112 +1038,165 @@ test_dir_formats(void *arg)
                 BASE64_ENCODE_MULTILINE);
     tor_cert_free(ntor_cc);
   }
-  tor_snprintf(buf2+strlen(buf2), sizeof(buf2)-strlen(buf2),
+  smartlist_add_asprintf(chunks,
                "ntor-onion-key-crosscert %d\n"
                "-----BEGIN ED25519 CERT-----\n"
                "%s"
                "-----END ED25519 CERT-----\n", ntor_cc_sign, cert_buf);
 
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r2_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "accept *:80\nreject 18.0.0.0/8:24\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\n", sizeof(buf2));
-  strlcat(buf2, "router-sig-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
 
-  /* Fake just enough of an ORPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r2_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "accept *:80\nreject 18.0.0.0/8:24\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
 
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9005;
-  smartlist_add(mocked_configured_ports, &orport);
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
 
-  buf = router_dump_router_to_string(r2, pk1, pk2, &r2_onion_keypair, &kp2);
-  tt_assert(buf);
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare either sig; they're never the same
                              * twice */
 
   tt_str_op(buf, OP_EQ, buf2);
   tor_free(buf);
 
-  buf = router_dump_router_to_string(r2, pk1, NULL, NULL, NULL);
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  buf = router_dump_router_to_string(r2, r2->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
 
-  /* Reset for later */
+  cleanup_mock_configured_ports();
+
+  /* Now, try to parse buf */
   cp = buf;
   rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp2);
-  tt_int_op(rp2->addr,OP_EQ, r2->addr);
-  tt_int_op(rp2->or_port,OP_EQ, r2->or_port);
-  tt_int_op(rp2->dir_port,OP_EQ, r2->dir_port);
-  tt_int_op(rp2->bandwidthrate,OP_EQ, r2->bandwidthrate);
-  tt_int_op(rp2->bandwidthburst,OP_EQ, r2->bandwidthburst);
-  tt_int_op(rp2->bandwidthcapacity,OP_EQ, r2->bandwidthcapacity);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
   tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
              r2->onion_curve25519_pkey->public_key,
              CURVE25519_PUBKEY_LEN);
-  onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey,
-                                         rp2->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0);
-  tt_assert(rp2->supports_tunnelled_dir_requests);
-
-  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2);
-
-  p = smartlist_get(rp2->exit_policy, 0);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT);
-  tt_assert(tor_addr_is_null(&p->addr));
-  tt_int_op(p->maskbits,OP_EQ, 0);
-  tt_int_op(p->prt_min,OP_EQ, 80);
-  tt_int_op(p->prt_max,OP_EQ, 80);
-
-  p = smartlist_get(rp2->exit_policy, 1);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT);
-  tt_assert(tor_addr_eq(&p->addr, &ex2->addr));
-  tt_int_op(p->maskbits,OP_EQ, 8);
-  tt_int_op(p->prt_min,OP_EQ, 24);
-  tt_int_op(p->prt_max,OP_EQ, 24);
-
-#if 0
-  /* Okay, now for the directories. */
-  {
-    fingerprint_list = smartlist_new();
-    crypto_pk_get_fingerprint(pk2, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
-    crypto_pk_get_fingerprint(pk1, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  tor_free(buf);
+  routerinfo_free(rp2);
+
+  /* Test extrainfo creation. */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r2, r2_onion_pkey);
+
+  /* router_build_fresh_descriptor() requires
+   * router_build_fresh_unsigned_routerinfo(), but the implementation is
+   * too complex. Instead, we re-use r2.
+   */
+  mocked_routerinfo = r2;
+  MOCK(router_build_fresh_unsigned_routerinfo,
+       mock_router_build_fresh_unsigned_routerinfo);
+
+  /* r2 uses ed25519, so we need to mock the ed key functions */
+  mocked_master_signing_key = &kp2;
+  MOCK(get_master_signing_keypair, mock_get_master_signing_keypair);
+
+  mocked_signing_key_cert = r2->cache_info.signing_key_cert;
+  MOCK(get_master_signing_key_cert, mock_get_master_signing_key_cert);
+
+  mocked_curve25519_onion_key = &r2_onion_keypair;
+  MOCK(get_current_curve25519_keypair, mock_get_current_curve25519_keypair);
+
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
+
+  /* Test the high-level interface. */
+  rv = router_build_fresh_descriptor(&r2_out, &e2);
+  if (rv < 0) {
+    /* router_build_fresh_descriptor() frees r2 on failure. */
+    r2 = NULL;
+    /* Get rid of an alias to rp2 */
+    r2_out = NULL;
   }
+  tt_assert(rv == 0);
+  tt_assert(r2_out);
+  tt_assert(e2);
+  /* Guaranteed by mock_router_build_fresh_unsigned_routerinfo() */
+  tt_ptr_op(r2_out, OP_EQ, r2);
+  /* Get rid of an alias to r2 */
+  r2_out = NULL;
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  mocked_routerinfo = NULL;
+  UNMOCK(router_build_fresh_unsigned_routerinfo);
+  mocked_master_signing_key = NULL;
+  UNMOCK(get_master_signing_keypair);
+  mocked_signing_key_cert = NULL;
+  UNMOCK(get_master_signing_key_cert);
+  mocked_curve25519_onion_key = NULL;
+  UNMOCK(get_current_curve25519_keypair);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, e2);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r2->cache_info.signed_descriptor_body);
+  cp = r2->cache_info.signed_descriptor_body;
+  rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
 
-#endif /* 0 */
-  dirserv_free_fingerprint_list();
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
+  tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
+            r2->onion_curve25519_pkey->public_key,
+            CURVE25519_PUBKEY_LEN);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  routerinfo_free(rp2);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e2->cache_info.signed_descriptor_body);
+  cp = e2->cache_info.signed_descriptor_body;
+  ep2 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, ep2);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep2);
 
  done:
-  if (r1)
-    routerinfo_free(r1);
-  if (r2)
-    routerinfo_free(r2);
-  if (rp2)
-    routerinfo_free(rp2);
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r2);
+  routerinfo_free(r2_out);
+  routerinfo_free(rp2);
+
+  extrainfo_free(e2);
+  extrainfo_free(ep2);
 
   tor_free(rsa_cc);
+  crypto_pk_free(r2_onion_pkey);
+
   tor_free(buf);
-  tor_free(pk1_str);
-  tor_free(pk2_str);
-  if (pk1) crypto_pk_free(pk1);
-  if (pk2) crypto_pk_free(pk2);
-  if (rp1) routerinfo_free(rp1);
-  tor_free(dir1); /* XXXX And more !*/
-  tor_free(dir2); /* And more !*/
+  tor_free(buf2);
 }
 
 #include "failing_routerdescs.inc"
@@ -1767,7 +2465,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, "", 0);
   setup_capture_of_logs(LOG_WARN);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
   expect_log_msg("Empty bandwidth file\n");
   teardown_capture_of_logs();
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1783,7 +2482,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op("", OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1794,7 +2495,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, header_lines_v100, 0);
   bw_file_headers = smartlist_new();
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1807,7 +2510,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1818,7 +2523,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100);
   write_str_to_file(fname, content, 0);
   tor_free(content);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
 
   /* Test bandwidth file including v1.1.0 bandwidth headers and
    * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */
@@ -1828,7 +2534,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1844,7 +2552,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1861,7 +2571,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1872,7 +2584,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110_no_terminator, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1883,7 +2597,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1898,7 +2614,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1913,7 +2631,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1929,7 +2649,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1946,7 +2668,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1964,7 +2688,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1985,7 +2711,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   /* force bw_file_headers to be bigger than
@@ -2014,7 +2742,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
@@ -2032,11 +2761,13 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_not_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
  done:
+  unlink(fname);
   tor_free(fname);
   tor_free(header_lines_v100);
   tor_free(header_lines_v110_no_terminator);
@@ -2274,6 +3005,7 @@ test_dir_param_voting_lookup(void *arg)
   tt_int_op(99, OP_EQ,
             dirvote_get_intermediate_param_value(lst, "abcd", 1000));
 
+#ifndef ALL_BUGS_ARE_FATAL
   /* moomin appears twice. That's a bug. */
   tor_capture_bugs_(1);
   tt_int_op(-100, OP_EQ,
@@ -2291,7 +3023,7 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "jack", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(! ok)");
+            "!(!ok)");
   tor_end_capture_bugs_();
   /* electricity and opa aren't integers. */
   tor_capture_bugs_(1);
@@ -2299,7 +3031,7 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "electricity", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(! ok)");
+            "!(!ok)");
   tor_end_capture_bugs_();
 
   tor_capture_bugs_(1);
@@ -2307,8 +3039,9 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "opa", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(! ok)");
+            "!(!ok)");
   tor_end_capture_bugs_();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
 
  done:
   SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp));
@@ -2863,17 +3596,23 @@ test_a_networkstatus(
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   /* Parse certificates and keys. */
-  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
   tt_assert(cert1);
-  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                           strlen(AUTHORITY_CERT_2),
+                                           NULL);
   tt_assert(cert2);
-  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                           strlen(AUTHORITY_CERT_3),
+                                           NULL);
   tt_assert(cert3);
   sign_skey_1 = crypto_pk_new();
   sign_skey_2 = crypto_pk_new();
   sign_skey_3 = crypto_pk_new();
   sign_skey_leg1 = pk_generate(4);
-  voting_schedule_recalculate_timing(get_options(), now);
+  dirauth_sched_recalculate_timing(get_options(), now);
   sr_state_init(0, 0);
 
   tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1,
@@ -2969,7 +3708,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_NS);
   tt_assert(consensus_text);
-  con = networkstatus_parse_vote_from_string(consensus_text, NULL,
+  con = networkstatus_parse_vote_from_string_(consensus_text, NULL,
                                              NS_TYPE_CONSENSUS);
   tt_assert(con);
   //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n",
@@ -2981,7 +3720,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_MICRODESC);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -3080,13 +3819,13 @@ test_a_networkstatus(
     tt_assert(consensus_text3);
     tt_assert(consensus_text_md2);
     tt_assert(consensus_text_md3);
-    con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL,
+    con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL,
+    con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL,
+    con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL,
+    con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL,
                                                 NS_TYPE_CONSENSUS);
     tt_assert(con2);
     tt_assert(con3);
@@ -3864,6 +4603,62 @@ mock_get_options(void)
   return mock_options;
 }
 
+/**
+ * Test dirauth_get_b64_digest_bw_file.
+ * This function should be near the other bwauth functions, but it needs
+ * mock_get_options, that is only defined here.
+ */
+
+static void
+test_dir_bwauth_bw_file_digest256(void *arg)
+{
+  (void)arg;
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  char *fname = tor_strdup(get_fname("V3BandwidthsFile"));
+  /* Initialize to a wrong digest. */
+  uint8_t digest[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+
+  /* Digest of an empty string. Initialize to a wrong digest. */
+  char digest_empty_str[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_empty_str, "", 0, DIGEST_SHA256);
+
+  /* Digest of the content. Initialize to a wrong digest. */
+  char digest_expected[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_expected, content, strlen(content), DIGEST_SHA256);
+
+  /* When the bandwidth file can not be found. */
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_empty_str, DIGEST256_LEN);
+
+  /* When there is a timestamp but it is too old. */
+  write_str_to_file(fname, content, 0);
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  /* The digest will be correct. */
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+  update_approx_time(1541171221);
+
+  /* When there is a bandwidth file and it can be read. */
+  tt_int_op(0, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+ done:
+  unlink(fname);
+  tor_free(fname);
+  update_approx_time(time(NULL));
+}
+
 static void
 reset_routerstatus(routerstatus_t *rs,
                    const char *hex_identity_digest,
@@ -3900,10 +4695,13 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   (void)arg;
 
   /* Init options */
+  dirauth_options_t *dirauth_options =
+    tor_malloc_zero(sizeof(dirauth_options_t));
+
   mock_options = tor_malloc(sizeof(or_options_t));
   reset_options(mock_options, &mock_get_options_calls);
-
   MOCK(get_options, mock_get_options);
+  dirauth_set_options(dirauth_options);
 
   /* Init routersets */
   routerset_t *routerset_all  = routerset_new();
@@ -3943,16 +4741,15 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check that "*" sets flags on all routers: Exit
    * Check the flags aren't being confused with each other */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_all;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_all;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_exit, OP_EQ, 1);
   tt_uint_op(rs_b->is_exit, OP_EQ, 1);
@@ -3965,18 +4762,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check that "*" sets flags on all routers: Guard & HSDir
    * Cover the remaining flags in one test */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteGuard = routerset_all;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_all;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_all;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_all;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_possible_guard, OP_EQ, 1);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
@@ -3989,20 +4785,19 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check routerset A sets all flags on router A,
    * but leaves router B unmodified */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_a;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
-  mock_options->TestingDirAuthVoteGuard = routerset_a;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_a;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_a;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_a;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_a;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_exit, OP_EQ, 1);
   tt_uint_op(rs_b->is_exit, OP_EQ, 0);
@@ -4013,21 +4808,21 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
 
   /* Check routerset A unsets all flags on router B when Strict is set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_a;
-  mock_options->TestingDirAuthVoteExitIsStrict = 1;
-  mock_options->TestingDirAuthVoteGuard = routerset_a;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 1;
-  mock_options->TestingDirAuthVoteHSDir = routerset_a;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteExit = routerset_a;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_a;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_a;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 1;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
   tt_uint_op(rs_b->is_exit, OP_EQ, 0);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
@@ -4035,21 +4830,21 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
 
   /* Check routerset A doesn't modify flags on router B without Strict set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_a;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
-  mock_options->TestingDirAuthVoteGuard = routerset_a;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_a;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_a;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_a;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_a;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
   tt_uint_op(rs_b->is_exit, OP_EQ, 1);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
@@ -4058,21 +4853,21 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check the empty routerset zeroes all flags
    * on routers A & B with Strict set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_none;
-  mock_options->TestingDirAuthVoteExitIsStrict = 1;
-  mock_options->TestingDirAuthVoteGuard = routerset_none;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 1;
-  mock_options->TestingDirAuthVoteHSDir = routerset_none;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteExit = routerset_none;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_none;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_none;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 1;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
   tt_uint_op(rs_b->is_exit, OP_EQ, 0);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
@@ -4081,24 +4876,23 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check the empty routerset doesn't modify any flags
    * on A or B without Strict set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_none;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
-  mock_options->TestingDirAuthVoteGuard = routerset_none;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_none;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_none;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_none;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_none;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_exit, OP_EQ, 0);
   tt_uint_op(rs_a->is_possible_guard, OP_EQ, 0);
@@ -4109,6 +4903,7 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
 
  done:
   tor_free(mock_options);
+  tor_free(dirauth_options);
   mock_options = NULL;
 
   UNMOCK(get_options);
@@ -4196,6 +4991,14 @@ test_dir_purpose_needs_anonymity_returns_true_by_default(void *arg)
 {
   (void)arg;
 
+#ifdef ALL_BUGS_ARE_FATAL
+  /* Coverity (and maybe clang analyser) complain that the code following
+   * tt_skip() is unconditionally unreachable. */
+#if !defined(__COVERITY__) && !defined(__clang_analyzer__)
+  tt_skip();
+#endif
+#endif /* defined(ALL_BUGS_ARE_FATAL) */
+
   tor_capture_bugs_(1);
   setup_full_capture_of_logs(LOG_WARN);
   tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, 0, NULL));
@@ -4683,15 +5486,15 @@ test_dir_conn_purpose_to_string(void *data)
   teardown_capture_of_logs();
 }
 
-NS_DECL(int,
-public_server_mode, (const or_options_t *options));
+static int dir_tests_public_server_mode(const or_options_t *options);
+ATTR_UNUSED static int dir_tests_public_server_mode_called = 0;
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+dir_tests_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
-  if (CALLED(public_server_mode)++ == 0) {
+  if (dir_tests_public_server_mode_called++ == 0) {
     return 1;
   }
 
@@ -4705,13 +5508,14 @@ test_dir_should_use_directory_guards(void *data)
   char *errmsg = NULL;
   (void)data;
 
-  NS_MOCK(public_server_mode);
+  MOCK(public_server_mode,
+       dir_tests_public_server_mode);
 
   options = options_new();
   options_init(options);
 
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 1);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 1);
 
   options->UseEntryGuards = 1;
   options->DownloadExtraInfo = 0;
@@ -4719,41 +5523,41 @@ test_dir_should_use_directory_guards(void *data)
   options->FetchDirInfoExtraEarly = 0;
   options->FetchUselessDescriptors = 0;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 1);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 2);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 2);
 
   options->UseEntryGuards = 0;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 3);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 3);
   options->UseEntryGuards = 1;
 
   options->DownloadExtraInfo = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 4);
   options->DownloadExtraInfo = 0;
 
   options->FetchDirInfoEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 5);
   options->FetchDirInfoEarly = 0;
 
   options->FetchDirInfoExtraEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 6);
   options->FetchDirInfoExtraEarly = 0;
 
   options->FetchUselessDescriptors = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 7);
   options->FetchUselessDescriptors = 0;
 
   done:
-    NS_UNMOCK(public_server_mode);
+    UNMOCK(public_server_mode);
     or_options_free(options);
     tor_free(errmsg);
 }
 
-NS_DECL(void,
-directory_initiate_request, (directory_request_t *req));
+static void dir_tests_directory_initiate_request(directory_request_t *req);
+ATTR_UNUSED static int dir_tests_directory_initiate_request_called = 0;
 
 static void
 test_dir_should_not_init_request_to_ourselves(void *data)
@@ -4763,7 +5567,8 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   crypto_pk_t *key = pk_generate(2);
   (void) data;
 
-  NS_MOCK(directory_initiate_request);
+  MOCK(directory_initiate_request,
+       dir_tests_directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4778,15 +5583,15 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   dir_server_add(ourself);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
 
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_request);
+    UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
     crypto_pk_free(key);
@@ -4800,7 +5605,8 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
                                 | MICRODESC_DIRINFO;
   (void) data;
 
-  NS_MOCK(directory_initiate_request);
+  MOCK(directory_initiate_request,
+       dir_tests_directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4811,14 +5617,14 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_request);
+    UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
@@ -4829,7 +5635,8 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_t *ds = NULL;
   (void) data;
 
-  NS_MOCK(directory_initiate_request);
+  MOCK(directory_initiate_request,
+       dir_tests_directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4840,23 +5647,23 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 1);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 1);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 2);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 2);
 
   done:
-    NS_UNMOCK(directory_initiate_request);
+    UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
 
 void
-NS(directory_initiate_request)(directory_request_t *req)
+dir_tests_directory_initiate_request(directory_request_t *req)
 {
   (void)req;
-  CALLED(directory_initiate_request)++;
+  dir_tests_directory_initiate_request_called++;
 }
 
 static void
@@ -6087,6 +6894,80 @@ test_dir_find_dl_min_delay(void* data)
 }
 
 static void
+test_dir_matching_flags(void *arg)
+{
+  (void) arg;
+  routerstatus_t *rs_noflags = NULL;
+  routerstatus_t *rs = NULL;
+  char *s = NULL;
+
+  smartlist_t *tokens = smartlist_new();
+  memarea_t *area = memarea_new();
+
+  int expected_val_when_unused = 0;
+
+  const char *ex_noflags =
+    "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 "
+       "192.168.0.1 9001 0\n"
+    "m thisoneislongerbecauseitisa256bitmddigest33\n"
+    "s\n";
+  const char *cp = ex_noflags;
+  rs_noflags = routerstatus_parse_entry_from_string(
+         area, &cp,
+         cp + strlen(cp),
+         tokens, NULL, NULL,
+         MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);
+  tt_assert(rs_noflags);
+
+#define FLAG(string, field) STMT_BEGIN {        \
+    tor_asprintf(&s,\
+                 "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \
+                 "192.168.0.1 9001 0\n"                                 \
+                 "m thisoneislongerbecauseitisa256bitmddigest33\n"      \
+                 "s %s\n", string);                                     \
+    cp = s;                                                             \
+    rs =  routerstatus_parse_entry_from_string(                         \
+      area, &cp,                                                        \
+      cp + strlen(cp),                                                  \
+      tokens, NULL, NULL,                                               \
+      MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);                  \
+    /* the field should usually be 0 when no flags are listed */        \
+    tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused);      \
+    /* the field should be 1 when this flags islisted */                \
+    tt_int_op(rs->field, OP_EQ, 1);                                     \
+    tor_free(s);                                                        \
+    routerstatus_free(rs);                                              \
+} STMT_END
+
+  FLAG("Authority", is_authority);
+  FLAG("BadExit", is_bad_exit);
+  FLAG("Exit", is_exit);
+  FLAG("Fast", is_fast);
+  FLAG("Guard", is_possible_guard);
+  FLAG("HSDir", is_hs_dir);
+  FLAG("Stable", is_stable);
+  FLAG("StaleDesc", is_staledesc);
+  FLAG("V2Dir", is_v2_dir);
+
+  // These flags are assumed to be set whether they're declared or not.
+  expected_val_when_unused = 1;
+  FLAG("Running", is_flagged_running);
+  FLAG("Valid", is_valid);
+  expected_val_when_unused = 0;
+
+  // These flags are no longer used, but still parsed.
+  FLAG("Named", is_named);
+  FLAG("Unnamed", is_unnamed);
+
+ done:
+  tor_free(s);
+  routerstatus_free(rs);
+  routerstatus_free(rs_noflags);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+}
+
+static void
 test_dir_assumed_flags(void *arg)
 {
   (void)arg;
@@ -6101,9 +6982,10 @@ test_dir_assumed_flags(void *arg)
        "192.168.0.1 9001 0\n"
     "m thisoneislongerbecauseitisa256bitmddigest33\n"
     "s Fast Guard Stable\n";
+  const char *eos = str1 + strlen(str1);
 
   const char *cp = str1;
-  rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
+  rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL,
                                             24, FLAV_MICRODESC);
   tt_assert(rs);
   tt_assert(rs->is_flagged_running);
@@ -6201,98 +7083,6 @@ test_dir_platform_str(void *arg)
   ;
 }
 
-static networkstatus_t *mock_networkstatus;
-
-static networkstatus_t *
-mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
-{
-  (void)f;
-  return mock_networkstatus;
-}
-
-static void
-test_dir_networkstatus_consensus_has_ipv6(void *arg)
-{
-  (void)arg;
-
-  int has_ipv6 = 0;
-
-  /* Init options and networkstatus */
-  or_options_t our_options;
-  mock_options = &our_options;
-  reset_options(mock_options, &mock_get_options_calls);
-  MOCK(get_options, mock_get_options);
-
-  networkstatus_t our_networkstatus;
-  mock_networkstatus = &our_networkstatus;
-  memset(mock_networkstatus, 0, sizeof(*mock_networkstatus));
-  MOCK(networkstatus_get_latest_consensus_by_flavor,
-       mock_networkstatus_get_latest_consensus_by_flavor);
-
-  /* A live consensus */
-  mock_networkstatus->valid_after = time(NULL) - 3600;
-  mock_networkstatus->valid_until = time(NULL) + 3600;
-
-  /* Test the bounds for A lines in the NS consensus */
-  mock_options->UseMicrodescriptors = 0;
-
-  mock_networkstatus->consensus_method = MIN_SUPPORTED_CONSENSUS_METHOD;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  /* Test the bounds for A lines in the microdesc consensus */
-  mock_options->UseMicrodescriptors = 1;
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD + 20;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 1;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 20;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS - 1;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(!has_ipv6);
-
-  /* Test the edge cases */
-  mock_options->UseMicrodescriptors = 1;
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
-
-  /* Reasonably live */
-  mock_networkstatus->valid_until = approx_time() - 60;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  /* Not reasonably live */
-  mock_networkstatus->valid_after = approx_time() - 24*60*60 - 3600;
-  mock_networkstatus->valid_until = approx_time() - 24*60*60 - 60;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(!has_ipv6);
-
-  /* NULL consensus */
-  mock_networkstatus = NULL;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(!has_ipv6);
-
- done:
-  UNMOCK(get_options);
-  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
-}
-
 static void
 test_dir_format_versions_list(void *arg)
 {
@@ -6344,6 +7134,301 @@ test_dir_format_versions_list(void *arg)
   teardown_capture_of_logs();
 }
 
+static void
+test_dir_add_fingerprint(void *arg)
+{
+  (void)arg;
+  authdir_config_t *list;
+  int ret;
+  ed25519_secret_key_t seckey;
+  ed25519_public_key_t pubkey_good, pubkey_bad;
+
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+
+  setup_capture_of_logs(LOG_WARN);
+
+  /* RSA test - successful */
+  ret = add_rsa_fingerprint_to_dir("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+                                   list, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* RSA test - failure */
+  ret = add_rsa_fingerprint_to_dir("ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ",
+                                   list, 0);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* ed25519 test - successful */
+  ed25519_secret_key_generate(&seckey, 0);
+  ed25519_public_key_generate(&pubkey_good, &seckey);
+
+  ret = add_ed25519_to_dir(&pubkey_good, list, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* ed25519 test - failure */
+  digest256_from_base64((char *) pubkey_bad.pubkey, "gibberish");
+
+  ret = add_ed25519_to_dir(&pubkey_bad, list, 0);
+  tt_int_op(ret, OP_EQ, -1);
+
+ done:
+  teardown_capture_of_logs();
+  dirserv_free_fingerprint_list();
+}
+
+static void
+test_dir_dirserv_load_fingerprint_file(void *arg)
+{
+  (void)arg;
+  char *fname = tor_strdup(get_fname("approved-routers"));
+
+  // Neither RSA nor ed25519
+  const char *router_lines_invalid =
+    "!badexit notafingerprint";
+  const char *router_lines_too_long =
+    "!badexit thisisareallylongstringthatislongerthanafingerprint\n";
+  const char *router_lines_bad_fmt_str =
+    "!badexit ABCDEFGH|%1$p|%2$p|%3$p|%4$p|%5$p|%6$p\n";
+  const char *router_lines_valid_rsa =
+    "!badexit AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n";
+  const char *router_lines_invalid_rsa =
+    "!badexit ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ\n";
+  const char *router_lines_valid_ed25519 =
+    "!badexit wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+  const char *router_lines_invalid_ed25519 =
+    "!badexit --fLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx--\n";
+
+  // Test: Invalid Fingerprint (not RSA or ed25519)
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_invalid, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Very long string (longer than RSA or ed25519 key)
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_too_long, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Formt string exploit
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_bad_fmt_str, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Valid RSA
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_valid_rsa, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  teardown_capture_of_logs();
+
+  // Test: Invalid RSA
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_invalid_rsa, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Valid ed25519
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_valid_ed25519, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  teardown_capture_of_logs();
+
+  // Test: Invalid ed25519
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_invalid_ed25519, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+ done:
+  tor_free(fname);
+  dirserv_free_fingerprint_list();
+}
+
+#define RESET_FP_LIST(list) STMT_BEGIN \
+    dirserv_free_fingerprint_list(); \
+    authdir_init_fingerprint_list(); \
+    list = authdir_return_fingerprint_list(); \
+  STMT_END
+
+static void
+test_dir_dirserv_router_get_status(void *arg)
+{
+  authdir_config_t *list;
+  routerinfo_t *ri = NULL;
+  ed25519_keypair_t kp1, kp2;
+  char d[DIGEST_LEN];
+  char fp[HEX_DIGEST_LEN+1];
+  int ret;
+  const char *msg;
+  time_t now = time(NULL);
+
+  (void)arg;
+
+  crypto_pk_t *pk = pk_generate(0);
+
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+
+  /* Set up the routerinfo */
+  ri = tor_malloc_zero(sizeof(routerinfo_t));
+  ri->addr = 0xc0a80001u;
+  ri->or_port = 9001;
+  ri->platform = tor_strdup("0.4.0.1-alpha");
+  ri->nickname = tor_strdup("Jessica");
+  ri->identity_pkey = crypto_pk_dup_key(pk);
+
+  curve25519_keypair_t ri_onion_keypair;
+  curve25519_keypair_generate(&ri_onion_keypair, 0);
+  ri->onion_curve25519_pkey = tor_memdup(&ri_onion_keypair.pubkey,
+                                         sizeof(curve25519_public_key_t));
+
+  ed25519_secret_key_from_seed(&kp1.seckey,
+                          (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
+  ed25519_public_key_generate(&kp1.pubkey, &kp1.seckey);
+  ed25519_secret_key_from_seed(&kp2.seckey,
+                          (const uint8_t*)"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
+  ed25519_public_key_generate(&kp2.pubkey, &kp2.seckey);
+  ri->cache_info.signing_key_cert = tor_cert_create(&kp1,
+                                         CERT_TYPE_ID_SIGNING,
+                                         &kp2.pubkey,
+                                         now, 86400,
+                                         CERT_FLAG_INCLUDE_SIGNING_KEY);
+
+  crypto_pk_get_digest(ri->identity_pkey, d);
+  base16_encode(fp, HEX_DIGEST_LEN + 1, d, DIGEST_LEN);
+
+  /* Try on an empty fingerprint list */
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  /* Try an accepted router */
+  add_rsa_fingerprint_to_dir(fp, list, 0);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp1.pubkey, list, 0);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  /* Try a rejected router */
+  add_rsa_fingerprint_to_dir(fp, list, RTR_REJECT);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, RTR_REJECT);
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp1.pubkey, list, RTR_REJECT);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, RTR_REJECT);
+  RESET_FP_LIST(list);
+
+ done:
+  dirserv_free_fingerprint_list();
+  routerinfo_free(ri);
+  crypto_pk_free(pk);
+}
+
+static void
+test_dir_dirserv_would_reject_router(void *arg)
+{
+  authdir_config_t *list;
+  routerstatus_t rs;
+  vote_routerstatus_t vrs;
+  ed25519_keypair_t kp;
+  char fp[HEX_DIGEST_LEN+1];
+
+  (void)arg;
+
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+
+  /* Set up the routerstatus */
+  memset(&rs, 0, sizeof(rs));
+  rs.addr = 0xc0a80001u;
+  rs.or_port = 9001;
+  strlcpy(rs.nickname, "Nicole", sizeof(rs.nickname));
+  memcpy(rs.identity_digest, "Cloud nine is great ", DIGEST_LEN);
+
+  ed25519_secret_key_from_seed(&kp.seckey,
+                          (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
+  ed25519_public_key_generate(&kp.pubkey, &kp.seckey);
+
+  base16_encode(fp, HEX_DIGEST_LEN + 1, rs.identity_digest, DIGEST_LEN);
+
+  /* Setup the vote_routerstatus_t. */
+  memcpy(vrs.ed25519_id, &kp.pubkey, ED25519_PUBKEY_LEN);
+
+  /* Try an empty fingerprint list */
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  /* Try an accepted router */
+  add_rsa_fingerprint_to_dir(fp, list, 0);
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp.pubkey, list, 0);
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  /* Try a rejected router */
+  add_rsa_fingerprint_to_dir(fp, list, RTR_REJECT);
+  tt_assert(dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp.pubkey, list, RTR_REJECT);
+  tt_assert(dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+ done:
+  dirserv_free_fingerprint_list();
+}
+
+static void
+test_dir_dirserv_add_own_fingerprint(void *arg)
+{
+  authdir_config_t *list;
+  char digest[DIGEST_LEN];
+  crypto_pk_t *pk = pk_generate(0);
+
+  (void)arg;
+
+  init_mock_ed_keys(pk);
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+  dirserv_add_own_fingerprint(pk, get_master_identity_key());
+
+  /* Check if we have a RSA key. */
+  crypto_pk_get_digest(pk, digest);
+  tt_assert(digestmap_get(list->status_by_digest, digest));
+
+  /* Check if we have a ed25519 key. */
+  tt_assert(digest256map_get(list->status_by_digest256,
+                             get_master_identity_key()->pubkey));
+
+  RESET_FP_LIST(list);
+
+ done:
+  dirserv_free_fingerprint_list();
+  crypto_pk_free(pk);
+}
+
+#ifndef COCCI
 #define DIR_LEGACY(name)                             \
   { #name, test_dir_ ## name , TT_FORK, NULL, NULL }
 
@@ -6353,10 +7438,26 @@ test_dir_format_versions_list(void *arg)
 /* where arg is a string constant */
 #define DIR_ARG(name,flags,arg)                      \
   { #name "_" arg, test_dir_##name, (flags), &passthrough_setup, (void*) arg }
+#endif /* !defined(COCCI) */
 
 struct testcase_t dir_tests[] = {
   DIR_LEGACY(nicknames),
-  DIR_LEGACY(formats),
+  /* extrainfo without any stats */
+  DIR_ARG(formats_rsa, TT_FORK, ""),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, ""),
+  /* on a bridge */
+  DIR_ARG(formats_rsa, TT_FORK, "b"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "b"),
+  /* extrainfo with basic stats */
+  DIR_ARG(formats_rsa, TT_FORK, "e"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "e"),
+  DIR_ARG(formats_rsa, TT_FORK, "be"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "be"),
+  /* extrainfo with all stats */
+  DIR_ARG(formats_rsa, TT_FORK, "es"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "es"),
+  DIR_ARG(formats_rsa, TT_FORK, "bes"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "bes"),
   DIR(routerinfo_parsing, 0),
   DIR(extrainfo_parsing, 0),
   DIR(parse_router_list, TT_FORK),
@@ -6370,6 +7471,7 @@ struct testcase_t dir_tests[] = {
   DIR_LEGACY(measured_bw_kb_line_is_after_headers),
   DIR_LEGACY(measured_bw_kb_cache),
   DIR_LEGACY(dirserv_read_measured_bandwidths),
+  DIR(bwauth_bw_file_digest256, 0),
   DIR_LEGACY(param_voting),
   DIR(param_voting_lookup, 0),
   DIR_LEGACY(v3_networkstatus),
@@ -6378,7 +7480,7 @@ struct testcase_t dir_tests[] = {
   DIR_LEGACY(clip_unmeasured_bw_kb),
   DIR_LEGACY(clip_unmeasured_bw_kb_alt),
   DIR(fmt_control_ns, 0),
-  DIR(dirserv_set_routerstatus_testing, 0),
+  DIR(dirserv_set_routerstatus_testing, TT_FORK),
   DIR(http_handling, 0),
   DIR(purpose_needs_anonymity_returns_true_for_bridges, 0),
   DIR(purpose_needs_anonymity_returns_false_for_own_bridge_desc, 0),
@@ -6410,9 +7512,14 @@ struct testcase_t dir_tests[] = {
   DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"),
   DIR_ARG(find_dl_min_delay, TT_FORK, "car"),
   DIR(assumed_flags, 0),
+  DIR(matching_flags, 0),
   DIR(networkstatus_compute_bw_weights_v10, 0),
   DIR(platform_str, 0),
-  DIR(networkstatus_consensus_has_ipv6, TT_FORK),
   DIR(format_versions_list, TT_FORK),
+  DIR(add_fingerprint, TT_FORK),
+  DIR(dirserv_load_fingerprint_file, TT_FORK),
+  DIR(dirserv_router_get_status, TT_FORK),
+  DIR(dirserv_would_reject_router, TT_FORK),
+  DIR(dirserv_add_own_fingerprint, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c
index 3723d6c31b..f2b4e8724b 100644
--- a/src/test/test_dir_common.c
+++ b/src/test/test_dir_common.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -13,7 +13,7 @@
 #include "feature/dirparse/authcert_parse.h"
 #include "feature/dirparse/ns_parse.h"
 #include "test/test_dir_common.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -42,14 +42,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1,
 {
   /* Parse certificates and keys. */
   authority_cert_t *cert;
-  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                          strlen(AUTHORITY_CERT_1),
+                                          NULL);
   tt_assert(cert);
   tt_assert(cert->identity_key);
   *cert1 = cert;
   tt_assert(*cert1);
-  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                            strlen(AUTHORITY_CERT_2),
+                                            NULL);
   tt_assert(*cert2);
-  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                            strlen(AUTHORITY_CERT_3),
+                                            NULL);
   tt_assert(*cert3);
   *sign_skey_1 = crypto_pk_new();
   *sign_skey_2 = crypto_pk_new();
@@ -266,7 +272,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out,
   /* dump the vote and try to parse it. */
   v_text = format_networkstatus_vote(sign_skey, vote);
   tt_assert(v_text);
-  *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE);
+  *vote_out = networkstatus_parse_vote_from_string(v_text,
+                                                   strlen(v_text),
+                                                   NULL, NS_TYPE_VOTE);
 
  done:
   if (v_text)
@@ -424,4 +432,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert,
 
   return 0;
 }
-
diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h
index d6c5241b14..d37496465c 100644
--- a/src/test/test_dir_common.h
+++ b/src/test/test_dir_common.h
@@ -1,8 +1,11 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_DIR_COMMON_H
+#define TOR_TEST_DIR_COMMON_H
+
 #include "core/or/or.h"
 #include "feature/nodelist/networkstatus.h"
 
@@ -49,3 +52,4 @@ int dir_common_construct_vote_3(networkstatus_t **vote,
                         networkstatus_t **vote_out, int *n_vrs, time_t now,
                         int clear_rl);
 
+#endif /* !defined(TOR_TEST_DIR_COMMON_H) */
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index 90691fff94..f446bbb5eb 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define RENDCOMMON_PRIVATE
@@ -20,6 +20,7 @@
 #include "lib/compress/compress.h"
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendcache.h"
+#include "feature/relay/relay_config.h"
 #include "feature/relay/router.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
@@ -37,7 +38,7 @@
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/dirvote.h"
 #include "test/log_test_helpers.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/dirclient/dir_server_st.h"
@@ -54,17 +55,15 @@
 #endif /* defined(_WIN32) */
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
 #include "vote_descriptors.inc"
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-ENABLE_GCC_WARNING(overlength-strings)
+ENABLE_GCC_WARNING("-Woverlength-strings")
 #endif
 
-#define NS_MODULE dir_handle_get
-
 #define NOT_FOUND "HTTP/1.0 404 Not found\r\n\r\n"
 #define BAD_REQUEST "HTTP/1.0 400 Bad request\r\n\r\n"
 #define SERVER_BUSY "HTTP/1.0 503 Directory busy, try again later\r\n\r\n"
@@ -72,6 +71,8 @@ ENABLE_GCC_WARNING(overlength-strings)
 #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \
   "Consensus not signed by sufficient number of requested authorities\r\n\r\n"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
 static dir_connection_t *
 new_dir_conn(void)
 {
@@ -116,7 +117,7 @@ test_dir_handle_get_v1_command_not_found(void *data)
   conn = new_dir_conn();
 
   // no frontpage configured
-  tt_ptr_op(get_dirportfrontpage(), OP_EQ, NULL);
+  tt_ptr_op(relay_get_dirportfrontpage(), OP_EQ, NULL);
 
   /* V1 path */
   tt_int_op(directory_handle_command_get(conn, GET("/tor/"), NULL, 0),
@@ -150,9 +151,9 @@ test_dir_handle_get_v1_command(void *data)
   (void) data;
 
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
-  MOCK(get_dirportfrontpage, mock_get_dirportfrontpage);
+  MOCK(relay_get_dirportfrontpage, mock_get_dirportfrontpage);
 
-  exp_body = get_dirportfrontpage();
+  exp_body = relay_get_dirportfrontpage();
   body_len = strlen(exp_body);
 
   conn = new_dir_conn();
@@ -175,7 +176,7 @@ test_dir_handle_get_v1_command(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    UNMOCK(get_dirportfrontpage);
+    UNMOCK(relay_get_dirportfrontpage);
     connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
@@ -255,7 +256,7 @@ test_dir_handle_get_rendezvous2_not_found_if_not_encrypted(void *data)
   conn = new_dir_conn();
 
   // connection is not encrypted
-  tt_assert(!connection_dir_is_encrypted(conn))
+  tt_assert(!connection_dir_is_encrypted(conn));
 
   tt_int_op(directory_handle_command_get(conn, RENDEZVOUS2_GET(), NULL, 0),
             OP_EQ, 0);
@@ -361,12 +362,13 @@ test_dir_handle_get_rendezvous2_not_found(void *data)
     rend_cache_free_all();
 }
 
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t * dhg_tests_router_get_my_routerinfo(void);
+ATTR_UNUSED static int dhg_tests_router_get_my_routerinfo_called = 0;
 
 static routerinfo_t *mock_routerinfo;
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+dhg_tests_router_get_my_routerinfo(void)
 {
   if (!mock_routerinfo) {
     mock_routerinfo = tor_malloc_zero(sizeof(routerinfo_t));
@@ -391,7 +393,8 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
   (void) data;
 
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -434,7 +437,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
 
     connection_free_minimal(TO_CONN(conn));
     tor_free(header);
@@ -477,8 +480,7 @@ static or_options_t *mock_options = NULL;
 static void
 init_mock_options(void)
 {
-  mock_options = tor_malloc(sizeof(or_options_t));
-  memset(mock_options, 0, sizeof(or_options_t));
+  mock_options = options_new();
   mock_options->TestingTorNetwork = 1;
   mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp"));
   mock_options->CacheDirectory = tor_strdup(mock_options->DataDirectory);
@@ -767,7 +769,8 @@ test_dir_handle_get_server_descriptors_all(void* data)
   helper_setup_fake_routerlist();
 
   //TODO: change to router_get_my_extrainfo when testing "extra" path
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
   // We are one of the routers
@@ -809,7 +812,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     connection_free_minimal(TO_CONN(conn));
     tor_free(header);
@@ -866,7 +869,8 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   crypto_pk_t *identity_pkey = pk_generate(0);
   (void) data;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
   /* init mock */
@@ -911,7 +915,7 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
@@ -931,7 +935,8 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   crypto_pk_t *identity_pkey = pk_generate(0);
   (void) data;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
   /* init mock */
@@ -983,7 +988,7 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
@@ -1275,7 +1280,9 @@ test_dir_handle_get_server_keys_authority(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
@@ -1425,7 +1432,9 @@ test_dir_handle_get_server_keys_sk(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
@@ -1783,13 +1792,14 @@ test_dir_handle_get_status_vote_current_consensus_too_old(void *data)
     or_options_free(mock_options); mock_options = NULL;
 }
 
-NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
+static int dhg_tests_geoip_get_country_by_addr(const tor_addr_t *addr);
+ATTR_UNUSED static int dhg_tests_geoip_get_country_by_addr_called = 0;
 
 int
-NS(geoip_get_country_by_addr)(const tor_addr_t *addr)
+dhg_tests_geoip_get_country_by_addr(const tor_addr_t *addr)
 {
   (void)addr;
-  CALLED(geoip_get_country_by_addr)++;
+  dhg_tests_geoip_get_country_by_addr_called++;
   return 1;
 }
 
@@ -1853,7 +1863,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data)
   dirserv_free_all();
   clear_geoip_db();
 
-  NS_MOCK(geoip_get_country_by_addr);
+  MOCK(geoip_get_country_by_addr,
+       dhg_tests_geoip_get_country_by_addr);
   MOCK(get_options, mock_get_options);
 
   init_mock_options();
@@ -1890,7 +1901,7 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data)
   tt_str_op("ab=8", OP_EQ, hist);
 
   done:
-    NS_UNMOCK(geoip_get_country_by_addr);
+    UNMOCK(geoip_get_country_by_addr);
     UNMOCK(get_options);
     tor_free(header);
     tor_free(comp_body);
@@ -2069,12 +2080,12 @@ test_dir_handle_get_status_vote_d(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  voting_schedule_recalculate_timing(mock_options, now);
+  dirauth_sched_recalculate_timing(mock_options, now);
 
   const char *msg_out = NULL;
   int status_out = 0;
-  struct pending_vote_t *pv = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
-                                               &status_out);
+  struct pending_vote_t *pv = dirvote_add_vote(VOTE_BODY_V3, 0,
+                                               &msg_out, &status_out);
   tt_assert(pv);
 
   status_vote_current_d_test(&header, &body, &body_used);
@@ -2217,11 +2228,36 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data)
     tor_free(header);
 }
 
-NS_DECL(const char*,
-dirvote_get_pending_consensus, (consensus_flavor_t flav));
+static void
+test_dir_handle_get_status_vote_next_bandwidth_not_found(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL;
+  (void) data;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwdith"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(NOT_FOUND, OP_EQ, header);
+
+  done:
+    UNMOCK(connection_write_to_buf_impl_);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+}
+
+static const char* dhg_tests_dirvote_get_pending_consensus(
+                                           consensus_flavor_t flav);
 
 const char*
-NS(dirvote_get_pending_consensus)(consensus_flavor_t flav)
+dhg_tests_dirvote_get_pending_consensus(consensus_flavor_t flav)
 {
   (void)flav;
   return "pending consensus";
@@ -2234,7 +2270,8 @@ test_dir_handle_get_status_vote_next_consensus(void* data)
   size_t body_used = 0;
   (void) data;
 
-  NS_MOCK(dirvote_get_pending_consensus);
+  MOCK(dirvote_get_pending_consensus,
+       dhg_tests_dirvote_get_pending_consensus);
 
   status_vote_next_consensus_test(&header, &body, &body_used);
   tt_assert(header);
@@ -2247,7 +2284,7 @@ test_dir_handle_get_status_vote_next_consensus(void* data)
   tt_str_op("pending consensus", OP_EQ, body);
 
   done:
-    NS_UNMOCK(dirvote_get_pending_consensus);
+    UNMOCK(dirvote_get_pending_consensus);
     tor_free(header);
     tor_free(body);
 }
@@ -2260,7 +2297,8 @@ test_dir_handle_get_status_vote_next_consensus_busy(void* data)
   (void) data;
 
   MOCK(get_options, mock_get_options);
-  NS_MOCK(dirvote_get_pending_consensus);
+  MOCK(dirvote_get_pending_consensus,
+       dhg_tests_dirvote_get_pending_consensus);
 
   //Make it busy
   init_mock_options();
@@ -2272,7 +2310,7 @@ test_dir_handle_get_status_vote_next_consensus_busy(void* data)
   tt_str_op(SERVER_BUSY, OP_EQ, header);
 
   done:
-    NS_UNMOCK(dirvote_get_pending_consensus);
+    UNMOCK(dirvote_get_pending_consensus);
     UNMOCK(get_options);
     tor_free(header);
     tor_free(body);
@@ -2316,11 +2354,10 @@ test_dir_handle_get_status_vote_next_consensus_signatures_not_found(void* data)
     tor_free(body);
 }
 
-NS_DECL(const char*,
-dirvote_get_pending_detached_signatures, (void));
+static const char* dhg_tests_dirvote_get_pending_detached_signatures(void);
 
 const char*
-NS(dirvote_get_pending_detached_signatures)(void)
+dhg_tests_dirvote_get_pending_detached_signatures(void)
 {
   return "pending detached sigs";
 }
@@ -2332,7 +2369,8 @@ test_dir_handle_get_status_vote_next_consensus_signatures(void* data)
   size_t body_used = 0;
   (void) data;
 
-  NS_MOCK(dirvote_get_pending_detached_signatures);
+  MOCK(dirvote_get_pending_detached_signatures,
+       dhg_tests_dirvote_get_pending_detached_signatures);
 
   status_vote_next_consensus_signatures_test(&header, &body, &body_used);
   tt_assert(header);
@@ -2345,7 +2383,7 @@ test_dir_handle_get_status_vote_next_consensus_signatures(void* data)
   tt_str_op("pending detached sigs", OP_EQ, body);
 
   done:
-    NS_UNMOCK(dirvote_get_pending_detached_signatures);
+    UNMOCK(dirvote_get_pending_detached_signatures);
     tor_free(header);
     tor_free(body);
 }
@@ -2357,7 +2395,8 @@ test_dir_handle_get_status_vote_next_consensus_signatures_busy(void* data)
   size_t body_used;
   (void) data;
 
-  NS_MOCK(dirvote_get_pending_detached_signatures);
+  MOCK(dirvote_get_pending_detached_signatures,
+       dhg_tests_dirvote_get_pending_detached_signatures);
   MOCK(get_options, mock_get_options);
 
   //Make it busy
@@ -2371,7 +2410,7 @@ test_dir_handle_get_status_vote_next_consensus_signatures_busy(void* data)
 
   done:
     UNMOCK(get_options);
-    NS_UNMOCK(dirvote_get_pending_detached_signatures);
+    UNMOCK(dirvote_get_pending_detached_signatures);
     tor_free(header);
     tor_free(body);
     or_options_free(mock_options); mock_options = NULL;
@@ -2393,7 +2432,9 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2416,10 +2457,10 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  voting_schedule_recalculate_timing(mock_options, now);
+  dirauth_sched_recalculate_timing(mock_options, now);
 
-  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
-                                                 &status_out);
+  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, 0,
+                                                 &msg_out, &status_out);
   tt_assert(vote);
 
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
@@ -2455,6 +2496,85 @@ test_dir_handle_get_status_vote_next_authority(void* data)
 }
 
 static void
+test_dir_handle_get_status_vote_next_bandwidth(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL, *body = NULL;
+  size_t body_used = 0;
+  (void) data;
+
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  init_mock_options();
+  MOCK(get_options, mock_get_options);
+  mock_options->V3BandwidthsFile = tor_strdup(
+    get_fname_rnd("V3BandwidthsFile")
+  );
+
+  write_str_to_file(mock_options->V3BandwidthsFile, content, 0);
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Type: text/plain\r\n"));
+  tt_assert(strstr(header, "Content-Encoding: identity\r\n"));
+  tt_assert(strstr(header, "Content-Length: 167\r\n"));
+
+  /* Check cache lifetime */
+  char expbuf[RFC1123_TIME_LEN+1];
+  time_t now = approx_time();
+  /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */
+  format_rfc1123_time(expbuf, (time_t)(now + 30*60));
+  char *expires = NULL;
+  /* Change to 'Cache-control: max-age=%d' if using http/1.1. */
+  tor_asprintf(&expires, "Expires: %s\r\n", expbuf);
+  tt_assert(strstr(header, expires));
+
+  tt_int_op(body_used, OP_EQ, strlen(body));
+  tt_str_op(content, OP_EQ, body);
+
+  tor_free(header);
+  tor_free(body);
+
+  /* Request the file using compression, the result should be the same. */
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth.z"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Encoding: deflate\r\n"));
+
+  /* Since using connection_write_to_buf_mock instead of mocking
+   * connection_buf_add_compress, the content is not actually compressed.
+   * If it would, the size and content would be different than the original.
+  */
+
+ done:
+  UNMOCK(get_options);
+  UNMOCK(connection_write_to_buf_impl_);
+  connection_free_minimal(TO_CONN(conn));
+  tor_free(header);
+  tor_free(body);
+  tor_free(expires);
+  or_options_free(mock_options);
+}
+
+static void
 test_dir_handle_get_status_vote_current_authority(void* data)
 {
   dir_connection_t *conn = NULL;
@@ -2471,7 +2591,9 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2495,10 +2617,10 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455;
-  voting_schedule_recalculate_timing(mock_options, now-1);
+  dirauth_sched_recalculate_timing(mock_options, now-1);
 
-  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
-                                                 &status_out);
+  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, 0,
+                                                 &msg_out, &status_out);
   tt_assert(vote);
 
   // move the pending vote to previous vote
@@ -2536,6 +2658,183 @@ test_dir_handle_get_status_vote_current_authority(void* data)
     dirvote_free_all();
 }
 
+/* Test that a late vote is rejected, but an on-time vote is accepted. */
+static void
+test_dir_handle_get_status_vote_too_late(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL, *body = NULL;
+  const char *msg_out = NULL;
+  int status_out = 0;
+  size_t body_used = 0;
+  const char digest[DIGEST_LEN] = "";
+
+  dir_server_t *ds = NULL;
+  const char* mode = (const char *)data;
+
+  clear_dir_servers();
+  routerlist_free_all();
+  dirvote_free_all();
+
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
+
+  /* create a trusted ds */
+  ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
+                              NULL, V3_DIRINFO, 1.0);
+  tt_assert(ds);
+  dir_server_add(ds);
+
+  /* ds v3_identity_digest is the certificate's identity_key */
+  base16_decode(ds->v3_identity_digest, DIGEST_LEN,
+                TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN);
+
+  tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE,
+    TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL));
+
+  init_mock_options();
+  mock_options->AuthoritativeDir = 1;
+  mock_options->V3AuthoritativeDir = 1;
+
+  int base_delay = 0;
+  int vote_interval = 0;
+  int start_offset = 0;
+
+  tt_assert(mode);
+  /* Set the required timings, see below for details */
+  if (strcmp(mode, "min") == 0) {
+    /* The minimum valid test network timing */
+    base_delay = 2;
+    vote_interval = 10;
+    start_offset = vote_interval - 5;
+  } else if (strcmp(mode, "chutney") == 0) {
+    /* The test network timing used by chutney */
+    base_delay = 4;
+    vote_interval = 20;
+    start_offset = vote_interval - 5;
+  } else if (strcmp(mode, "half-public") == 0) {
+    /* The short consensus failure timing used in the public network */
+    base_delay = 5*60;
+    vote_interval = 30*60;
+    start_offset = vote_interval - 9*60 - 5;
+  } else if (strcmp(mode, "public") == 0) {
+    /* The standard timing used in the public network */
+    base_delay = 5*60;
+    vote_interval = 60*60;
+    start_offset = vote_interval - 9*60 - 5;
+  }
+
+  tt_assert(base_delay > 0);
+  tt_assert(vote_interval > 0);
+  tt_assert(start_offset > 0);
+
+  /* Skew the time to fit the fixed time in the vote */
+  mock_options->TestingV3AuthVotingStartOffset = start_offset;
+  /* Calculate the rest of the timings */
+  mock_options->TestingV3AuthInitialVotingInterval = vote_interval;
+  mock_options->TestingV3AuthInitialVoteDelay = base_delay;
+  mock_options->TestingV3AuthInitialDistDelay = base_delay;
+
+  time_t now = 1441223455;
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+  const time_t voting_starts = voting_schedule.voting_starts;
+  const time_t fetch_missing = voting_schedule.fetch_missing_votes;
+
+  struct pending_vote_t *vote = NULL;
+
+  /* Next voting interval */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          fetch_missing + vote_interval,
+                          &msg_out, &status_out);
+  tt_assert(!vote);
+  tt_int_op(status_out, OP_EQ, 400);
+  tt_str_op(msg_out, OP_EQ,
+            "Posted vote received too late, would be dangerous to count it");
+
+  /* Just after fetch missing */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          fetch_missing + 1,
+                          &msg_out, &status_out);
+  tt_assert(!vote);
+  tt_int_op(status_out, OP_EQ, 400);
+  tt_str_op(msg_out, OP_EQ,
+            "Posted vote received too late, would be dangerous to count it");
+
+  /* On fetch missing */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          fetch_missing,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+  /* And reset the timing */
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+
+  /* Between voting starts and fetch missing */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          voting_starts + 1,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+  /* And reset the timing */
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+
+  /* On voting starts */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          voting_starts,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+  /* And reset the timing */
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+
+  /* Just before voting starts */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          voting_starts - 1,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+
+  MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/authority"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(VOTE_BODY_V3)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Type: text/plain\r\n"));
+  tt_assert(strstr(header, "Content-Encoding: identity\r\n"));
+  tt_assert(strstr(header, "Content-Length: 4135\r\n"));
+
+  tt_str_op(VOTE_BODY_V3, OP_EQ, body);
+
+  done:
+    UNMOCK(connection_write_to_buf_impl_);
+    UNMOCK(get_my_v3_authority_cert);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+    tor_free(body);
+    authority_cert_free(mock_cert); mock_cert = NULL;
+    or_options_free(mock_options); mock_options = NULL;
+
+    clear_dir_servers();
+    routerlist_free_all();
+    dirvote_free_all();
+}
+
 static void
 test_dir_handle_get_parse_accept_encoding(void *arg)
 {
@@ -2586,6 +2885,16 @@ test_dir_handle_get_parse_accept_encoding(void *arg)
 #define DIR_HANDLE_CMD(name,flags) \
   { #name, test_dir_handle_get_##name, (flags), NULL, NULL }
 
+#ifdef COCCI
+/* Coccinelle doesn't like the stringification in this macro */
+#define DIR_HANDLE_CMD_ARG(name,flags,arg) \
+  DIR_HANDLE_CMD(name,flags)
+#else
+#define DIR_HANDLE_CMD_ARG(name,flags,arg) \
+  { #name "/" arg, test_dir_handle_get_##name, (flags), \
+    &passthrough_setup, (void *)(arg) }
+#endif /* defined(COCCI) */
+
 struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(not_found, 0),
   DIR_HANDLE_CMD(bad_request, 0),
@@ -2625,8 +2934,14 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_next_not_found, 0),
   DIR_HANDLE_CMD(status_vote_current_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_current_authority, 0),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "min"),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "chutney"),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "half-public"),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "public"),
   DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_authority, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth_not_found, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth, 0),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK),
diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c
new file mode 100644
index 0000000000..77f33e4b15
--- /dev/null
+++ b/src/test/test_dispatch.c
@@ -0,0 +1,278 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static dispatch_t *dispatcher_in_use=NULL;
+
+static void
+test_dispatch_max_in_u16_sl(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  uint16_t nums[] = { 10, 20, 30 };
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[1]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[0]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[2]);
+  tt_int_op(30, OP_EQ, max_in_u16_sl(sl, -1));
+
+ done:
+  smartlist_free(sl);
+}
+
+/* Construct an empty dispatch_t. */
+static void
+test_dispatch_empty(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+
+  cfg = dcfg_new();
+  d = dispatch_new(cfg);
+  tt_assert(d);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+static int total_recv1_simple = 0;
+static int total_recv2_simple = 0;
+
+static void
+simple_recv1(const msg_t *m)
+{
+  total_recv1_simple += m->aux_data__.u64;
+}
+
+static char *recv2_received = NULL;
+
+static void
+simple_recv2(const msg_t *m)
+{
+  tor_free(recv2_received);
+  recv2_received = dispatch_fmt_msg_data(dispatcher_in_use, m);
+
+  total_recv2_simple += m->aux_data__.u64*10;
+}
+
+/* Construct a dispatch_t with two messages, make sure that they both get
+ * delivered. */
+static void
+test_dispatch_simple(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  r += dcfg_add_recv(cfg,0,1,simple_recv1);
+  r += dcfg_msg_set_type(cfg,1,0);
+  r += dcfg_msg_set_chan(cfg,1,0);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2); /* second copy */
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = {.u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 7);
+  tt_int_op(total_recv2_simple, OP_EQ, 0);
+
+  total_recv1_simple = 0;
+  r = dispatch_send(d, 99, 0, 1, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+  tt_int_op(total_recv2_simple, OP_EQ, 140);
+
+  tt_str_op(recv2_received, OP_EQ, "<>"); // no format function was set.
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(recv2_received);
+}
+
+/* Construct a dispatch_t with a message and no reciever; make sure that it
+ * gets dropped properly. */
+static void
+test_dispatch_no_recipient(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = { .u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+struct coord_t { int x; int y; };
+static void
+free_coord(msg_aux_data_t d)
+{
+  tor_free(d.ptr);
+}
+static char *
+fmt_coord(msg_aux_data_t d)
+{
+  char *v;
+  struct coord_t *c = d.ptr;
+  tor_asprintf(&v, "[%d, %d]", c->x, c->y);
+  return v;
+}
+static dispatch_typefns_t coord_fns = {
+  .fmt_fn = fmt_coord,
+  .free_fn = free_coord,
+};
+static void
+alert_run_immediate(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)arg;
+  dispatch_flush(d, ch, INT_MAX);
+}
+
+static char *received_data=NULL;
+
+static void
+recv_typed_data(const msg_t *m)
+{
+  tor_free(received_data);
+  received_data = dispatch_fmt_msg_data(dispatcher_in_use, m);
+}
+
+static void
+test_dispatch_with_types(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,5,3);
+  r += dcfg_msg_set_chan(cfg,5,2);
+  r += dcfg_add_recv(cfg,5,0,recv_typed_data);
+  r += dcfg_type_set_fns(cfg,3,&coord_fns);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  /* Make this message get run immediately. */
+  r = dispatch_set_alert_fn(d, 2, alert_run_immediate, NULL);
+  tt_int_op(r, OP_EQ, 0);
+
+  struct coord_t *xy = tor_malloc(sizeof(*xy));
+  xy->x = 13;
+  xy->y = 37;
+  msg_aux_data_t data = {.ptr = xy};
+  r = dispatch_send(d, 99/*sender*/, 2/*channel*/, 5/*msg*/, 3/*type*/, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_str_op(received_data, OP_EQ, "[13, 37]");
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(received_data);
+  dispatcher_in_use = NULL;
+}
+
+static void
+test_dispatch_bad_type_setup(void *arg)
+{
+  (void)arg;
+  static dispatch_typefns_t fns;
+  dispatch_cfg_t *cfg = dcfg_new();
+
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &coord_fns));
+
+  fns = coord_fns;
+  fns.fmt_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  fns.free_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+ done:
+  dcfg_free(cfg);
+}
+
+#define T(name)                                                 \
+  { #name, test_dispatch_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t dispatch_tests[] = {
+  T(max_in_u16_sl),
+  T(empty),
+  T(simple),
+  T(no_recipient),
+  T(with_types),
+  T(bad_type_setup),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_dns.c b/src/test/test_dns.c
index 41a56f65d8..299321ab64 100644
--- a/src/test/test_dns.c
+++ b/src/test/test_dns.c
@@ -1,6 +1,7 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#include "orconfig.h"
 #include "core/or/or.h"
 #include "test/test.h"
 
@@ -13,29 +14,81 @@
 
 #include "core/or/edge_connection_st.h"
 #include "core/or/or_circuit_st.h"
+#include "app/config/or_options_st.h"
+#include "app/config/config.h"
 
-#define NS_MODULE dns
+#include <event2/event.h>
+#include <event2/dns.h>
 
-#define NS_SUBMODULE clip_ttl
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+
+static or_options_t options = {
+  .ORPort_set = 1,
+};
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &options;
+}
 
 static void
-NS(test_main)(void *arg)
+test_dns_configure_ns_fallback(void *arg)
 {
   (void)arg;
+  tor_addr_t *nameserver_addr = NULL;
 
-  uint32_t ttl_mid = MIN_DNS_TTL_AT_EXIT / 2 + MAX_DNS_TTL_AT_EXIT / 2;
+  MOCK(get_options, mock_get_options);
 
-  tt_int_op(dns_clip_ttl(MIN_DNS_TTL_AT_EXIT - 1),OP_EQ,MIN_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(ttl_mid),OP_EQ,MAX_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(MAX_DNS_TTL_AT_EXIT + 1),OP_EQ,MAX_DNS_TTL_AT_EXIT);
+  options.ServerDNSResolvConfFile = (char *)"no_such_file!!!";
 
-  done:
+  dns_init(); // calls configure_nameservers()
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+
+#ifndef _WIN32
+  tor_free(nameserver_addr);
+
+  options.ServerDNSResolvConfFile = (char *)"/dev/null";
+
+  dns_init();
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+#endif /* !defined(_WIN32) */
+
+  UNMOCK(get_options);
+
+ done:
+  tor_free(nameserver_addr);
   return;
 }
 
-#undef NS_SUBMODULE
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
+static void
+test_dns_clip_ttl(void *arg)
+{
+  (void)arg;
 
-#define NS_SUBMODULE resolve
+  uint32_t ttl_mid = MIN_DNS_TTL / 2 + MAX_DNS_TTL / 2;
+
+  tt_int_op(clip_dns_ttl(MIN_DNS_TTL - 1),OP_EQ,MIN_DNS_TTL);
+  tt_int_op(clip_dns_ttl(ttl_mid),OP_EQ,MAX_DNS_TTL);
+  tt_int_op(clip_dns_ttl(MAX_DNS_TTL + 1),OP_EQ,MAX_DNS_TTL);
+
+  done:
+  return;
+}
 
 static int resolve_retval = 0;
 static int resolve_made_conn_pending = 0;
@@ -44,10 +97,11 @@ static cached_resolve_t *cache_entry_mock = NULL;
 
 static int n_fake_impl = 0;
 
-NS_DECL(int, dns_resolve_impl, (edge_connection_t *exitconn, int is_resolve,
-                                or_circuit_t *oncirc, char **hostname_out,
-                                int *made_connection_pending_out,
-                                cached_resolve_t **resolve_out));
+static int dns_resolve_dns_resolve_impl(edge_connection_t *exitconn,
+                     int is_resolve, or_circuit_t *oncirc,
+                     char **hostname_out, int *made_connection_pending_out,
+                     cached_resolve_t **resolve_out);
+ATTR_UNUSED static int dns_resolve_dns_resolve_impl_called = 0;
 
 /** This will be our configurable substitute for <b>dns_resolve_impl</b> in
  * dns.c. It will return <b>resolve_retval</b>,
@@ -58,7 +112,7 @@ NS_DECL(int, dns_resolve_impl, (edge_connection_t *exitconn, int is_resolve,
  * 1.
  */
 static int
-NS(dns_resolve_impl)(edge_connection_t *exitconn, int is_resolve,
+dns_resolve_dns_resolve_impl(edge_connection_t *exitconn, int is_resolve,
                      or_circuit_t *oncirc, char **hostname_out,
                      int *made_connection_pending_out,
                      cached_resolve_t **resolve_out)
@@ -88,7 +142,7 @@ static uint8_t last_answer_type = 0;
 static cached_resolve_t *last_resolved;
 
 static void
-NS(send_resolved_cell)(edge_connection_t *conn, uint8_t answer_type,
+dns_resolve_send_resolved_cell(edge_connection_t *conn, uint8_t answer_type,
                        const cached_resolve_t *resolved)
 {
   conn_for_resolved_cell = conn;
@@ -104,7 +158,7 @@ static int n_send_resolved_hostname_cell_replacement = 0;
 static char *last_resolved_hostname = NULL;
 
 static void
-NS(send_resolved_hostname_cell)(edge_connection_t *conn,
+dns_resolve_send_resolved_hostname_cell(edge_connection_t *conn,
                                 const char *hostname)
 {
   conn_for_resolved_cell = conn;
@@ -118,7 +172,7 @@ NS(send_resolved_hostname_cell)(edge_connection_t *conn,
 static int n_dns_cancel_pending_resolve_replacement = 0;
 
 static void
-NS(dns_cancel_pending_resolve)(const char *address)
+dns_resolve_dns_cancel_pending_resolve(const char *address)
 {
   (void) address;
   n_dns_cancel_pending_resolve_replacement++;
@@ -128,7 +182,7 @@ static int n_connection_free = 0;
 static connection_t *last_freed_conn = NULL;
 
 static void
-NS(connection_free_)(connection_t *conn)
+dns_resolve_connection_free_(connection_t *conn)
 {
    n_connection_free++;
 
@@ -136,7 +190,7 @@ NS(connection_free_)(connection_t *conn)
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_resolve(void *arg)
 {
   (void) arg;
   int retval;
@@ -155,9 +209,12 @@ NS(test_main)(void *arg)
   memset(exitconn,0,sizeof(edge_connection_t));
   memset(nextconn,0,sizeof(edge_connection_t));
 
-  NS_MOCK(dns_resolve_impl);
-  NS_MOCK(send_resolved_cell);
-  NS_MOCK(send_resolved_hostname_cell);
+  MOCK(dns_resolve_impl,
+       dns_resolve_dns_resolve_impl);
+  MOCK(send_resolved_cell,
+       dns_resolve_send_resolved_cell);
+  MOCK(send_resolved_hostname_cell,
+       dns_resolve_send_resolved_hostname_cell);
 
   /*
    * CASE 1: dns_resolve_impl returns 1 and sets a hostname. purpose is
@@ -270,8 +327,10 @@ NS(test_main)(void *arg)
    * on exitconn with type being RESOLVED_TYPE_ERROR.
    */
 
-  NS_MOCK(dns_cancel_pending_resolve);
-  NS_MOCK(connection_free_);
+  MOCK(dns_cancel_pending_resolve,
+       dns_resolve_dns_cancel_pending_resolve);
+  MOCK(connection_free_,
+       dns_resolve_connection_free_);
 
   exitconn->on_circuit = &(on_circuit->base_);
   exitconn->base_.purpose = EXIT_PURPOSE_RESOLVE;
@@ -294,11 +353,11 @@ NS(test_main)(void *arg)
   tt_assert(last_freed_conn == TO_CONN(exitconn));
 
   done:
-  NS_UNMOCK(dns_resolve_impl);
-  NS_UNMOCK(send_resolved_cell);
-  NS_UNMOCK(send_resolved_hostname_cell);
-  NS_UNMOCK(dns_cancel_pending_resolve);
-  NS_UNMOCK(connection_free_);
+  UNMOCK(dns_resolve_impl);
+  UNMOCK(send_resolved_cell);
+  UNMOCK(send_resolved_hostname_cell);
+  UNMOCK(dns_cancel_pending_resolve);
+  UNMOCK(connection_free_);
   tor_free(on_circuit);
   tor_free(exitconn);
   tor_free(nextconn);
@@ -308,8 +367,6 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
 /** Create an <b>edge_connection_t</b> instance that is considered a
  * valid exit connection by asserts in dns_resolve_impl.
  */
@@ -326,8 +383,6 @@ create_valid_exitconn(void)
   return exitconn;
 }
 
-#define NS_SUBMODULE ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve)
-
 /*
  * Given that <b>exitconn->base_.address</b> is IP address string, we
  * want dns_resolve_impl() to parse it and store in
@@ -336,7 +391,7 @@ create_valid_exitconn(void)
  */
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_addr_is_ip(void *arg)
 {
   int retval;
   int made_pending;
@@ -369,21 +424,17 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, non_exit)
-
 /** Given that Tor instance is not configured as an exit node, we want
  * dns_resolve_impl() to fail with return value -1.
  */
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_non_exit_router_my_exit_policy_is_reject_star(void)
 {
   return 1;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_non_exit(void *arg)
 {
   int retval;
   int made_pending;
@@ -395,7 +446,8 @@ NS(test_main)(void *arg)
 
   TO_CONN(exitconn)->address = tor_strdup("torproject.org");
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_non_exit_router_my_exit_policy_is_reject_star);
 
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
@@ -406,27 +458,23 @@ NS(test_main)(void *arg)
   tor_free(TO_CONN(exitconn)->address);
   tor_free(exitconn);
   tor_free(on_circ);
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, addr_is_invalid_dest)
-
 /** Given that address is not a valid destination (as judged by
  * address_is_invalid_destination() function), we want dns_resolve_impl()
  * function to fail with return value -1.
  */
 
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_addr_is_invalid_dest_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_addr_is_invalid_dest(void *arg)
 {
   int retval;
   int made_pending;
@@ -436,7 +484,8 @@ NS(test_main)(void *arg)
 
   (void)arg;
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_addr_is_invalid_dest_router_my_exit_policy_is_reject_star);
 
   TO_CONN(exitconn)->address = tor_strdup("invalid#@!.org");
 
@@ -446,29 +495,25 @@ NS(test_main)(void *arg)
   tt_int_op(retval,OP_EQ,-1);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(exitconn);
   tor_free(on_circ);
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, malformed_ptr)
-
 /** Given that address is a malformed PTR name, we want dns_resolve_impl to
  * fail.
  */
 
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_malformed_ptr_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_malformed_ptr(void *arg)
 {
   int retval;
   int made_pending;
@@ -480,7 +525,8 @@ NS(test_main)(void *arg)
 
   TO_CONN(exitconn)->address = tor_strdup("1.0.0.127.in-addr.arpa");
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_malformed_ptr_router_my_exit_policy_is_reject_star);
 
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
@@ -498,30 +544,26 @@ NS(test_main)(void *arg)
   tt_int_op(retval,OP_EQ,-1);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(exitconn);
   tor_free(on_circ);
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, cache_hit_pending)
-
 /* Given that there is already a pending resolve for the given address,
  * we want dns_resolve_impl to append our exit connection to list
  * of pending connections for the pending DNS request and return 0.
  */
 
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_cache_hit_pending_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_cache_hit_pending(void *arg)
 {
   int retval;
   int made_pending = 0;
@@ -544,7 +586,8 @@ NS(test_main)(void *arg)
   strlcpy(cache_entry->address, TO_CONN(exitconn)->address,
           sizeof(cache_entry->address));
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_cache_hit_pending_router_my_exit_policy_is_reject_star);
 
   dns_init();
 
@@ -562,7 +605,7 @@ NS(test_main)(void *arg)
   tt_assert(pending_conn->conn == exitconn);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   tor_free(on_circ);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(cache_entry->pending_connections);
@@ -571,16 +614,12 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, cache_hit_cached)
-
 /* Given that a finished DNS resolve is available in our cache, we want
  * dns_resolve_impl() return it to called via resolve_out and pass the
  * handling to set_exitconn_info_from_resolve function.
  */
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_cache_hit_cached_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
@@ -589,7 +628,8 @@ static edge_connection_t *last_exitconn = NULL;
 static cached_resolve_t *last_resolve = NULL;
 
 static int
-NS(set_exitconn_info_from_resolve)(edge_connection_t *exitconn,
+dns_impl_cache_hit_cached_set_exitconn_info_from_resolve(
+                                   edge_connection_t *exitconn,
                                    const cached_resolve_t *resolve,
                                    char **hostname_out)
 {
@@ -602,7 +642,7 @@ NS(set_exitconn_info_from_resolve)(edge_connection_t *exitconn,
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_cache_hit_cached(void *arg)
 {
   int retval;
   int made_pending = 0;
@@ -625,8 +665,10 @@ NS(test_main)(void *arg)
   strlcpy(cache_entry->address, TO_CONN(exitconn)->address,
           sizeof(cache_entry->address));
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
-  NS_MOCK(set_exitconn_info_from_resolve);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_cache_hit_cached_router_my_exit_policy_is_reject_star);
+  MOCK(set_exitconn_info_from_resolve,
+       dns_impl_cache_hit_cached_set_exitconn_info_from_resolve);
 
   dns_init();
 
@@ -643,8 +685,8 @@ NS(test_main)(void *arg)
   tt_assert(last_resolve == cache_entry);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
-  NS_UNMOCK(set_exitconn_info_from_resolve);
+  UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(set_exitconn_info_from_resolve);
   tor_free(on_circ);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(cache_entry->pending_connections);
@@ -652,10 +694,6 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, cache_miss)
-
 /* Given that there are neither pending nor pre-cached resolve for a given
  * address, we want dns_resolve_impl() to create a new cached_resolve_t
  * object, mark it as pending, insert it into the cache, attach the exit
@@ -663,7 +701,7 @@ NS(test_main)(void *arg)
  * with the cached_resolve_t object it created.
  */
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_cache_miss_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
@@ -671,7 +709,7 @@ NS(router_my_exit_policy_is_reject_star)(void)
 static cached_resolve_t *last_launched_resolve = NULL;
 
 static int
-NS(launch_resolve)(cached_resolve_t *resolve)
+dns_impl_cache_miss_launch_resolve(cached_resolve_t *resolve)
 {
   last_launched_resolve = resolve;
 
@@ -679,7 +717,7 @@ NS(launch_resolve)(cached_resolve_t *resolve)
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_cache_miss(void *arg)
 {
   int retval;
   int made_pending = 0;
@@ -698,8 +736,10 @@ NS(test_main)(void *arg)
 
   strlcpy(query.address, TO_CONN(exitconn)->address, sizeof(query.address));
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
-  NS_MOCK(launch_resolve);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_cache_miss_router_my_exit_policy_is_reject_star);
+  MOCK(launch_resolve,
+       dns_impl_cache_miss_launch_resolve);
 
   dns_init();
 
@@ -722,8 +762,8 @@ NS(test_main)(void *arg)
   tt_str_op(cache_entry->address,OP_EQ,TO_CONN(exitconn)->address);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
-  NS_UNMOCK(launch_resolve);
+  UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(launch_resolve);
   tor_free(on_circ);
   tor_free(TO_CONN(exitconn)->address);
   if (cache_entry)
@@ -733,19 +773,22 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t dns_tests[] = {
-   TEST_CASE(clip_ttl),
-   TEST_CASE(resolve),
-   TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve),
-   TEST_CASE_ASPECT(resolve_impl, non_exit),
-   TEST_CASE_ASPECT(resolve_impl, addr_is_invalid_dest),
-   TEST_CASE_ASPECT(resolve_impl, malformed_ptr),
-   TEST_CASE_ASPECT(resolve_impl, cache_hit_pending),
-   TEST_CASE_ASPECT(resolve_impl, cache_hit_cached),
-   TEST_CASE_ASPECT(resolve_impl, cache_miss),
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+   { "configure_ns_fallback", test_dns_configure_ns_fallback,
+     TT_FORK, NULL, NULL },
+#endif
+   { "clip_ttl", test_dns_clip_ttl, TT_FORK, NULL, NULL },
+   { "resolve", test_dns_resolve, TT_FORK, NULL, NULL },
+   { "impl_addr_is_ip", test_dns_impl_addr_is_ip, TT_FORK, NULL, NULL },
+   { "impl_non_exit", test_dns_impl_non_exit, TT_FORK, NULL, NULL },
+   { "impl_addr_is_invalid_dest", test_dns_impl_addr_is_invalid_dest,
+     TT_FORK, NULL, NULL },
+   { "impl_malformed_ptr", test_dns_impl_malformed_ptr, TT_FORK, NULL, NULL },
+   { "impl_cache_hit_pending", test_dns_impl_cache_hit_pending,
+     TT_FORK, NULL, NULL },
+   { "impl_cache_hit_cached", test_dns_impl_cache_hit_cached,
+     TT_FORK, NULL, NULL },
+   { "impl_cache_miss", test_dns_impl_cache_miss, TT_FORK, NULL, NULL },
    END_OF_TESTCASES
 };
-
-#undef NS_MODULE
diff --git a/src/test/test_dos.c b/src/test/test_dos.c
index 01d7cd006e..527e5bbe7f 100644
--- a/src/test/test_dos.c
+++ b/src/test/test_dos.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DOS_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITLIST_PRIVATE
 
 #include "core/or/or.h"
@@ -411,7 +411,7 @@ test_dos_bucket_refill(void *arg)
   }
   tt_uint_op(current_circ_count, OP_EQ, 0);
   tt_uint_op(dos_stats->cc_stats.circuit_bucket, OP_EQ, current_circ_count);
-#endif
+#endif /* SIZEOF_TIME_T == 8 */
 
  done:
   tor_free(chan);
diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c
index fc7c5d5800..9cdd7f6d0e 100644
--- a/src/test/test_entryconn.c
+++ b/src/test/test_entryconn.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,7 +11,7 @@
 
 #include "feature/client/addressmap.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "feature/nodelist/nodelist.h"
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index a486b13ae1..5ddd1a3db0 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -1,10 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CIRCUITLIST_PRIVATE
 #define CIRCUITBUILD_PRIVATE
+#define CONFIG_PRIVATE
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
@@ -17,7 +18,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dirclient/dirclient.h"
@@ -67,7 +68,7 @@ static networkstatus_t *dummy_consensus = NULL;
 
 static smartlist_t *big_fake_net_nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 bfn_mock_nodelist_get_list(void)
 {
   return big_fake_net_nodes;
@@ -127,6 +128,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
   return 1; /* NOP */
 }
 
+#define REASONABLY_FUTURE " reasonably-future"
+#define REASONABLY_PAST " reasonably-past"
+
 /* Unittest setup function: Setup a fake network. */
 static void *
 big_fake_network_setup(const struct testcase_t *testcase)
@@ -138,9 +142,10 @@ big_fake_network_setup(const struct testcase_t *testcase)
   const int N_NODES = 271;
 
   const char *argument = testcase->setup_data;
-  int reasonably_live_consensus = 0;
+  int reasonably_future_consensus = 0, reasonably_past_consensus = 0;
   if (argument) {
-    reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL;
+    reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL;
+    reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL;
   }
 
   big_fake_net_nodes = smartlist_new();
@@ -193,16 +198,21 @@ big_fake_network_setup(const struct testcase_t *testcase)
       n->md->exit_policy = parse_short_policy("accept 443");
     }
 
+    n->nodelist_idx = smartlist_len(big_fake_net_nodes);
     smartlist_add(big_fake_net_nodes, n);
   }
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
   dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
-  if (reasonably_live_consensus) {
-    /* Make the dummy consensus valid from 4 hours ago, but expired an hour
+  if (reasonably_future_consensus) {
+    /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */
+    dummy_consensus->valid_after = approx_time() + 6*3600;
+    dummy_consensus->valid_until = approx_time() + 7*3600;
+  } else if (reasonably_past_consensus) {
+    /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours
      * ago. */
-    dummy_consensus->valid_after = approx_time() - 4*3600;
-    dummy_consensus->valid_until = approx_time() - 3600;
+    dummy_consensus->valid_after = approx_time() - 16*3600;
+    dummy_consensus->valid_until = approx_time() - 12*3600;
   } else {
     /* Make the dummy consensus valid for an hour either side of now. */
     dummy_consensus->valid_after = approx_time() - 3600;
@@ -226,12 +236,12 @@ mock_randomize_time_no_randomization(time_t a, time_t b)
   return a;
 }
 
-static or_options_t mocked_options;
+static or_options_t *mocked_options;
 
 static const or_options_t *
 mock_get_options(void)
 {
-  return &mocked_options;
+  return mocked_options;
 }
 
 #define TEST_IPV4_ADDR "123.45.67.89"
@@ -250,7 +260,7 @@ test_node_preferred_orport(void *arg)
   tor_addr_port_t ap;
 
   /* Setup options */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  mocked_options = options_new();
   /* We don't test ClientPreferIPv6ORPort here, because it's used in
    * nodelist_set_consensus to setup node.ipv6_preferred, which we set
    * directly. */
@@ -273,8 +283,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're only using IPv4, regardless
    * of whether we prefer it or not */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 0;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 0;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -287,8 +297,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
    * don't prefer the IPv6 address */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -296,28 +306,29 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv6 if we prefer it and
    * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 1;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
-  mocked_options.ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv4 = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
   /* Check the preferred address is IPv6 if we don't prefer it, but
    * ClientUseIPv4 is 0 */
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
+  mocked_options->ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv6 = 1;
+  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(mocked_options);
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
  done:
+  or_options_free(mocked_options);
   UNMOCK(get_options);
 }
 
@@ -379,12 +390,13 @@ test_entry_guard_encode_for_state_minimal(void *arg)
   eg->confirmed_idx = -1;
 
   char *s = NULL;
-  s = entry_guard_encode_for_state(eg);
+  s = entry_guard_encode_for_state(eg, 0);
 
   tt_str_op(s, OP_EQ,
             "in=wubwub "
             "rsa_id=706C75727079666C75727079736C75727079646F "
             "sampled_on=2016-11-14T00:00:00 "
+            "sampled_idx=0 "
             "listed=0");
 
  done:
@@ -410,10 +422,11 @@ test_entry_guard_encode_for_state_maximal(void *arg)
   eg->currently_listed = 1;
   eg->confirmed_on_date = 1479081690;
   eg->confirmed_idx = 333;
+  eg->sampled_idx = 42;
   eg->extra_state_fields = tor_strdup("and the green grass grew all around");
 
   char *s = NULL;
-  s = entry_guard_encode_for_state(eg);
+  s = entry_guard_encode_for_state(eg, 0);
 
   tt_str_op(s, OP_EQ,
             "in=default "
@@ -421,6 +434,7 @@ test_entry_guard_encode_for_state_maximal(void *arg)
             "bridge_addr=8.8.4.4:9999 "
             "nickname=Fred "
             "sampled_on=2016-11-14T00:00:00 "
+            "sampled_idx=0 "
             "sampled_by=1.2.3 "
             "unlisted_since=2016-11-14T00:00:45 "
             "listed=1 "
@@ -610,39 +624,47 @@ test_entry_guard_parse_from_state_full(void *arg)
   const char STATE[] =
   "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
     "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1\n"
   "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
     "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_idx=1 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
     "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
     "pb_successful_circuits_closed=2.000000\n"
   "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=2 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
     "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
     "pb_successful_circuits_closed=5.000000\n"
   "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1\n"
   "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
     "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_idx=3 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
     "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_idx=10 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
     "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev listed=1 "
     "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
     "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
     "pb_successful_circuits_closed=13.000000\n"
   "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
     "bridge_addr=37.218.246.143:28366 "
-    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n";
+    "sampled_on=2016-11-18T15:07:34 sampled_idx=1 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n";
 
   config_line_t *lines = NULL;
   or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
@@ -718,35 +740,42 @@ test_entry_guard_parse_from_state_full(void *arg)
   tt_str_op(joined, OP_EQ,
   "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
     "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
     "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
     "pb_successful_circuits_closed=2.000000\n"
   "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=1 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=1 "
     "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
     "pb_successful_circuits_closed=5.000000\n"
   "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
     "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_idx=2 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
     "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_idx=3 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1\n"
   "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
     "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev listed=1 "
     "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
     "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
     "pb_successful_circuits_closed=13.000000\n"
   "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
     "bridge_addr=37.218.246.143:28366 "
-    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n");
+    "sampled_on=2016-11-18T15:07:34 sampled_idx=1 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n");
 
  done:
   config_free_lines(lines);
@@ -1450,8 +1479,8 @@ test_entry_guard_confirming_guards(void *arg)
   tt_i64_op(g1->confirmed_on_date, OP_EQ, start+10);
   tt_i64_op(g2->confirmed_on_date, OP_EQ, start);
   tt_i64_op(g3->confirmed_on_date, OP_EQ, start+10);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g2);
   tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
 
   /* Now make sure we can regenerate the confirmed_entry_guards list. */
@@ -1463,8 +1492,8 @@ test_entry_guard_confirming_guards(void *arg)
   tt_int_op(g1->confirmed_idx, OP_EQ, 1);
   tt_int_op(g2->confirmed_idx, OP_EQ, 0);
   tt_int_op(g3->confirmed_idx, OP_EQ, 2);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g2);
   tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
 
   /* Now make sure we can regenerate the confirmed_entry_guards list if
@@ -1481,9 +1510,9 @@ test_entry_guard_confirming_guards(void *arg)
   g1 = smartlist_get(gs->confirmed_entry_guards, 0);
   g2 = smartlist_get(gs->confirmed_entry_guards, 1);
   g3 = smartlist_get(gs->confirmed_entry_guards, 2);
-  tt_int_op(g1->confirmed_idx, OP_EQ, 0);
-  tt_int_op(g2->confirmed_idx, OP_EQ, 1);
-  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_int_op(g1->sampled_idx, OP_EQ, 0);
+  tt_int_op(g2->sampled_idx, OP_EQ, 1);
+  tt_int_op(g3->sampled_idx, OP_EQ, 8);
   tt_assert(g1 != g2);
   tt_assert(g1 != g3);
   tt_assert(g2 != g3);
@@ -1499,9 +1528,6 @@ test_entry_guard_sample_reachable_filtered(void *arg)
   (void)arg;
   guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
   entry_guards_expand_sample(gs);
-  const int N = 10000;
-  bitarray_t *selected = NULL;
-  int i, j;
 
   /* We've got a sampled list now; let's make one non-usable-filtered; some
    * confirmed, some primary, some pending.
@@ -1536,32 +1562,21 @@ test_entry_guard_sample_reachable_filtered(void *arg)
     { SAMPLE_EXCLUDE_PENDING, 0 },
     { -1, -1},
   };
-
+  int j;
   for (j = 0; tests[j].flag >= 0; ++j) {
-    selected = bitarray_init_zero(n_guards);
     const int excluded_flags = tests[j].flag;
     const int excluded_idx = tests[j].idx;
-    for (i = 0; i < N; ++i) {
-      g = sample_reachable_filtered_entry_guards(gs, NULL, excluded_flags);
-      tor_assert(g);
-      int pos = smartlist_pos(gs->sampled_entry_guards, g);
-      tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
-      tt_int_op(pos, OP_GE, 0);
-      tt_int_op(pos, OP_LT, n_guards);
-      bitarray_set(selected, pos);
-    }
-    for (i = 0; i < n_guards; ++i) {
-      const int should_be_set = (i != excluded_idx &&
-                                 i != 3); // filtered out.
-      tt_int_op(!!bitarray_is_set(selected, i), OP_EQ, should_be_set);
-    }
-    bitarray_free(selected);
-    selected = NULL;
+    g = first_reachable_filtered_entry_guard(gs, NULL, excluded_flags);
+    tor_assert(g);
+    int pos = smartlist_pos(gs->sampled_entry_guards, g);
+    tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+    const int should_be_set = (pos != excluded_idx &&
+                                 pos != 3); // filtered out.
+    tt_int_op(1, OP_EQ, should_be_set);
   }
 
  done:
   guard_selection_free(gs);
-  bitarray_free(selected);
 }
 
 static void
@@ -1573,7 +1588,7 @@ test_entry_guard_sample_reachable_filtered_empty(void *arg)
   SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
                     n->is_possible_guard = 0);
 
-  entry_guard_t *g = sample_reachable_filtered_entry_guards(gs, NULL, 0);
+  entry_guard_t *g = first_reachable_filtered_entry_guard(gs, NULL, 0);
   tt_ptr_op(g, OP_EQ, NULL);
 
  done:
@@ -1664,10 +1679,13 @@ test_entry_guard_manage_primary(void *arg)
     tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
   });
 
-  /* If we have one confirmed guard, that guards becomes the first primary
-   * guard, and the other primary guards get kept. */
+  /**
+   * If we have one confirmed guard, that guards becomes the first primary
+   * only if its sampled_idx is smaller
+   * */
 
-  /* find a non-primary guard... */
+  /* find a non-primary guard... it should have a sampled_idx higher than
+   * existing primary guards */
   entry_guard_t *confirmed = NULL;
   SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, g, {
     if (! g->is_primary) {
@@ -1683,15 +1701,13 @@ test_entry_guard_manage_primary(void *arg)
   smartlist_add_all(prev_guards, gs->primary_entry_guards);
   entry_guards_update_primary(gs);
 
-  /*  and see what's primary now! */
+  /* the confirmed guard should be at the end of the primary list! Hopefully,
+   * one of the primary guards with a lower sampled_idx will confirm soon :)
+   * Doing this won't make the client switches between primaries depending on
+   * the order of confirming events */
   tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
-  tt_ptr_op(smartlist_get(gs->primary_entry_guards, 0), OP_EQ, confirmed);
-  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
-    tt_assert(g->is_primary);
-    if (g_sl_idx == 0)
-      continue;
-    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx - 1));
-  });
+  tt_ptr_op(smartlist_get(gs->primary_entry_guards,
+        smartlist_len(gs->primary_entry_guards)-1), OP_EQ, confirmed);
   {
     entry_guard_t *prev_last_guard = smartlist_get(prev_guards, n_primary-1);
     tt_assert(! prev_last_guard->is_primary);
@@ -1782,6 +1798,57 @@ test_entry_guard_guard_preferred(void *arg)
 }
 
 static void
+test_entry_guard_correct_cascading_order(void *arg)
+{
+  (void)arg;
+  smartlist_t *old_primary_guards = smartlist_new();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guards_expand_sample(gs);
+  /** First, a test in which the primary guards need be pulled from different
+   * lists to fill up the primary list -- this may happen, if for example, not
+   * enough guards have confirmed yet */
+  entry_guard_t *g;
+  /** just one confirmed */
+  g = smartlist_get(gs->sampled_entry_guards, 2);
+  make_guard_confirmed(gs, g);
+  entry_guards_update_primary(gs);
+  g = smartlist_get(gs->primary_entry_guards, 0);
+  tt_int_op(g->sampled_idx, OP_EQ, 0);
+  g = smartlist_get(gs->primary_entry_guards, 1);
+  tt_int_op(g->sampled_idx, OP_EQ, 1);
+  g = smartlist_get(gs->primary_entry_guards, 2);
+  tt_int_op(g->sampled_idx, OP_EQ, 2);
+
+  /** Now the primaries get all confirmed, and the primary list should not
+   * change */
+  make_guard_confirmed(gs, smartlist_get(gs->primary_entry_guards, 0));
+  make_guard_confirmed(gs, smartlist_get(gs->primary_entry_guards, 1));
+  smartlist_add_all(old_primary_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  smartlist_ptrs_eq(gs->primary_entry_guards, old_primary_guards);
+  /** the confirmed guards should also have the same set of guards, in the same
+   * order :-) */
+  smartlist_ptrs_eq(gs->confirmed_entry_guards, gs->primary_entry_guards);
+  /** Now select a guard for a circuit, and make sure it is the first primary
+   * guard */
+  unsigned state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+  /** Now, let's mark this guard as unreachable and let's update the lists */
+  g->is_reachable = GUARD_REACHABLE_NO;
+  g->failing_since = approx_time() - 10;
+  g->last_tried_to_connect = approx_time() - 10;
+  state = 9999;
+  entry_guards_update_primary(gs);
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  /** we should have switched to the next one is sampled order */
+  tt_int_op(g->sampled_idx, OP_EQ, 1);
+ done:
+  smartlist_free(old_primary_guards);
+  guard_selection_free(gs);
+}
+
+static void
 test_entry_guard_select_for_circuit_no_confirmed(void *arg)
 {
   /* Simpler cases: no gaurds are confirmed yet. */
@@ -3028,6 +3095,7 @@ static const struct testcase_setup_t upgrade_circuits = {
   upgrade_circuits_setup, upgrade_circuits_cleanup
 };
 
+#ifndef COCCI
 #define NO_PREFIX_TEST(name) \
   { #name, test_ ## name, 0, NULL, NULL }
 
@@ -3039,13 +3107,18 @@ static const struct testcase_setup_t upgrade_circuits = {
 
 #define BFN_TEST(name) \
   EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &big_fake_network, (void*)("reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_PAST) }
 
 #define UPGRADE_TEST(name, arg) \
   EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &upgrade_circuits, (void*)(arg " reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_PAST) }
+#endif /* !defined(COCCI) */
 
 struct testcase_t entrynodes_tests[] = {
   NO_PREFIX_TEST(node_preferred_orport),
@@ -3077,6 +3150,7 @@ struct testcase_t entrynodes_tests[] = {
   BFN_TEST(sample_reachable_filtered_empty),
   BFN_TEST(retry_unreachable),
   BFN_TEST(manage_primary),
+  BFN_TEST(correct_cascading_order),
 
   EN_TEST_FORK(guard_preferred),
 
diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c
index 0c34d37a71..7935530653 100644
--- a/src/test/test_extorport.c
+++ b/src/test/test_extorport.c
@@ -1,15 +1,15 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
 #define EXT_ORPORT_PRIVATE
 #define MAINLOOP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/relay/ext_orport.h"
 #include "core/mainloop/mainloop.h"
@@ -18,6 +18,7 @@
 
 #include "test/test.h"
 #include "test/test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -176,7 +177,7 @@ test_ext_or_init_auth(void *arg)
   /* Shouldn't be initialized already, or our tests will be a bit
    * meaningless */
   ext_or_auth_cookie = tor_malloc_zero(32);
-  tt_assert(tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Now make sure we use a temporary file */
   fn = get_fname("ext_cookie_file");
@@ -201,7 +202,7 @@ test_ext_or_init_auth(void *arg)
   tt_mem_op(cp,OP_EQ, "! Extended ORPort Auth Cookie !\x0a", 32);
   tt_mem_op(cp+32,OP_EQ, ext_or_auth_cookie, 32);
   memcpy(cookie0, ext_or_auth_cookie, 32);
-  tt_assert(!tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(!fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Operation should be idempotent. */
   tt_int_op(0, OP_EQ, init_ext_or_cookie_authentication(1));
@@ -303,16 +304,6 @@ test_ext_or_cookie_auth(void *arg)
 }
 
 static void
-crypto_rand_return_tse_str(char *to, size_t n)
-{
-  if (n != 32) {
-    TT_FAIL(("Asked for %d bytes, not 32", (int)n));
-    return;
-  }
-  memcpy(to, "te road There is always another ", 32);
-}
-
-static void
 test_ext_or_cookie_auth_testvec(void *arg)
 {
   char *reply=NULL, *client_hash=NULL;
@@ -326,7 +317,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
   memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32);
   ext_or_auth_cookie_is_set = 1;
 
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
 
   tt_int_op(0, OP_EQ,
             handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply,
@@ -351,7 +342,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
                  "33b3cd77ff79bd80c2074bbf438119a2");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(reply);
   tor_free(client_hash);
   tor_free(mem_op_hex_tmp);
@@ -414,9 +405,9 @@ do_ext_or_handshake(or_connection_t *conn)
   CONTAINS("\x01\x00", 2);
   WRITE("\x01", 1);
   WRITE("But when I look ahead up the whi", 32);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tt_int_op(TO_CONN(conn)->state, OP_EQ,
             EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH);
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
@@ -481,9 +472,9 @@ test_ext_or_handshake(void *arg)
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
   /* send the rest of the nonce. */
   WRITE("ahead up the whi", 16);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   /* We should get the right reply from the server. */
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
            "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21"
@@ -582,7 +573,7 @@ test_ext_or_handshake(void *arg)
 
  done:
   UNMOCK(connection_write_to_buf_impl_);
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   if (conn)
     connection_free_minimal(TO_CONN(conn));
 #undef CONTAINS
@@ -596,6 +587,6 @@ struct testcase_t extorport_tests[] = {
   { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL },
   { "cookie_auth_testvec", test_ext_or_cookie_auth_testvec, TT_FORK,
     NULL, NULL },
-  { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL },
+  { "handshake", test_ext_or_handshake, TT_FORK, &helper_pubsub_setup, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c
index 16c566bdbc..bf9932c169 100644
--- a/src/test/test_geoip.c
+++ b/src/test/test_geoip.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c
index ac8bfbfded..00c200e0fd 100644
--- a/src/test/test_guardfraction.c
+++ b/src/test/test_guardfraction.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define GUARDFRACTION_PRIVATE
diff --git a/src/test/test_handles.c b/src/test/test_handles.c
index 7f1d6e1898..dbb5b1a18e 100644
--- a/src/test/test_handles.c
+++ b/src/test/test_handles.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c
index 802d0a9ebe..f31c28b24d 100644
--- a/src/test/test_helpers.c
+++ b/src/test/test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,25 +9,33 @@
 #define ROUTERLIST_PRIVATE
 #define CONFIG_PRIVATE
 #define CONNECTION_PRIVATE
+#define CONNECTION_OR_PRIVATE
 #define MAINLOOP_PRIVATE
 
 #include "orconfig.h"
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/nodelist.h"
 #include "core/or/relay.h"
 #include "feature/nodelist/routerlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/connection_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "feature/nodelist/routerlist_st.h"
@@ -37,14 +45,14 @@
 #include "test/test_connection.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
 #include "test_descriptors.inc"
 #include "core/or/circuitlist.h"
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-ENABLE_GCC_WARNING(overlength-strings)
+ENABLE_GCC_WARNING("-Woverlength-strings")
 #endif
 
 /* Return a statically allocated string representing yesterday's date
@@ -78,7 +86,7 @@ helper_setup_fake_routerlist(void)
 {
   int retval;
   routerlist_t *our_routerlist = NULL;
-  smartlist_t *our_nodelist = NULL;
+  const smartlist_t *our_nodelist = NULL;
 
   /* Read the file that contains our test descriptors. */
 
@@ -189,6 +197,14 @@ fake_close_socket(tor_socket_t sock)
   return 0;
 }
 
+/* Helper for test_conn_get_proxy_or_connection() */
+void
+mock_connection_or_change_state(or_connection_t *conn, uint8_t state)
+{
+  tor_assert(conn);
+  conn->base_.state = state;
+}
+
 static int mock_connection_connect_sockaddr_called = 0;
 static int fake_socket_number = TEST_CONN_FD_INIT;
 
@@ -223,6 +239,77 @@ mock_connection_connect_sockaddr(connection_t *conn,
   return 1;
 }
 
+or_connection_t *
+test_conn_get_proxy_or_connection(unsigned int proxy_type)
+{
+  or_connection_t *conn = NULL;
+  tor_addr_t dst_addr;
+  tor_addr_t proxy_addr;
+  int socket_err = 0;
+  int in_progress = 0;
+
+  MOCK(connection_connect_sockaddr,
+       mock_connection_connect_sockaddr);
+  MOCK(connection_write_to_buf_impl_,
+       connection_write_to_buf_mock);
+  MOCK(connection_or_change_state,
+       mock_connection_or_change_state);
+  MOCK(tor_close_socket, fake_close_socket);
+
+  tor_init_connection_lists();
+
+  conn = or_connection_new(CONN_TYPE_OR, TEST_CONN_FAMILY);
+  tt_assert(conn);
+
+  /* Set up a destination address. */
+  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS, TEST_CONN_FAMILY,
+                               &dst_addr);
+  tt_assert(!tor_addr_is_null(&dst_addr));
+
+  conn->proxy_type = proxy_type;
+  conn->base_.proxy_state = PROXY_INFANT;
+
+  tor_addr_copy_tight(&conn->base_.addr, &dst_addr);
+  conn->base_.address = tor_addr_to_str_dup(&dst_addr);
+  conn->base_.port = TEST_CONN_PORT;
+
+  /* Set up a proxy address. */
+  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS_2, TEST_CONN_FAMILY,
+                               &proxy_addr);
+  tt_assert(!tor_addr_is_null(&proxy_addr));
+
+  conn->base_.state = OR_CONN_STATE_CONNECTING;
+
+  mock_connection_connect_sockaddr_called = 0;
+  in_progress = connection_connect(TO_CONN(conn), TEST_CONN_ADDRESS_PORT,
+                                   &proxy_addr, TEST_CONN_PORT, &socket_err);
+  tt_int_op(mock_connection_connect_sockaddr_called, OP_EQ, 1);
+  tt_assert(!socket_err);
+  tt_assert(in_progress == 0 || in_progress == 1);
+
+  assert_connection_ok(TO_CONN(conn), time(NULL));
+
+  in_progress = connection_or_finished_connecting(conn);
+  tt_int_op(in_progress, OP_EQ, 0);
+
+  assert_connection_ok(TO_CONN(conn), time(NULL));
+
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(connection_or_change_state);
+  UNMOCK(tor_close_socket);
+  return conn;
+
+  /* On failure */
+ done:
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(connection_or_change_state);
+  UNMOCK(tor_close_socket);
+  connection_free_(TO_CONN(conn));
+  return NULL;
+}
+
 /** Create and return a new connection/stream */
 connection_t *
 test_conn_get_connection(uint8_t state, uint8_t type, uint8_t purpose)
@@ -290,7 +377,7 @@ helper_parse_options(const char *conf)
   if (ret != 0) {
     goto done;
   }
-  ret = config_assign(&options_format, opt, line, 0, &msg);
+  ret = config_assign(get_options_mgr(), opt, line, 0, &msg);
   if (ret != 0) {
     goto done;
   }
@@ -303,3 +390,54 @@ helper_parse_options(const char *conf)
   }
   return opt;
 }
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Setup helper for tests that need pubsub active
+ *
+ * Does not hook up mainloop events.  Does set immediate delivery for
+ * all channels.
+ */
+void *
+helper_setup_pubsub(const struct testcase_t *testcase)
+{
+  dispatch_t *dispatcher = NULL;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  channel_id_t chan = get_channel_id("orconn");
+
+  (void)testcase;
+  (void)subsystems_add_pubsub(builder);
+  dispatcher = pubsub_builder_finalize(builder, NULL);
+  tor_assert(dispatcher);
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  chan = get_channel_id("ocirc");
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  return dispatcher;
+}
+
+/**
+ * Cleanup helper for tests that need pubsub active
+ */
+int
+helper_cleanup_pubsub(const struct testcase_t *testcase, void *dispatcher_)
+{
+  dispatch_t *dispatcher = dispatcher_;
+
+  (void)testcase;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+const struct testcase_setup_t helper_pubsub_setup = {
+  helper_setup_pubsub, helper_cleanup_pubsub
+};
diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h
index 9e376a563d..eaf18e19e2 100644
--- a/src/test/test_helpers.h
+++ b/src/test/test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_HELPERS_H
@@ -7,6 +7,7 @@
 #define BUFFERS_PRIVATE
 
 #include "core/or/or.h"
+#include "tinytest.h"
 
 const char *get_yesterday_date_str(void);
 
@@ -25,11 +26,19 @@ char *buf_get_contents(buf_t *buf, size_t *sz_out);
 int mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
                                             uint16_t family, tor_addr_t *out);
 
+void mock_connection_or_change_state(or_connection_t *conn, uint8_t state);
+
+or_connection_t *test_conn_get_proxy_or_connection(unsigned int proxy_type);
 connection_t *test_conn_get_connection(uint8_t state,
                                        uint8_t type, uint8_t purpose);
 or_options_t *helper_parse_options(const char *conf);
 
 extern const char TEST_DESCRIPTORS[];
 
+void *helper_setup_pubsub(const struct testcase_t *);
+int helper_cleanup_pubsub(const struct testcase_t *, void *);
+
+extern const struct testcase_setup_t helper_pubsub_setup;
+
 #endif /* !defined(TOR_TEST_HELPERS_H) */
 
diff --git a/src/test/test_hs.c b/src/test/test_hs.c
index a611b46ca6..46b4493a3d 100644
--- a/src/test/test_hs.c
+++ b/src/test/test_hs.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,7 +6,7 @@
  * \brief Unit tests for hidden service.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define RENDCOMMON_PRIVATE
 #define RENDSERVICE_PRIVATE
@@ -15,6 +15,8 @@
 #include "core/or/or.h"
 #include "test/test.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
 #include "feature/rend/rendcommon.h"
@@ -321,6 +323,16 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg,OP_EQ, expected_msg);
   tor_free(received_msg);
 
+  /* test HSDir rate limited */
+  rend_query.auth_type = REND_NO_AUTH;
+  control_event_hsv2_descriptor_failed(&rend_query.base_, NULL,
+                                     "QUERY_RATE_LIMITED");
+  expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \
+                 "UNKNOWN REASON=QUERY_RATE_LIMITED\r\n";
+  tt_assert(received_msg);
+  tt_str_op(received_msg,OP_EQ, expected_msg);
+  tor_free(received_msg);
+
   /* Test invalid content with no HSDir fingerprint. */
   char *exp_msg;
   control_event_hs_descriptor_content(rend_query.onion_address,
@@ -436,7 +448,7 @@ test_hs_rend_data(void *arg)
   tt_int_op(client_v2->auth_type, OP_EQ, REND_BASIC_AUTH);
   tt_int_op(strlen(client_v2->onion_address), OP_EQ, 0);
   tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie,
+  tt_int_op(fast_mem_is_zero(client_v2->descriptor_cookie,
                             sizeof(client_v2->descriptor_cookie)), OP_EQ, 1);
   tt_assert(client->hsdirs_fp);
   tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
index 742fa349d9..df96b2c791 100644
--- a/src/test/test_hs_cache.c
+++ b/src/test/test_hs_cache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,6 +10,7 @@
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
 #define HS_CACHE_PRIVATE
+#define CHANNEL_OBJECT_PRIVATE
 
 #include "trunnel/ed25519_cert.h"
 #include "feature/hs/hs_cache.h"
@@ -19,8 +20,14 @@
 #include "feature/nodelist/networkstatus.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_http.h"
+#include "core/or/circuitlist.h"
+#include "core/or/channel.h"
 #include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
 
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 
@@ -232,22 +239,31 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
   /* The dir conn we are going to simulate */
   dir_connection_t *conn = NULL;
+  edge_connection_t *edge_conn = NULL;
+  or_circuit_t *or_circ = NULL;
 
   /* First extract the blinded public key that we are going to use in our
      query, and then build the actual query string. */
   {
     char hsdir_cache_key[ED25519_BASE64_LEN+1];
 
-    retval = ed25519_public_to_base64(hsdir_cache_key,
-                                      blinded_key);
-    tt_int_op(retval, OP_EQ, 0);
+    ed25519_public_to_base64(hsdir_cache_key, blinded_key);
     tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key);
   }
 
   /* Simulate an HTTP GET request to the HSDir */
   conn = dir_connection_new(AF_INET);
+  tt_assert(conn);
+  TO_CONN(conn)->linked = 1; /* Signal that it is encrypted. */
   tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
-  TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */
+
+  /* Pretend this conn is anonymous. */
+  edge_conn = edge_connection_new(CONN_TYPE_EXIT, AF_INET);
+  TO_CONN(conn)->linked_conn = TO_CONN(edge_conn);
+  or_circ = or_circuit_new(0, NULL);
+  or_circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+  edge_conn->on_circuit = TO_CIRCUIT(or_circ);
+
   retval = directory_handle_command_get(conn, hsdir_query_str,
                                         NULL, 0);
   tt_int_op(retval, OP_EQ, 0);
@@ -264,8 +280,11 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
  done:
   tor_free(hsdir_query_str);
-  if (conn)
+  if (conn) {
+    tor_free(or_circ->p_chan);
+    connection_free_minimal(TO_CONN(conn)->linked_conn);
     connection_free_minimal(TO_CONN(conn));
+  }
 
   return received_desc;
 }
@@ -351,7 +370,7 @@ test_hsdir_revision_counter_check(void *arg)
   hs_descriptor_t *published_desc = NULL;
   char *published_desc_str = NULL;
 
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
   char *received_desc_str = NULL;
   hs_descriptor_t *received_desc = NULL;
 
@@ -388,12 +407,12 @@ test_hsdir_revision_counter_check(void *arg)
     const ed25519_public_key_t *blinded_key;
 
     blinded_key = &published_desc->plaintext_data.blinded_pubkey;
-    hs_get_subcredential(&signing_kp.pubkey, blinded_key, subcredential);
+    hs_get_subcredential(&signing_kp.pubkey, blinded_key, &subcredential);
     received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
 
     retval = hs_desc_decode_descriptor(received_desc_str,
-                                       subcredential, NULL, &received_desc);
-    tt_int_op(retval, OP_EQ, 0);
+                                       &subcredential, NULL, &received_desc);
+    tt_int_op(retval, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(received_desc);
 
     /* Check that the revision counter is correct */
@@ -425,8 +444,8 @@ test_hsdir_revision_counter_check(void *arg)
     received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
 
     retval = hs_desc_decode_descriptor(received_desc_str,
-                                       subcredential, NULL, &received_desc);
-    tt_int_op(retval, OP_EQ, 0);
+                                       &subcredential, NULL, &received_desc);
+    tt_int_op(retval, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(received_desc);
 
     /* Check that the revision counter is the latest */
@@ -458,7 +477,7 @@ test_client_cache(void *arg)
   ed25519_keypair_t signing_kp;
   hs_descriptor_t *published_desc = NULL;
   char *published_desc_str = NULL;
-  uint8_t wanted_subcredential[DIGEST256_LEN];
+  hs_subcredential_t wanted_subcredential;
   response_handler_args_t *args = NULL;
   dir_connection_t *conn = NULL;
 
@@ -487,8 +506,10 @@ test_client_cache(void *arg)
     retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
                                        NULL, &published_desc_str);
     tt_int_op(retval, OP_EQ, 0);
-    memcpy(wanted_subcredential, published_desc->subcredential, DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
+    memcpy(&wanted_subcredential, &published_desc->subcredential,
+           sizeof(hs_subcredential_t));
+    tt_assert(!fast_mem_is_zero((char*)wanted_subcredential.subcred,
+                                DIGEST256_LEN));
   }
 
   /* Test handle_response_fetch_hsdesc_v3() */
@@ -522,8 +543,9 @@ test_client_cache(void *arg)
     const hs_descriptor_t *cached_desc = NULL;
     cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
     tt_assert(cached_desc);
-    tt_mem_op(cached_desc->subcredential, OP_EQ, wanted_subcredential,
-              DIGEST256_LEN);
+    tt_mem_op(cached_desc->subcredential.subcred,
+              OP_EQ, wanted_subcredential.subcred,
+              SUBCRED_LEN);
   }
 
   /* Progress time to next TP and check that desc was cleaned */
@@ -550,6 +572,136 @@ test_client_cache(void *arg)
   }
 }
 
+/** Test that we can store HS descriptors in the client HS cache. */
+static void
+test_client_cache_decrypt(void *arg)
+{
+  int ret;
+  char *desc_encoded = NULL;
+  uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+  curve25519_keypair_t client_kp;
+  ed25519_keypair_t service_kp;
+  hs_descriptor_t *desc = NULL;
+  const hs_descriptor_t *search_desc;
+  const char *search_desc_encoded;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  hs_init();
+
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    ret = ed25519_keypair_generate(&service_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = curve25519_keypair_generate(&client_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    crypto_rand((char *) descriptor_cookie, sizeof(descriptor_cookie));
+
+    desc = hs_helper_build_hs_desc_with_client_auth(descriptor_cookie,
+                                                    &client_kp.pubkey,
+                                                    &service_kp);
+    tt_assert(desc);
+    ret = hs_desc_encode_descriptor(desc, &service_kp, descriptor_cookie,
+                                    &desc_encoded);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Put it in the cache. Should not be decrypted since the client
+   * authorization creds were not added to the global map. */
+  ret = hs_cache_store_as_client(desc_encoded, &service_kp.pubkey);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
+
+  /* We should not be able to decrypt anything. */
+  ret = hs_cache_client_new_auth_parse(&service_kp.pubkey);
+  tt_int_op(ret, OP_EQ, false);
+
+  /* Add client auth to global map. */
+  hs_helper_add_client_auth(&service_kp.pubkey, &client_kp.seckey);
+
+  /* We should not be able to decrypt anything. */
+  ret = hs_cache_client_new_auth_parse(&service_kp.pubkey);
+  tt_int_op(ret, OP_EQ, true);
+
+  /* Lookup the cache to make sure it is usable and there. */
+  search_desc = hs_cache_lookup_as_client(&service_kp.pubkey);
+  tt_assert(search_desc);
+  search_desc_encoded = hs_cache_lookup_encoded_as_client(&service_kp.pubkey);
+  tt_mem_op(search_desc_encoded, OP_EQ, desc_encoded, strlen(desc_encoded));
+
+ done:
+  hs_descriptor_free(desc);
+  tor_free(desc_encoded);
+
+  hs_free_all();
+
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+}
+
+static void
+test_client_cache_remove(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  /* Set consensus time. Lookup will not return the entry if it has expired
+   * and it is checked against the consensus valid_after time. */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Build a descriptor and cache it. */
+  {
+    char *encoded;
+    desc1 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc1);
+    ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* Store it */
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* Remove the cached entry. */
+  hs_cache_remove_as_client(&service_kp.pubkey);
+  tt_assert(!hs_cache_lookup_as_client(&service_kp.pubkey));
+
+ done:
+  hs_descriptor_free(desc1);
+  hs_free_all();
+
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+}
+
 struct testcase_t hs_cache[] = {
   /* Encoding tests. */
   { "directory", test_directory, TT_FORK,
@@ -562,6 +714,10 @@ struct testcase_t hs_cache[] = {
     NULL, NULL },
   { "client_cache", test_client_cache, TT_FORK,
     NULL, NULL },
+  { "client_cache_decrypt", test_client_cache_decrypt, TT_FORK,
+    NULL, NULL },
+  { "client_cache_remove", test_client_cache_remove, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c
index f8af631c8b..5406339276 100644
--- a/src/test/test_hs_cell.c
+++ b/src/test/test_hs_cell.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,6 +20,7 @@
 #include "feature/hs/hs_service.h"
 
 /* Trunnel. */
+#include "trunnel/hs/cell_common.h"
 #include "trunnel/hs/cell_establish_intro.h"
 
 /** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we
@@ -38,11 +39,13 @@ test_gen_establish_intro_cell(void *arg)
   /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
      attempt to parse it. */
   {
+    hs_service_config_t config;
+    memset(&config, 0, sizeof(config));
     /* We only need the auth key pair here. */
-    hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+    hs_service_intro_point_t *ip = service_intro_point_new(NULL);
     /* Auth key pair is generated in the constructor so we are all set for
      * using this IP object. */
-    ret = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+    ret = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
     service_intro_point_free(ip);
     tt_u64_op(ret, OP_GT, 0);
   }
@@ -97,6 +100,9 @@ test_gen_establish_intro_cell_bad(void *arg)
   trn_cell_establish_intro_t *cell = NULL;
   char circ_nonce[DIGEST_LEN] = {0};
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed);
 
@@ -107,8 +113,8 @@ test_gen_establish_intro_cell_bad(void *arg)
      ed25519_sign_prefixed() function and make it fail. */
   cell = trn_cell_establish_intro_new();
   tt_assert(cell);
-  ip = service_intro_point_new(NULL, 0, 0);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL);
+  ip = service_intro_point_new(NULL);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, NULL);
   service_intro_point_free(ip);
   expect_log_msg_containing("Unable to make signature for "
                             "ESTABLISH_INTRO cell.");
@@ -120,11 +126,97 @@ test_gen_establish_intro_cell_bad(void *arg)
   UNMOCK(ed25519_sign_prefixed);
 }
 
+static void
+test_gen_establish_intro_dos_ext(void *arg)
+{
+  ssize_t ret;
+  hs_service_config_t config;
+  hs_service_intro_point_t *ip = NULL;
+  trn_cell_extension_t *extensions = NULL;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  (void) arg;
+
+  memset(&config, 0, sizeof(config));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+
+  /* Case 1: No DoS parameters so no extension to be built. */
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 0);
+  trn_cell_extension_free(extensions);
+  extensions = NULL;
+
+  /* Case 2: Enable the DoS extension. Parameter set to 0 should indicate to
+   * disable the defense on the intro point but there should be an extension
+   * nonetheless in the cell. */
+  config.has_dos_defense_enabled = 1;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  const trn_cell_extension_field_t *field =
+    trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  const trn_cell_extension_dos_param_t *param =
+    trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+  /* Case 3: Enable the DoS extension. Parameter set to some normal values. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 42;
+  config.intro_dos_burst_per_sec = 250;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  field = trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 42);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 250);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+ done:
+  service_intro_point_free(ip);
+  trn_cell_extension_dos_free(dos);
+  trn_cell_extension_free(extensions);
+}
+
 struct testcase_t hs_cell_tests[] = {
   { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK,
     NULL, NULL },
   { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK,
     NULL, NULL },
+  { "gen_establish_intro_dos_ext", test_gen_establish_intro_dos_ext, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c
index 53ee3c53d2..0cd7d81eea 100644
--- a/src/test/test_hs_client.c
+++ b/src/test/test_hs_client.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,10 +10,11 @@
 #define CRYPTO_PRIVATE
 #define MAINLOOP_PRIVATE
 #define HS_CLIENT_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -24,6 +25,7 @@
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/channeltls.h"
 #include "feature/dircommon/directory.h"
 #include "core/mainloop/mainloop.h"
@@ -36,6 +38,7 @@
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_cache.h"
+#include "feature/rend/rendcache.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "core/mainloop/connection.h"
@@ -44,6 +47,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "core/or/entry_connection_st.h"
 #include "core/or/extend_info_st.h"
@@ -78,6 +82,23 @@ mock_networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
 }
 
 static int
+mock_write_str_to_file(const char *path, const char *str, int bin)
+{
+  (void) bin;
+  (void) path;
+  (void) str;
+  return 0;
+}
+
+static or_options_t mocked_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mocked_options;
+}
+
+static int
 helper_config_client(const char *conf, int validate_only)
 {
   int ret = 0;
@@ -91,6 +112,24 @@ helper_config_client(const char *conf, int validate_only)
   return ret;
 }
 
+static void
+helper_add_random_client_auth(const ed25519_public_key_t *service_pk)
+{
+  char *conf = NULL;
+#define conf_fmt "ClientOnionAuthDir %s\n"
+  tor_asprintf(&conf, conf_fmt, get_fname("auth_keys"));
+#undef conf_fmt
+  helper_config_client(conf, 0);
+  tor_free(conf);
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  hs_client_service_authorization_t *auth =
+    tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+  curve25519_secret_key_generate(&auth->enc_seckey, 0);
+  hs_build_address(service_pk, HS_VERSION_THREE, auth->onion_address);
+  digest256map_set(client_auths, service_pk->pubkey, auth);
+}
+
 /* Test helper function: Setup a circuit and a stream with the same hidden
  * service destination, and put them in <b>circ_out</b> and
  * <b>conn_out</b>. Make the stream wait for circuits to be established to the
@@ -159,8 +198,7 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out,
     or_circ->rend_data = rend_data_dup(conn_rend_data);
   } else {
     /* prop224: Setup hs ident on the circuit */
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
   }
 
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -243,12 +281,14 @@ test_e2e_rend_circuit_setup_legacy(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -285,7 +325,7 @@ test_e2e_rend_circuit_setup(void *arg)
        mock_connection_ap_handshake_send_begin);
 
   /* Setup */
-  retval = helper_get_circ_and_stream_for_test( &or_circ, &conn, 0);
+  retval = helper_get_circ_and_stream_for_test(&or_circ, &conn, 0);
   tt_int_op(retval, OP_EQ, 0);
   tt_assert(or_circ);
   tt_assert(conn);
@@ -301,9 +341,8 @@ test_e2e_rend_circuit_setup(void *arg)
   /**********************************************/
 
   /* Setup the circuit */
-  retval = hs_circuit_setup_e2e_rend_circ(or_circ,
-                                          ntor_key_seed, sizeof(ntor_key_seed),
-                                          0);
+  retval = hs_circuit_setup_e2e_rend_circ(or_circ, ntor_key_seed,
+                                          sizeof(ntor_key_seed), 0);
   tt_int_op(retval, OP_EQ, 0);
 
   /**********************************************/
@@ -313,12 +352,12 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check that the crypt path has prop224 algorithm parameters */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -389,15 +428,17 @@ test_client_pick_intro(void *arg)
     tt_assert(encoded);
 
     /* store it */
-    hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
 
     /* fetch it to make sure it works */
     const hs_descriptor_t *fetched_desc =
       hs_cache_lookup_as_client(&service_kp.pubkey);
     tt_assert(fetched_desc);
-    tt_mem_op(fetched_desc->subcredential, OP_EQ, desc->subcredential,
-              DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)fetched_desc->subcredential,
+    tt_mem_op(fetched_desc->subcredential.subcred,
+              OP_EQ, desc->subcredential.subcred,
+              SUBCRED_LEN);
+    tt_assert(!fast_mem_is_zero((char*)fetched_desc->subcredential.subcred,
                                DIGEST256_LEN));
     tor_free(encoded);
   }
@@ -405,6 +446,9 @@ test_client_pick_intro(void *arg)
   /* 2) Mark all intro points except _the chosen one_ as failed. Then query the
    *   desc and get a random intro: check that we got _the chosen one_. */
   {
+    /* Tell hs_get_extend_info_from_lspecs() to skip the private address check.
+     */
+    get_options_mutable()->ExtendAllowPrivateAddresses = 1;
     /* Pick the chosen intro point and get its ei */
     hs_desc_intro_point_t *chosen_intro_point =
       smartlist_get(desc->encrypted_data.intro_points, 0);
@@ -432,7 +476,7 @@ test_client_pick_intro(void *arg)
     for (int i = 0; i < 64; ++i) {
       extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
       tor_assert(ip);
-      tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
+      tt_assert(!fast_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
       tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest,
                 DIGEST_LEN);
       extend_info_free(ip);
@@ -478,6 +522,18 @@ test_client_pick_intro(void *arg)
     SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                             hs_desc_intro_point_t *, ip) {
       extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip);
+      /* desc_intro_point_to_extend_info() doesn't return IPv6 intro points
+       * yet, because we can't extend to them. See #24404, #24451, and #24181.
+       */
+      if (intro_ei == NULL) {
+        /* Pretend we're making a direct connection, and that we can use IPv6
+         */
+        get_options_mutable()->ClientUseIPv6 = 1;
+        intro_ei = hs_get_extend_info_from_lspecs(ip->link_specifiers,
+                                                  &ip->onion_key, 1);
+        tt_assert(tor_addr_family(&intro_ei->addr) == AF_INET6);
+      }
+      tt_assert(intro_ei);
       if (intro_ei) {
         const char *ptr;
         char ip_addr[TOR_ADDR_BUF_LEN];
@@ -531,6 +587,17 @@ mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
 }
 
 static void
+mock_connection_mark_unattached_ap_no_close(entry_connection_t *conn,
+                                            int endreason, int line,
+                                            const char *file)
+{
+  (void) conn;
+  (void) endreason;
+  (void) line;
+  (void) file;
+}
+
+static void
 test_descriptor_fetch(void *arg)
 {
   int ret;
@@ -668,6 +735,10 @@ test_parse_auth_file_content(void *arg)
   /* Bigger key than it should be */
   tt_assert(!parse_auth_file_content("xx:descriptor:x25519:"
                      "vjqea4jbhwwc4hto7ekyvqfbeodghbaq6nxi45hz4wr3qvhqv3yqa"));
+  /* All-zeroes key */
+  tt_assert(!parse_auth_file_content("xx:descriptor:x25519:"
+            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+
  done:
   tor_free(auth);
 }
@@ -805,6 +876,7 @@ test_desc_has_arrived_cleanup(void *arg)
   ed25519_keypair_t signing_kp;
   entry_connection_t *socks1 = NULL, *socks2 = NULL;
   hs_ident_dir_conn_t hs_dir_ident;
+  dir_connection_t *dir_conn = NULL;
 
   (void) arg;
 
@@ -833,7 +905,7 @@ test_desc_has_arrived_cleanup(void *arg)
 
   /* Store in the client cache. */
   ret = hs_cache_store_as_client(desc_str, &signing_kp.pubkey);
-  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
   cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
   tt_assert(cached_desc);
   hs_helper_desc_equal(desc, cached_desc);
@@ -861,9 +933,11 @@ test_desc_has_arrived_cleanup(void *arg)
    * SOCKS connection to be ended with a resolved failed. */
   hs_ident_dir_conn_init(&signing_kp.pubkey,
                          &desc->plaintext_data.blinded_pubkey, &hs_dir_ident);
-  hs_client_desc_has_arrived(&hs_dir_ident);
+  dir_conn = dir_connection_new(AF_INET);
+  dir_conn->hs_ident = hs_ident_dir_conn_dup(&hs_dir_ident);
+  hs_client_dir_fetch_done(dir_conn, "A reason", desc_str, 200);
+  connection_free_minimal(TO_CONN(dir_conn));
   tt_int_op(socks1->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
-  /* XXX: MUST work with OP_EQ. */
   tt_int_op(socks2->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
 
   /* Now let say tor cleans up the intro state cache which resets all intro
@@ -872,7 +946,6 @@ test_desc_has_arrived_cleanup(void *arg)
 
   /* Retrying all SOCKS which should basically do nothing since we don't have
    * any pending SOCKS connection in AP_CONN_STATE_RENDDESC_WAIT state. */
-  /* XXX: BUG() is triggered here, shouldn't if socks2 wasn't alive. */
   retry_all_socks_conn_waiting_for_desc();
 
  done:
@@ -899,6 +972,7 @@ test_close_intro_circuits_new_desc(void *arg)
   (void) arg;
 
   hs_init();
+  rend_cache_init();
 
   /* This is needed because of the client cache expiration timestamp is based
    * on having a consensus. See cached_client_descriptor_has_expired(). */
@@ -923,6 +997,51 @@ test_close_intro_circuits_new_desc(void *arg)
   circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
   ocirc = TO_ORIGIN_CIRCUIT(circ);
 
+  /* Build a descriptor _without_ client authorization and thus not
+   * decryptable. Make sure the close circuit code path is not triggered. */
+  {
+    char *desc_encoded = NULL;
+    uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+    curve25519_keypair_t client_kp;
+    hs_descriptor_t *desc = NULL;
+
+    tt_int_op(0, OP_EQ, curve25519_keypair_generate(&client_kp, 0));
+    crypto_rand((char *) descriptor_cookie, sizeof(descriptor_cookie));
+
+    desc = hs_helper_build_hs_desc_with_client_auth(descriptor_cookie,
+                                                    &client_kp.pubkey,
+                                                    &service_kp);
+    tt_assert(desc);
+    ret = hs_desc_encode_descriptor(desc, &service_kp, descriptor_cookie,
+                                    &desc_encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Associate descriptor intro key with the dummy circuit. */
+    const hs_desc_intro_point_t *ip =
+      smartlist_get(desc->encrypted_data.intro_points, 0);
+    tt_assert(ip);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+    ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+    hs_descriptor_free(desc);
+    tt_assert(desc_encoded);
+    /* Put it in the cache. Should not be decrypted since the client
+     * authorization creds were not added to the global map. */
+    ret = hs_cache_store_as_client(desc_encoded, &service_kp.pubkey);
+    tor_free(desc_encoded);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
+
+    /* Clean cache with a future timestamp. It will trigger the clean up and
+     * attempt to close the circuit but only if the descriptor is decryptable.
+     * Cache object should be removed and circuit untouched. */
+    hs_cache_clean_as_client(mock_ns.valid_after + (60 * 60 * 24));
+    tt_assert(!hs_cache_lookup_as_client(&service_kp.pubkey));
+
+    /* Make sure the circuit still there. */
+    tt_assert(circuit_get_next_intro_circ(NULL, true));
+    /* Get rid of the ident, it will be replaced in the next tests. */
+    hs_ident_circuit_free(ocirc->hs_ident);
+  }
+
   /* Build the first descriptor and cache it. */
   {
     char *encoded;
@@ -934,7 +1053,7 @@ test_close_intro_circuits_new_desc(void *arg)
 
     /* Store it */
     ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tor_free(encoded);
     tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
   }
@@ -944,8 +1063,7 @@ test_close_intro_circuits_new_desc(void *arg)
     const hs_desc_intro_point_t *ip =
       smartlist_get(desc1->encrypted_data.intro_points, 0);
     tt_assert(ip);
-    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey,
-                                           HS_IDENT_CIRCUIT_INTRO);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
     ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
                         &ip->auth_key_cert->signed_key);
   }
@@ -971,7 +1089,7 @@ test_close_intro_circuits_new_desc(void *arg)
     tt_assert(encoded);
 
     ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tor_free(encoded);
     tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
   }
@@ -988,6 +1106,451 @@ test_close_intro_circuits_new_desc(void *arg)
   UNMOCK(networkstatus_get_reasonably_live_consensus);
 }
 
+static void
+test_close_intro_circuits_cache_clean(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  circuit_t *circ = NULL;
+  origin_circuit_t *ocirc = NULL;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  hs_init();
+  rend_cache_init();
+
+  /* This is needed because of the client cache expiration timestamp is based
+   * on having a consensus. See cached_client_descriptor_has_expired(). */
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Create and add to the global list a dummy client introduction circuits.
+   * We'll then make sure the hs_ident is attached to a dummy descriptor. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+  /* Build the first descriptor and cache it. */
+  {
+    char *encoded;
+    desc1 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc1);
+    ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* Store it */
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, 0);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* We'll pick one introduction point and associate it with the circuit. */
+  {
+    const hs_desc_intro_point_t *ip =
+      smartlist_get(desc1->encrypted_data.intro_points, 0);
+    tt_assert(ip);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+    ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+  }
+
+  /* Before we are about to clean up the intro circuits, make sure it is
+   * actually there. */
+  tt_assert(circuit_get_next_intro_circ(NULL, true));
+
+  /* Cleanup the client cache. The ns valid after time is what decides if the
+   * descriptor has expired so put it in the future enough (72h) so we are
+   * sure to always expire. */
+  mock_ns.valid_after = approx_time() + (72 * 24 * 60 * 60);
+  hs_cache_clean_as_client(0);
+
+  /* Once stored, our intro circuit should be closed because it is related to
+   * an old introduction point that doesn't exists anymore. */
+  tt_assert(!circuit_get_next_intro_circ(NULL, true));
+
+ done:
+  circuit_free(circ);
+  hs_descriptor_free(desc1);
+  hs_free_all();
+  rend_cache_free_all();
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+}
+
+static void
+test_socks_hs_errors(void *arg)
+{
+  int ret;
+  char digest[DIGEST_LEN];
+  char *desc_encoded = NULL;
+  circuit_t *circ = NULL;
+  origin_circuit_t *ocirc = NULL;
+  tor_addr_t addr;
+  ed25519_keypair_t service_kp;
+  ed25519_keypair_t signing_kp;
+  entry_connection_t *socks_conn = NULL;
+  dir_connection_t *dir_conn = NULL;
+  hs_descriptor_t *desc = NULL;
+  uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+
+  (void) arg;
+
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+  MOCK(connection_mark_unattached_ap_,
+       mock_connection_mark_unattached_ap_no_close);
+  MOCK(read_file_to_str, mock_read_file_to_str);
+  MOCK(tor_listdir, mock_tor_listdir);
+  MOCK(check_private_dir, mock_check_private_dir);
+
+    /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                           &mock_ns.valid_until);
+
+  hs_init();
+
+  ret = ed25519_keypair_generate(&service_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  socks_conn = helper_build_socks_connection(&service_kp.pubkey,
+                                             AP_CONN_STATE_RENDDESC_WAIT);
+  tt_assert(socks_conn);
+
+  /* Create directory connection. */
+  dir_conn = dir_connection_new(AF_INET);
+  dir_conn->hs_ident = tor_malloc_zero(sizeof(hs_ident_dir_conn_t));
+  TO_CONN(dir_conn)->purpose = DIR_PURPOSE_FETCH_HSDESC;
+  ed25519_pubkey_copy(&dir_conn->hs_ident->identity_pk, &service_kp.pubkey);
+
+  /* Encode descriptor so we can decode it. */
+  desc = hs_helper_build_hs_desc_with_ip(&service_kp);
+  tt_assert(desc);
+
+  /* Before testing the client authentication error code, encode the
+   * descriptor with no client auth. */
+  ret = hs_desc_encode_descriptor(desc, &service_kp, NULL, &desc_encoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(desc_encoded);
+
+  /*
+   * Test the introduction failure codes (X'F2' and X'F7')
+   */
+
+  /* First, we have to put all the IPs in the failure cache. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          hs_desc_intro_point_t *, ip) {
+    hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                     &ip->auth_key_cert->signed_key,
+                                     INTRO_POINT_FAILURE_GENERIC);
+  } SMARTLIST_FOREACH_END(ip);
+
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_INTRO_FAILED);
+
+  /* Purge client cache of the descriptor so we can go again. */
+  hs_cache_purge_as_client();
+
+  /* Second, set all failures to be time outs. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          hs_desc_intro_point_t *, ip) {
+    hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                     &ip->auth_key_cert->signed_key,
+                                     INTRO_POINT_FAILURE_TIMEOUT);
+  } SMARTLIST_FOREACH_END(ip);
+
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_INTRO_TIMEDOUT);
+
+  /* Purge client cache of the descriptor so we can go again. */
+  hs_cache_purge_as_client();
+
+  /*
+   * Test the rendezvous failure codes (X'F3')
+   */
+
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_REND_READY;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  /* Attach socks connection to this rendezvous circuit. */
+  ocirc->p_streams = ENTRY_TO_EDGE_CONN(socks_conn);
+  /* Trigger the rendezvous failure. Timeout the circuit and free. */
+  circuit_mark_for_close(circ, END_CIRC_REASON_TIMEOUT);
+
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_REND_FAILED);
+
+  /*
+   * Test client authorization codes.
+   */
+
+  tor_free(desc_encoded);
+  crypto_rand((char *) descriptor_cookie, sizeof(descriptor_cookie));
+  ret = hs_desc_encode_descriptor(desc, &service_kp, descriptor_cookie,
+                                  &desc_encoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(desc_encoded);
+
+  /* Try decoding. Point this to an existing descriptor. The following should
+   * fail thus the desc_out should be set to NULL. */
+  hs_descriptor_t *desc_out = desc;
+  ret = hs_client_decode_descriptor(desc_encoded, &service_kp.pubkey,
+                                    &desc_out);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
+  tt_assert(desc_out == NULL);
+
+  /* The caching will fail to decrypt because the descriptor_cookie used above
+   * is not known to the HS subsystem. This will lead to a missing client
+   * auth. */
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_MISSING_CLIENT_AUTH);
+
+  /* Add in the global client auth list bad creds for this service. */
+  helper_add_random_client_auth(&service_kp.pubkey);
+
+  ret = hs_client_decode_descriptor(desc_encoded, &service_kp.pubkey,
+                                    &desc_out);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_BAD_CLIENT_AUTH);
+  tt_assert(desc_out == NULL);
+
+  /* Simmulate a fetch done again. This should replace the cached descriptor
+   * and signal a bad client authorization. */
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_BAD_CLIENT_AUTH);
+
+ done:
+  connection_free_minimal(ENTRY_TO_CONN(socks_conn));
+  connection_free_minimal(TO_CONN(dir_conn));
+  hs_descriptor_free(desc);
+  tor_free(desc_encoded);
+  circuit_free(circ);
+
+  hs_free_all();
+
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+  UNMOCK(connection_mark_unattached_ap_);
+  UNMOCK(read_file_to_str);
+  UNMOCK(tor_listdir);
+  UNMOCK(check_private_dir);
+}
+
+static void
+test_close_intro_circuit_failure(void *arg)
+{
+  char digest[DIGEST_LEN];
+  circuit_t *circ = NULL;
+  ed25519_keypair_t service_kp, intro_kp;
+  origin_circuit_t *ocirc = NULL;
+  tor_addr_t addr;
+  const hs_cache_intro_state_t *entry;
+
+  (void) arg;
+
+  hs_init();
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&intro_kp, 0));
+
+  /* Create and add to the global list a dummy client introduction circuit at
+   * the ACK WAIT state. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  /* We'll make for close the circuit for a timeout failure. It should _NOT_
+   * end up in the failure cache just yet. We do that on free() only. */
+  circuit_mark_for_close(circ, END_CIRC_REASON_TIMEOUT);
+  tt_assert(!hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                              &intro_kp.pubkey));
+  /* Time to free. It should get removed. */
+  circuit_free(circ);
+  entry = hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                           &intro_kp.pubkey);
+  tt_assert(entry);
+  tt_uint_op(entry->timed_out, OP_EQ, 1);
+  hs_cache_client_intro_state_purge();
+
+  /* Again, create and add to the global list a dummy client introduction
+   * circuit at the INTRODUCING state. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  /* On free, we should get an unreachable failure. */
+  circuit_free(circ);
+  entry = hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                           &intro_kp.pubkey);
+  tt_assert(entry);
+  tt_uint_op(entry->unreachable_count, OP_EQ, 1);
+  hs_cache_client_intro_state_purge();
+
+  /* Again, create and add to the global list a dummy client introduction
+   * circuit at the INTRODUCING state but we'll close it for timeout. It
+   * should not be noted as a timeout failure. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  circuit_mark_for_close(circ, END_CIRC_REASON_TIMEOUT);
+  circuit_free(circ);
+  tt_assert(!hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                              &intro_kp.pubkey));
+
+  /* Again, create and add to the global list a dummy client introduction
+   * circuit at the INTRODUCING state but without a chosen_exit. In theory, it
+   * can not happen but we'll make sure it doesn't end up in the failure cache
+   * anyway. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  circuit_free(circ);
+  tt_assert(!hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                              &intro_kp.pubkey));
+
+ done:
+  circuit_free(circ);
+  hs_free_all();
+}
+
+static void
+test_purge_ephemeral_client_auth(void *arg)
+{
+  ed25519_keypair_t service_kp;
+  hs_client_service_authorization_t *auth = NULL;
+  hs_client_register_auth_status_t status;
+
+  (void) arg;
+
+  /* We will try to write on disk client credentials. */
+  MOCK(check_private_dir, mock_check_private_dir);
+  MOCK(get_options, mock_get_options);
+  MOCK(write_str_to_file, mock_write_str_to_file);
+
+  /* Boggus directory so when we try to write the permanent client
+   * authorization data to disk, we don't fail. See
+   * store_permanent_client_auth_credentials() for more details. */
+  mocked_options.ClientOnionAuthDir = tor_strdup("auth_dir");
+
+  hs_init();
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Generate a client authorization object. */
+  auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+
+  /* Set it up. No flags meaning it is ephemeral. */
+  curve25519_secret_key_generate(&auth->enc_seckey, 0);
+  hs_build_address(&service_kp.pubkey, HS_VERSION_THREE, auth->onion_address);
+  auth->flags = 0;
+
+  /* Confirm that there is nothing in the client auth map. It is unallocated
+   * until we add the first entry. */
+  tt_assert(!get_hs_client_auths_map());
+
+  /* Add an entry to the client auth list. We loose ownership of the auth
+   * object so nullify it. */
+  status = hs_client_register_auth_credentials(auth);
+  auth = NULL;
+  tt_int_op(status, OP_EQ, REGISTER_SUCCESS);
+
+  /* We should have the entry now. */
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(client_auths);
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 1);
+
+  /* Purge the cache that should remove all ephemeral values. */
+  purge_ephemeral_client_auth();
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 0);
+
+  /* Now add a new authorization object but permanent. */
+  /* Generate a client authorization object. */
+  auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+  curve25519_secret_key_generate(&auth->enc_seckey, 0);
+  hs_build_address(&service_kp.pubkey, HS_VERSION_THREE, auth->onion_address);
+  auth->flags = CLIENT_AUTH_FLAG_IS_PERMANENT;
+
+  /* Add an entry to the client auth list. We loose ownership of the auth
+   * object so nullify it. */
+  status = hs_client_register_auth_credentials(auth);
+  auth = NULL;
+  tt_int_op(status, OP_EQ, REGISTER_SUCCESS);
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 1);
+
+  /* Purge again, the entry should still be there. */
+  purge_ephemeral_client_auth();
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 1);
+
+ done:
+  client_service_authorization_free(auth);
+  hs_free_all();
+  tor_free(mocked_options.ClientOnionAuthDir);
+
+  UNMOCK(check_private_dir);
+  UNMOCK(get_options);
+  UNMOCK(write_str_to_file);
+}
+
 struct testcase_t hs_client_tests[] = {
   { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy,
     TT_FORK, NULL, NULL },
@@ -1005,8 +1568,19 @@ struct testcase_t hs_client_tests[] = {
     TT_FORK, NULL, NULL },
   { "desc_has_arrived_cleanup", test_desc_has_arrived_cleanup,
     TT_FORK, NULL, NULL },
+  { "close_intro_circuit_failure", test_close_intro_circuit_failure,
+    TT_FORK, NULL, NULL },
   { "close_intro_circuits_new_desc", test_close_intro_circuits_new_desc,
     TT_FORK, NULL, NULL },
+  { "close_intro_circuits_cache_clean", test_close_intro_circuits_cache_clean,
+    TT_FORK, NULL, NULL },
+
+  /* SOCKS5 Extended Error Code. */
+  { "socks_hs_errors", test_socks_hs_errors, TT_FORK, NULL, NULL },
+
+  /* Client authorization. */
+  { "purge_ephemeral_client_auth", test_purge_ephemeral_client_auth, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c
index c6104301e1..e3d130fb32 100644
--- a/src/test/test_hs_common.c
+++ b/src/test/test_hs_common.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,6 +6,7 @@
  * \brief Test hidden service common functionalities.
  */
 
+#define CONNECTION_EDGE_PRIVATE
 #define HS_COMMON_PRIVATE
 #define HS_CLIENT_PRIVATE
 #define HS_SERVICE_PRIVATE
@@ -31,7 +32,7 @@
 #include "app/config/statefile.h"
 #include "core/or/circuitlist.h"
 #include "feature/dirauth/shared_random.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -52,14 +53,14 @@ test_validate_address(void *arg)
   setup_full_capture_of_logs(LOG_WARN);
   ret = hs_address_is_valid("blah");
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg_containing("has an invalid length");
+  expect_log_msg_containing("Invalid length");
   teardown_capture_of_logs();
 
   setup_full_capture_of_logs(LOG_WARN);
   ret = hs_address_is_valid(
            "p3xnclpu4mu22dwaurjtsybyqk4xfjmcfz6z62yl24uwmhjatiwnlnadb");
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg_containing("has an invalid length");
+  expect_log_msg_containing("Invalid length");
   teardown_capture_of_logs();
 
   /* Invalid checksum (taken from prop224) */
@@ -82,7 +83,7 @@ test_validate_address(void *arg)
   ret = hs_address_is_valid(
            "????????????????????????????????????????????????????????");
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg_containing("can't be decoded");
+  expect_log_msg_containing("Unable to base32 decode");
   teardown_capture_of_logs();
 
   /* Valid address. */
@@ -275,7 +276,7 @@ test_start_time_of_next_time_period(void *arg)
 static void
 cleanup_nodelist(void)
 {
-  smartlist_t *nodelist = nodelist_get_list();
+  const smartlist_t *nodelist = nodelist_get_list();
   SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
     tor_free(node->md);
     node->md = NULL;
@@ -509,6 +510,7 @@ test_desc_reupload_logic(void *arg)
                 pubkey_hex, strlen(pubkey_hex));
   hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
   service = tor_malloc_zero(sizeof(hs_service_t));
+  tt_assert(service);
   memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
   ed25519_secret_key_generate(&service->keys.identity_sk, 0);
   ed25519_public_key_generate(&service->keys.identity_pk,
@@ -610,6 +612,10 @@ test_desc_reupload_logic(void *arg)
   SMARTLIST_FOREACH(ns->routerstatus_list,
                     routerstatus_t *, rs, routerstatus_free(rs));
   smartlist_clear(ns->routerstatus_list);
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   networkstatus_vote_free(ns);
   cleanup_nodelist();
   hs_free_all();
@@ -637,7 +643,7 @@ test_disaster_srv(void *arg)
   get_disaster_srv(1, srv_one);
   get_disaster_srv(2, srv_two);
 
-  /* Check that the cached ones where updated */
+  /* Check that the cached ones were updated */
   tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
   tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
 
@@ -780,6 +786,7 @@ static void
 test_parse_extended_hostname(void *arg)
 {
   (void) arg;
+  hostname_type_t type;
 
   char address1[] = "fooaddress.onion";
   char address2[] = "aaaaaaaaaaaaaaaa.onion";
@@ -790,21 +797,48 @@ test_parse_extended_hostname(void *arg)
   char address7[] = ".abcdefghijklmnop.onion";
   char address8[] =
     "www.25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion";
+  char address9[] =
+    "www.15njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion";
+  char address10[] =
+    "15njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid7jdl.onion";
+
+  tt_assert(!parse_extended_hostname(address1, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
+  tt_assert(parse_extended_hostname(address2, &type));
+  tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME);
+  tt_str_op(address2, OP_EQ, "aaaaaaaaaaaaaaaa");
+
+  tt_assert(parse_extended_hostname(address3, &type));
+  tt_int_op(type, OP_EQ, EXIT_HOSTNAME);
+
+  tt_assert(parse_extended_hostname(address4, &type));
+  tt_int_op(type, OP_EQ, NORMAL_HOSTNAME);
 
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address1));
-  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address2));
-  tt_str_op(address2,OP_EQ, "aaaaaaaaaaaaaaaa");
-  tt_assert(EXIT_HOSTNAME == parse_extended_hostname(address3));
-  tt_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4));
-  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address5));
-  tt_str_op(address5,OP_EQ, "abcdefghijklmnop");
-  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address6));
-  tt_str_op(address6,OP_EQ, "abcdefghijklmnop");
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address7));
-  tt_assert(ONION_V3_HOSTNAME == parse_extended_hostname(address8));
+  tt_assert(parse_extended_hostname(address5, &type));
+  tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME);
+  tt_str_op(address5, OP_EQ, "abcdefghijklmnop");
+
+  tt_assert(parse_extended_hostname(address6, &type));
+  tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME);
+  tt_str_op(address6, OP_EQ, "abcdefghijklmnop");
+
+  tt_assert(!parse_extended_hostname(address7, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
+  tt_assert(parse_extended_hostname(address8, &type));
+  tt_int_op(type, OP_EQ, ONION_V3_HOSTNAME);
   tt_str_op(address8, OP_EQ,
             "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
 
+  /* Invalid v3 address. */
+  tt_assert(!parse_extended_hostname(address9, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
+  /* Invalid v3 address: too long */
+  tt_assert(!parse_extended_hostname(address10, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
  done: ;
 }
 
@@ -832,7 +866,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 01:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -840,7 +874,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -848,7 +882,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 1);
 
@@ -856,7 +890,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 1);
 
@@ -864,7 +898,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -1354,7 +1388,7 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario)
                       &mock_service_ns->valid_until);
   set_consensus_times(cfg->service_valid_until,
                       &mock_service_ns->fresh_until);
-  voting_schedule_recalculate_timing(get_options(),
+  dirauth_sched_recalculate_timing(get_options(),
                                      mock_service_ns->valid_after);
   /* Check that service is in the right time period point */
   tt_int_op(hs_in_period_between_tp_and_srv(mock_service_ns, 0), OP_EQ,
@@ -1367,7 +1401,7 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario)
                       &mock_client_ns->valid_until);
   set_consensus_times(cfg->client_valid_until,
                       &mock_client_ns->fresh_until);
-  voting_schedule_recalculate_timing(get_options(),
+  dirauth_sched_recalculate_timing(get_options(),
                                      mock_client_ns->valid_after);
   /* Check that client is in the right time period point */
   tt_int_op(hs_in_period_between_tp_and_srv(mock_client_ns, 0), OP_EQ,
@@ -1590,7 +1624,7 @@ helper_set_consensus_and_system_time(networkstatus_t *ns, int position)
   } else {
     tt_assert(0);
   }
-  voting_schedule_recalculate_timing(get_options(), ns->valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns->valid_after);
 
   /* Set system time: pretend to be just 2 minutes before consensus expiry */
   real_time = ns->valid_until - 120;
diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c
index c2c556307d..dc3b598c34 100644
--- a/src/test/test_hs_config.c
+++ b/src/test/test_hs_config.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
@@ -61,8 +62,9 @@ test_invalid_service(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, 1);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceAllowUnknownPorts must be "
-                              "between 0 and 1, not 2");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceAllowUnknownPorts: Unrecognized "
+                              "value 2. Allowed values are 0 and 1.");
     teardown_capture_of_logs();
   }
 
@@ -75,8 +77,9 @@ test_invalid_service(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, 1);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceDirGroupReadable must be "
-                              "between 0 and 1, not 2");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceDirGroupReadable: "
+                              "Unrecognized value 2.");
     teardown_capture_of_logs();
   }
 
@@ -89,8 +92,9 @@ test_invalid_service(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, 1);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceMaxStreamsCloseCircuit must "
-                              "be between 0 and 1, not 2");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceMaxStreamsCloseCircuit: "
+                              "Unrecognized value 2");
     teardown_capture_of_logs();
   }
 
@@ -227,8 +231,8 @@ test_invalid_service_v2(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, validate_only);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
-                              "be between 0 and 10, not 11");
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints must "
+                              "be between 0 and 10, not 11.");
     teardown_capture_of_logs();
   }
 
@@ -242,8 +246,9 @@ test_invalid_service_v2(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, validate_only);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
-                              "be between 0 and 10, not -1");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceNumIntroductionPoints: "
+                              "Integer -1 is malformed or out of bounds.");
     teardown_capture_of_logs();
   }
 
@@ -272,6 +277,7 @@ test_valid_service_v2(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. Basic client authorization. */
   {
@@ -314,7 +320,7 @@ test_valid_service_v2(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -392,6 +398,7 @@ test_valid_service_v3(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. */
   {
@@ -448,7 +455,7 @@ test_valid_service_v3(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -489,6 +496,114 @@ test_staging_service_v3(void *arg)
   hs_free_all();
 }
 
+static void
+test_dos_parameters(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  hs_init();
+
+  /* Valid configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_INFO);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    expect_log_msg_containing("Service INTRO2 DoS defenses rate set to: 42");
+    expect_log_msg_containing("Service INTRO2 DoS defenses burst set to: 87");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid rate. Value of 2^37. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 137438953472\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceEnableIntroDoSRatePerSec: "
+                              "Integer 137438953472 is malformed or out of "
+                              "bounds.");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid burst. Value of 2^38. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 274877906944\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceEnableIntroDoSBurstPerSec: "
+                              "Integer 274877906944 is malformed or out "
+                              "of bounds.");
+    teardown_capture_of_logs();
+  }
+
+  /* Burst is smaller than rate. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 27\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Hidden service DoS defenses burst (27) can "
+                              "not be smaller than the rate value (42).");
+    teardown_capture_of_logs();
+  }
+
+  /* Negative value. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec -1\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 42\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceEnableIntroDoSRatePerSec: "
+                              "Integer -1 is malformed or out of bounds.");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  hs_free_all();
+}
+
 struct testcase_t hs_config_tests[] = {
   /* Invalid service not specific to any version. */
   { "invalid_service", test_invalid_service, TT_FORK,
@@ -512,6 +627,9 @@ struct testcase_t hs_config_tests[] = {
   { "staging_service_v3", test_staging_service_v3, TT_FORK,
     NULL, NULL },
 
+  /* Test HS DoS parameters. */
+  { "dos_parameters", test_dos_parameters, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c
index ba67712f1b..80bbf547dc 100644
--- a/src/test/test_hs_control.c
+++ b/src/test/test_hs_control.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,13 +6,21 @@
  * \brief Unit tests for hidden service control port event and command.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define HS_CLIENT_PRIVATE
 
 #include "core/or/or.h"
 #include "test/test.h"
+#include "test/test_helpers.h"
+#include "core/mainloop/connection.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_connection_st.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_control.h"
 #include "feature/nodelist/nodelist.h"
 
@@ -20,7 +28,16 @@
 #include "feature/nodelist/routerstatus_st.h"
 #include "lib/crypt_ops/crypto_format.h"
 
-#include "test/test_helpers.h"
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#ifdef _WIN32
+/* For mkdir() */
+#include <direct.h>
+#else
+#include <dirent.h>
+#endif /* defined(_WIN32) */
 
 /* mock ID digest and longname for node that's in nodelist */
 #define HSDIR_EXIST_ID \
@@ -105,8 +122,7 @@ test_hs_desc_event(void *arg)
   memset(&blinded_pk, 'B', sizeof(blinded_pk));
   memset(&hsdir_rs, 0, sizeof(hsdir_rs));
   memcpy(hsdir_rs.identity_digest, HSDIR_EXIST_ID, DIGEST_LEN);
-  ret = ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   memcpy(&ident.identity_pk, &identity_kp.pubkey,
          sizeof(ed25519_public_key_t));
   memcpy(&ident.blinded_pk, &blinded_pk, sizeof(blinded_pk));
@@ -186,9 +202,552 @@ test_hs_desc_event(void *arg)
   tor_free(expected_msg);
 }
 
+/** Test that we can correctly add, remove and view client auth credentials
+ *  using the control port. */
+static void
+test_hs_control_good_onion_client_auth_add(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  ed25519_public_key_t service_identity_pk_2fv, service_identity_pk_jt4,
+                       service_identity_pk_jam;
+  control_connection_t conn;
+  char *args = NULL;
+  char *cp1 = NULL;
+  size_t sz;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_ADD");
+  }
+
+  { /* Setup the services */
+    retval = hs_parse_address(
+                 "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd",
+                 &service_identity_pk_2fv,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = hs_parse_address(
+                 "jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd",
+                 &service_identity_pk_jt4,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = hs_parse_address(
+                 "jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd",
+                 &service_identity_pk_jam,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+  /* Register first service */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ= ");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+           "x25519:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA= DropSound=No");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd "
+           "x25519:FCV0c0ELDKKDpSFgVIB8Yow8Evj5iD+GoiTtK878NkQ= "
+           "ClientName=MeganNicole ");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  client_auths = get_hs_client_auths_map();
+  tt_assert(client_auths);
+  tt_uint_op(digest256map_size(client_auths), OP_EQ, 3);
+
+  hs_client_service_authorization_t *client_2fv =
+    digest256map_get(client_auths, service_identity_pk_2fv.pubkey);
+  tt_assert(client_2fv);
+  tt_int_op(client_2fv->flags, OP_EQ, 0);
+
+  hs_client_service_authorization_t *client_jt4 =
+    digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(client_jt4);
+  tt_int_op(client_jt4->flags, OP_EQ, 0);
+
+  hs_client_service_authorization_t *client_jam =
+    digest256map_get(client_auths, service_identity_pk_jam.pubkey);
+  tt_assert(client_jam);
+  tt_int_op(client_jam->flags, OP_EQ, 0);
+
+  /* Now let's VIEW the auth credentials */
+  tor_free(conn.current_cmd);
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_VIEW");
+
+  /* First go with no arguments, so that we view all the credentials */
+  tor_free(args);
+  args = tor_strdup("");
+
+#define VIEW_CORRECT_REPLY_NO_ADDR "250-ONION_CLIENT_AUTH_VIEW\r\n"   \
+  "250-CLIENT 2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "  \
+    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ=\r\n"   \
+  "250-CLIENT jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd " \
+    "x25519:FCV0c0ELDKKDpSFgVIB8Yow8Evj5iD+GoiTtK878NkQ= " \
+    "ClientName=MeganNicole\r\n" \
+  "250-CLIENT jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd " \
+    "x25519:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA=\r\n"             \
+  "250 OK\r\n"
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, VIEW_CORRECT_REPLY_NO_ADDR);
+  tor_free(cp1);
+
+  /* Now specify an HS addr, and see that we only view those creds */
+  tor_free(args);
+  args =
+    tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd");
+
+#define VIEW_CORRECT_REPLY_JT4 "250-ONION_CLIENT_AUTH_VIEW " \
+    "jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd\r\n"   \
+  "250-CLIENT jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd " \
+    "x25519:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA=\r\n" \
+  "250 OK\r\n"
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, VIEW_CORRECT_REPLY_JT4);
+  tor_free(cp1);
+
+  /* Now try to REMOVE the auth credentials */
+  tor_free(conn.current_cmd);
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_REMOVE");
+
+  /* First try with a wrong addr */
+  tor_free(args);
+  args = tor_strdup("thatsok");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Invalid v3 address \"thatsok\"\r\n");
+  tor_free(cp1);
+
+  client_jt4 = digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(client_jt4);
+
+  /* Now actually remove them. */
+  tor_free(args);
+  args =tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  client_jt4 = digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(!client_jt4);
+
+  /* Now try another time (we should get 'already removed' msg) */
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "251 No credentials for "
+           "\"jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd\"\r\n");
+  tor_free(cp1);
+
+  client_jt4 = digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(!client_jt4);
+
+  /* Now also remove the other one */
+  tor_free(args);
+  args =
+    tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  /* Now also remove the other one */
+  tor_free(args);
+  args =
+    tor_strdup("jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  /* Finally, do another VIEW and see that we get nothing. */
+  tor_free(conn.current_cmd);
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_VIEW");
+  tor_free(args);
+  args = tor_strdup("");
+
+#define VIEW_CORRECT_REPLY_NOTHING "250-ONION_CLIENT_AUTH_VIEW\r\n250 OK\r\n"
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, VIEW_CORRECT_REPLY_NOTHING);
+  tor_free(cp1);
+
+  /* And a final VIEW with a wrong HS addr */
+  tor_free(args);
+  args = tor_strdup("house");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Invalid v3 address \"house\"\r\n");
+
+ done:
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+  hs_client_free_all();
+}
+
+/** Test some error cases of ONION_CLIENT_AUTH_ADD */
+static void
+test_hs_control_bad_onion_client_auth_add(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  control_connection_t conn;
+  char *cp1 = NULL;
+  size_t sz;
+  char *args = NULL;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_ADD");
+  }
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+  /* Register first service */
+  args = tor_strdup(
+                "badaddr x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ=");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Invalid v3 address \"badaddr\"\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+                    "love:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA=");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "552 Unrecognized key type \"love\"\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+                    "x25519:QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEK");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Failed to decode x25519 private key\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register with an all zero client key */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+                    "x25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "553 Invalid private key \"AAAAAAAAAAAAAAAAAAAA"
+                        "AAAAAAAAAAAAAAAAAAAAAAA=\"\r\n");
+
+  client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+ done:
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+  hs_client_free_all();
+}
+
+/** Test that we can correctly add permanent client auth credentials using the
+ *  control port. */
+static void
+test_hs_control_store_permanent_creds(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  ed25519_public_key_t service_identity_pk_2fv;
+  control_connection_t conn;
+  char *args = NULL;
+  char *cp1 = NULL;
+  char *creds_file_str = NULL;
+  char *creds_fname = NULL;
+
+  size_t sz;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_ADD");
+  }
+
+  { /* Setup the services */
+    retval = hs_parse_address(
+                 "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd",
+                 &service_identity_pk_2fv,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+  /* Try registering first service with no ClientOnionAuthDir set */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ= "
+                    "Flags=Permanent");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response. This one should fail. */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "553 Unable to store creds for "
+        "\"2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd\"\r\n");
+
+  { /* Setup ClientOnionAuthDir */
+    int ret;
+    char *perm_creds_dir = tor_strdup(get_fname("permanent_credentials"));
+    get_options_mutable()->ClientOnionAuthDir = perm_creds_dir;
+
+    #ifdef _WIN32
+      ret = mkdir(perm_creds_dir);
+    #else
+      ret = mkdir(perm_creds_dir, 0700);
+    #endif
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  tor_free(args);
+  tor_free(cp1);
+
+  /* Try the control port command again. This time it should work! */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ= "
+                    "Flags=Permanent");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+
+  /* Check file contents! */
+  creds_fname = tor_strdup(get_fname("permanent_credentials/"
+     "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd.auth_private"));
+  creds_file_str = read_file_to_str(creds_fname, RFTS_BIN, NULL);
+
+  tt_assert(creds_file_str);
+  tt_str_op(creds_file_str, OP_EQ,
+         "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd:descriptor:"
+         /* This is the base32 represenation of the base64 iJ1t... key above */
+         "x25519:rcow3dfavmyanyqvhwnvnmfdqw34ydtrgv7jnelmqs4wi4uuxrca");
+
+  tor_free(args);
+  tor_free(cp1);
+
+  /* Overwrite the credentials and check that they got overwrited. */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:UDRvZLvcJo0QRLvDfkpgbtsqbkhIUQZyeo2FNBrgS18= "
+                    "Flags=Permanent");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response: we replaced! */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "251 Client for onion existed and replaced\r\n");
+
+  tor_free(creds_file_str);
+
+  /* Check creds file contents again. See that the key got updated */
+  creds_file_str = read_file_to_str(creds_fname, RFTS_BIN, NULL);
+  tt_assert(creds_file_str);
+  tt_str_op(creds_file_str, OP_EQ,
+         "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd:descriptor:"
+         /* This is the base32 represenation of the base64 UDRv... key above */
+         "x25519:ka2g6zf33qti2ecexpbx4stan3nsu3sijbiqm4t2rwctigxajnpq");
+
+  /* Now for our next act!!! Actually get the HS client subsystem to parse the
+   * whole directory and make sure that it extracted the right credential! */
+  hs_config_client_authorization(get_options(), 0);
+
+  client_auths = get_hs_client_auths_map();
+  tt_assert(client_auths);
+  tt_uint_op(digest256map_size(client_auths), OP_EQ, 1);
+
+  hs_client_service_authorization_t *client_2fv =
+    digest256map_get(client_auths, service_identity_pk_2fv.pubkey);
+  tt_assert(client_2fv);
+  tt_int_op(client_2fv->flags, OP_EQ, CLIENT_AUTH_FLAG_IS_PERMANENT);
+  tt_str_op(hex_str((char*)client_2fv->enc_seckey.secret_key, 32), OP_EQ,
+           "50346F64BBDC268D1044BBC37E4A606EDB2A6E48485106727A8D85341AE04B5F");
+
+  /* And now for the final act! Use the REMOVE control port command to remove
+     the credential, and ensure that the file has also been removed! */
+  tor_free(conn.current_cmd);
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Ensure that the creds file exists */
+  tt_int_op(file_status(creds_fname), OP_EQ, FN_FILE);
+
+  /* Do the REMOVE */
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_REMOVE");
+  args =tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+
+  /* Ensure that the file has been removed and the map is empty */
+  tt_int_op(file_status(creds_fname), OP_EQ, FN_NOENT);
+  tt_uint_op(digest256map_size(client_auths), OP_EQ, 0);
+
+ done:
+  tor_free(get_options_mutable()->ClientOnionAuthDir);
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+  tor_free(creds_fname);
+  tor_free(creds_file_str);
+  hs_client_free_all();
+}
+
+/** Test that ADD_ONION properly handles an attacker passing it a bad private
+ *  key. */
+static void
+test_hs_control_add_onion_with_bad_pubkey(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  control_connection_t conn;
+  char *args = NULL;
+  char *cp1 = NULL;
+  size_t sz;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ADD_ONION");
+  }
+
+  args = tor_strdup("ED25519-V3:AAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+                    "AAAAAAAAAAAAAAAAAAAAAAA"
+                    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+                    "Port=9735,127.0.0.1 Flags=DiscardPK");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "551 Failed to generate onion address\r\n");
+
+ done:
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+}
+
 struct testcase_t hs_control_tests[] = {
   { "hs_desc_event", test_hs_desc_event, TT_FORK,
     NULL, NULL },
+  { "hs_control_good_onion_client_auth_add",
+    test_hs_control_good_onion_client_auth_add, TT_FORK,
+    NULL, NULL },
+  { "hs_control_bad_onion_client_auth_add",
+    test_hs_control_bad_onion_client_auth_add, TT_FORK,
+    NULL, NULL },
+  { "hs_control_store_permanent_creds",
+    test_hs_control_store_permanent_creds, TT_FORK, NULL, NULL },
+  { "hs_control_add_onion_with_bad_pubkey",
+    test_hs_control_add_onion_with_bad_pubkey, TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c
index de584ed47a..782b78306c 100644
--- a/src/test/test_hs_descriptor.c
+++ b/src/test/test_hs_descriptor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "trunnel/ed25519_cert.h"
 #include "core/or/or.h"
+#include "app/config/config.h"
 #include "feature/hs/hs_descriptor.h"
 #include "test/test.h"
 #include "feature/nodelist/torcert.h"
@@ -21,21 +22,15 @@
 #include "test/hs_test_helpers.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
 #include "test_hs_descriptor.inc"
-ENABLE_GCC_WARNING(overlength-strings)
-
-/* Mock function to fill all bytes with 1 */
-static void
-mock_crypto_strongest_rand(uint8_t *out, size_t out_len)
-{
-  memset(out, 1, out_len);
-}
+ENABLE_GCC_WARNING("-Woverlength-strings")
 
 /* Test certificate encoding put in a descriptor. */
 static void
@@ -43,7 +38,6 @@ test_cert_encoding(void *arg)
 {
   int ret;
   char *encoded = NULL;
-  time_t now = time(NULL);
   ed25519_keypair_t kp;
   ed25519_public_key_t signed_key;
   ed25519_secret_key_t secret_key;
@@ -51,6 +45,10 @@ test_cert_encoding(void *arg)
 
   (void) arg;
 
+  /* Change time to 03-01-2002 23:36 UTC */
+  update_approx_time(1010101010);
+  time_t now = approx_time();
+
   ret = ed25519_keypair_generate(&kp, 0);
   tt_int_op(ret, == , 0);
   ret = ed25519_secret_key_generate(&secret_key, 0);
@@ -94,13 +92,31 @@ test_cert_encoding(void *arg)
     /* The cert did have the signing key? */
     ret= ed25519_pubkey_eq(&parsed_cert->signing_key, &kp.pubkey);
     tt_int_op(ret, OP_EQ, 1);
-    tor_cert_free(parsed_cert);
 
     /* Get to the end part of the certificate. */
     pos += b64_cert_len;
     tt_int_op(strcmpstart(pos, "-----END ED25519 CERT-----"), OP_EQ, 0);
     pos += strlen("-----END ED25519 CERT-----");
     tt_str_op(pos, OP_EQ, "");
+
+    /* Check that certificate expiry works properly and emits the right log
+       message */
+    const char *msg = "fire";
+    /* Move us forward 4 hours so that the the certificate is definitely
+       expired */
+    update_approx_time(approx_time() + 3600*4);
+    setup_full_capture_of_logs(LOG_PROTOCOL_WARN);
+    ret = cert_is_valid(parsed_cert, CERT_TYPE_SIGNING_AUTH, msg);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Since the current time at the creation of the cert was "03-01-2002
+     * 23:36", and the expiration date of the cert was two hours, the Tor code
+     * will ceiling that and make it 02:00. Make sure that the right log
+     * message is emitted */
+    expect_log_msg_containing("Invalid signature for fire: expired"
+                              " (2002-01-04 02:00:00)");
+    teardown_capture_of_logs();
+
+    tor_cert_free(parsed_cert);
   }
 
  done:
@@ -132,7 +148,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -149,7 +165,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -166,7 +182,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -179,115 +195,6 @@ test_descriptor_padding(void *arg)
 }
 
 static void
-test_link_specifier(void *arg)
-{
-  ssize_t ret;
-  hs_desc_link_specifier_t spec;
-  smartlist_t *link_specifiers = smartlist_new();
-  char buf[256];
-  char *b64 = NULL;
-  link_specifier_t *ls = NULL;
-
-  (void) arg;
-
-  /* Always this port. */
-  spec.u.ap.port = 42;
-  smartlist_add(link_specifiers, &spec);
-
-  /* Test IPv4 for starter. */
-  {
-    uint32_t ipv4;
-
-    spec.type = LS_IPV4;
-    ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4");
-    tt_int_op(ret, OP_EQ, AF_INET);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 8);
-    /* Should be 2 bytes for port and 4 bytes for IPv4. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 6);
-    ipv4 = link_specifier_get_un_ipv4_addr(ls);
-    tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), OP_EQ, ipv4);
-    tt_int_op(link_specifier_get_un_ipv4_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test IPv6. */
-  {
-    uint8_t ipv6[16];
-
-    spec.type = LS_IPV6;
-    ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]");
-    tt_int_op(ret, OP_EQ, AF_INET6);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 20);
-    /* Should be 2 bytes for port and 16 bytes for IPv6. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 18);
-    for (unsigned int i = 0; i < sizeof(ipv6); i++) {
-      ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i);
-    }
-    tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), OP_EQ, ipv6,
-              sizeof(ipv6));
-    tt_int_op(link_specifier_get_un_ipv6_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test legacy. */
-  {
-    uint8_t *id;
-
-    spec.type = LS_LEGACY_ID;
-    memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id));
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    /* 20 bytes digest + 1 byte type + 1 byte len. */
-    tt_int_op(ret, OP_EQ, 22);
-    tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN);
-    /* Digest length is 20 bytes. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN);
-    id = link_specifier_getarray_un_legacy_id(ls);
-    tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
- done:
-  link_specifier_free(ls);
-  tor_free(b64);
-  smartlist_free(link_specifiers);
-}
-
-static void
 test_encode_descriptor(void *arg)
 {
   int ret;
@@ -336,7 +243,7 @@ test_decode_descriptor(void *arg)
   hs_descriptor_t *desc = NULL;
   hs_descriptor_t *decoded = NULL;
   hs_descriptor_t *desc_no_ip = NULL;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   (void) arg;
 
@@ -345,19 +252,19 @@ test_decode_descriptor(void *arg)
   desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
 
   hs_helper_get_subcred_from_identity_keypair(&signing_kp,
-                                              subcredential);
+                                              &subcredential);
 
   /* Give some bad stuff to the decoding function. */
-  ret = hs_desc_decode_descriptor("hladfjlkjadf", subcredential,
+  ret = hs_desc_decode_descriptor("hladfjlkjadf", &subcredential,
                                   NULL, &decoded);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
 
   ret = hs_desc_encode_descriptor(desc, &signing_kp, NULL, &encoded);
-  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
   tt_assert(encoded);
 
-  ret = hs_desc_decode_descriptor(encoded, subcredential, NULL, &decoded);
-  tt_int_op(ret, OP_EQ, 0);
+  ret = hs_desc_decode_descriptor(encoded, &subcredential, NULL, &decoded);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
   tt_assert(decoded);
 
   hs_helper_desc_equal(desc, decoded);
@@ -368,7 +275,7 @@ test_decode_descriptor(void *arg)
     ret = ed25519_keypair_generate(&signing_kp_no_ip, 0);
     tt_int_op(ret, OP_EQ, 0);
     hs_helper_get_subcred_from_identity_keypair(&signing_kp_no_ip,
-                                                subcredential);
+                                                &subcredential);
     desc_no_ip = hs_helper_build_hs_desc_no_ip(&signing_kp_no_ip);
     tt_assert(desc_no_ip);
     tor_free(encoded);
@@ -377,8 +284,8 @@ test_decode_descriptor(void *arg)
     tt_int_op(ret, OP_EQ, 0);
     tt_assert(encoded);
     hs_descriptor_free(decoded);
-    ret = hs_desc_decode_descriptor(encoded, subcredential, NULL, &decoded);
-    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_desc_decode_descriptor(encoded, &subcredential, NULL, &decoded);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(decoded);
   }
 
@@ -401,14 +308,14 @@ test_decode_descriptor(void *arg)
            &auth_ephemeral_kp.pubkey, CURVE25519_PUBKEY_LEN);
 
     hs_helper_get_subcred_from_identity_keypair(&signing_kp,
-                                                subcredential);
+                                                &subcredential);
 
     /* Build and add the auth client to the descriptor. */
     clients = desc->superencrypted_data.clients;
     if (!clients) {
       clients = smartlist_new();
     }
-    hs_desc_build_authorized_client(subcredential,
+    hs_desc_build_authorized_client(&subcredential,
                                     &client_kp.pubkey,
                                     &auth_ephemeral_kp.seckey,
                                     descriptor_cookie, client);
@@ -430,23 +337,23 @@ test_decode_descriptor(void *arg)
 
     /* If we do not have the client secret key, the decoding must fail. */
     hs_descriptor_free(decoded);
-    ret = hs_desc_decode_descriptor(encoded, subcredential,
+    ret = hs_desc_decode_descriptor(encoded, &subcredential,
                                     NULL, &decoded);
-    tt_int_op(ret, OP_LT, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
     tt_assert(!decoded);
 
     /* If we have an invalid client secret key, the decoding must fail. */
     hs_descriptor_free(decoded);
-    ret = hs_desc_decode_descriptor(encoded, subcredential,
+    ret = hs_desc_decode_descriptor(encoded, &subcredential,
                                     &invalid_client_kp.seckey, &decoded);
-    tt_int_op(ret, OP_LT, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_BAD_CLIENT_AUTH);
     tt_assert(!decoded);
 
     /* If we have the client secret key, the decoding must succeed and the
      * decoded descriptor must be correct. */
-    ret = hs_desc_decode_descriptor(encoded, subcredential,
+    ret = hs_desc_decode_descriptor(encoded, &subcredential,
                                     &client_kp.seckey, &decoded);
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(decoded);
 
     hs_helper_desc_equal(desc, decoded);
@@ -682,7 +589,7 @@ test_decode_bad_signature(void *arg)
 
   setup_full_capture_of_logs(LOG_WARN);
   ret = hs_desc_decode_plaintext(HS_DESC_BAD_SIG, &desc_plaintext);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   expect_log_msg_containing("Malformed signature line. Rejecting.");
   teardown_capture_of_logs();
 
@@ -722,14 +629,14 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, bad_value, "180", "42", "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Missing fields. */
   {
     const char *plaintext = "hs-descriptor 3\n";
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Max length. */
@@ -742,7 +649,7 @@ test_decode_plaintext(void *arg)
     plaintext[big - 1] = '\0';
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Bad lifetime value. */
@@ -751,7 +658,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", bad_value, "42", "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Huge lifetime value. */
@@ -760,7 +667,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", "7181615", "42", "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Invalid encrypted section. */
@@ -769,7 +676,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", "180", "42", bad_value);
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Invalid revision counter. */
@@ -778,7 +685,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", "180", bad_value, "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
  done:
@@ -848,8 +755,7 @@ test_desc_signature(void *arg)
   ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data),
                               "Tor onion service descriptor sig v3", &kp);
   tt_int_op(ret, OP_EQ, 0);
-  ret = ed25519_signature_to_base64(sig_b64, &sig);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_signature_to_base64(sig_b64, &sig);
   /* Build the descriptor that should be valid. */
   tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64);
   ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc));
@@ -878,7 +784,7 @@ test_build_authorized_client(void *arg)
     "07d087f1d8c68393721f6e70316d3b29";
   const char client_pubkey_b16[] =
     "8c1298fa6050e372f8598f6deca32e27b0ad457741422c2629ebb132cf7fae37";
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
   char *mem_op_hex_tmp=NULL;
 
   (void) arg;
@@ -890,7 +796,7 @@ test_build_authorized_client(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   curve25519_public_key_generate(&client_auth_pk, &client_auth_sk);
 
-  memset(subcredential, 42, sizeof(subcredential));
+  memset(subcredential.subcred, 42, sizeof(subcredential));
 
   desc_client = tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
 
@@ -909,9 +815,9 @@ test_build_authorized_client(void *arg)
                 client_pubkey_b16,
                 strlen(client_pubkey_b16));
 
-  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+  testing_enable_prefilled_rng("\x01", 1);
 
-  hs_desc_build_authorized_client(subcredential,
+  hs_desc_build_authorized_client(&subcredential,
                                   &client_auth_pk, &auth_ephemeral_sk,
                                   descriptor_cookie, desc_client);
 
@@ -925,15 +831,13 @@ test_build_authorized_client(void *arg)
  done:
   tor_free(desc_client);
   tor_free(mem_op_hex_tmp);
-  UNMOCK(crypto_strongest_rand_);
+  testing_disable_prefilled_rng();
 }
 
 struct testcase_t hs_descriptor[] = {
   /* Encoding tests. */
   { "cert_encoding", test_cert_encoding, TT_FORK,
     NULL, NULL },
-  { "link_specifier", test_link_specifier, TT_FORK,
-    NULL, NULL },
   { "encode_descriptor", test_encode_descriptor, TT_FORK,
     NULL, NULL },
   { "descriptor_padding", test_descriptor_padding, TT_FORK,
diff --git a/src/test/test_hs_dos.c b/src/test/test_hs_dos.c
new file mode 100644
index 0000000000..642513efce
--- /dev/null
+++ b/src/test/test_hs_dos.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cell.c
+ * \brief Test hidden service cell functionality.
+ */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define HS_DOS_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+
+#include "feature/hs/hs_dos.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/nodelist/networkstatus.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_ns_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  current_ns_consensus->net_params = smartlist_new();
+  smartlist_add(current_ns_consensus->net_params,
+                (void *) "HiddenServiceEnableIntroDoSDefense=1");
+  hs_dos_consensus_has_changed(current_ns_consensus);
+}
+
+static void
+free_mock_consensus(void)
+{
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_can_send_intro2(void *arg)
+{
+  uint32_t now = (uint32_t) approx_time();
+  or_circuit_t *or_circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  hs_dos_init();
+
+  get_options_mutable()->ORPort_set = 1;
+  setup_mock_consensus();
+
+  or_circ =  or_circuit_new(1, NULL);
+
+  /* Make that circuit a service intro point. */
+  circuit_change_purpose(TO_CIRCUIT(or_circ), CIRCUIT_PURPOSE_INTRO_POINT);
+  hs_dos_setup_default_intro2_defenses(or_circ);
+  or_circ->introduce2_dos_defense_enabled = 1;
+
+  /* Brand new circuit, we should be able to send INTRODUCE2 cells. */
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+
+  /* Simulate that 10 cells have arrived in 1 second. There should be no
+   * refill since the bucket is already at maximum on the first cell. */
+  update_approx_time(++now);
+  for (int i = 0; i < 10; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 10);
+
+  /* Fully refill the bucket minus 1 cell. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Receive an INTRODUCE2 at each second. We should have the bucket full
+   * since at every second it gets refilled. */
+  for (int i = 0; i < 10; i++) {
+    update_approx_time(++now);
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  /* Last check if we can send the cell decrements the bucket so minus 1. */
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Manually reset bucket for next test. */
+  token_bucket_ctr_reset(&or_circ->introduce2_bucket, now);
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL));
+
+  /* Do a full burst in the current second which should empty the bucket and
+   * we shouldn't be allowed to send one more cell after that. We go minus 1
+   * cell else the very last check if we can send the INTRO2 cell returns
+   * false because the bucket goes down to 0. */
+  for (uint32_t i = 0; i < get_intro2_burst_consensus_param(NULL) - 1; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 1);
+  /* Get the last remaining cell, we shouldn't be allowed to send it. */
+  tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+
+  /* Make sure the next 100 cells aren't allowed and bucket stays at 0. */
+  for (int i = 0; i < 100; i++) {
+    tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+    tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+  }
+
+  /* One second has passed, we should have the rate minus 1 cell added. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_rate_consensus_param(NULL) - 1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(or_circ));
+
+  hs_free_all();
+  free_mock_consensus();
+}
+
+static void
+test_validate_dos_extension_params(void *arg)
+{
+  bool ret;
+
+  (void) arg;
+
+  /* Validate the default values. */
+  ret = cell_dos_extension_parameters_are_valid(
+                                      get_intro2_rate_consensus_param(NULL),
+                                      get_intro2_burst_consensus_param(NULL));
+  tt_assert(ret);
+
+  /* Valid custom rate/burst. */
+  ret = cell_dos_extension_parameters_are_valid(17, 42);
+  tt_assert(ret);
+  ret = cell_dos_extension_parameters_are_valid(INT32_MAX, INT32_MAX);
+  tt_assert(ret);
+
+  /* Invalid rate. */
+  ret = cell_dos_extension_parameters_are_valid(UINT64_MAX, 42);
+  tt_assert(!ret);
+
+  /* Invalid burst. */
+  ret = cell_dos_extension_parameters_are_valid(42, UINT64_MAX);
+  tt_assert(!ret);
+
+  /* Value of 0 is valid (but should disable defenses) */
+  ret = cell_dos_extension_parameters_are_valid(0, 0);
+  tt_assert(ret);
+
+  /* Can't have burst smaller than rate. */
+  ret = cell_dos_extension_parameters_are_valid(42, 40);
+  tt_assert(!ret);
+
+ done:
+  return;
+}
+
+struct testcase_t hs_dos_tests[] = {
+  { "can_send_intro2", test_can_send_intro2, TT_FORK,
+    NULL, NULL },
+  { "validate_dos_extension_params", test_validate_dos_extension_params,
+    TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c
index 558fc32c54..e6b27d7a50 100644
--- a/src/test/test_hs_intropoint.c
+++ b/src/test/test_hs_intropoint.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 
 #include "core/or/or.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "ht.h"
@@ -25,6 +26,8 @@
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 
@@ -43,6 +46,9 @@ new_establish_intro_cell(const char *circ_nonce,
   uint8_t buf[RELAY_PAYLOAD_SIZE] = {0};
   trn_cell_establish_intro_t *cell = NULL;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Ensure that *cell_out is NULL such that we can use to check if we need to
    * free `cell` in case of an error. */
@@ -50,9 +56,9 @@ new_establish_intro_cell(const char *circ_nonce,
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
   tt_i64_op(cell_len, OP_GT, 0);
 
   cell_len = trn_cell_establish_intro_parse(&cell, buf, sizeof(buf));
@@ -73,12 +79,15 @@ new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out)
 {
   ssize_t cell_len = 0;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell_out);
   tt_i64_op(cell_len, OP_GT, 0);
 
  done:
@@ -118,6 +127,8 @@ helper_create_intro_circuit(void)
   or_circuit_t *circ = or_circuit_new(0, NULL);
   tt_assert(circ);
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+  token_bucket_ctr_init(&circ->introduce2_bucket, 100, 100,
+                        (uint32_t) approx_time());
  done:
   return circ;
 }
@@ -693,6 +704,17 @@ test_introduce1_suitable_circuit(void *arg)
     tt_int_op(ret, OP_EQ, 0);
   }
 
+  /* Single hop circuit should not be allowed. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+    circ->p_chan->is_client = 1;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    tor_free(circ->p_chan);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
  done:
   ;
 }
@@ -735,12 +757,15 @@ test_introduce1_validation(void *arg)
   cell = helper_create_introduce1_cell();
   tt_assert(cell);
 
+#ifndef ALL_BUGS_ARE_FATAL
   /* It should NOT be a legacy cell which will trigger a BUG(). */
   memset(cell->legacy_key_id, 'a', sizeof(cell->legacy_key_id));
   tor_capture_bugs_(1);
   ret = validate_introduce1_parsed_cell(cell);
   tor_end_capture_bugs_();
   tt_int_op(ret, OP_EQ, -1);
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
   /* Reset legacy ID and make sure it's correct. */
   memset(cell->legacy_key_id, 0, sizeof(cell->legacy_key_id));
   ret = validate_introduce1_parsed_cell(cell);
@@ -888,43 +913,213 @@ test_received_introduce1_handling(void *arg)
   UNMOCK(relay_send_command_from_edge_);
 }
 
+static void
+test_received_establish_intro_dos_ext(void *arg)
+{
+  int ret;
+  ssize_t cell_len = 0;
+  uint8_t cell[RELAY_PAYLOAD_SIZE] = {0};
+  char circ_nonce[DIGEST_LEN] = {0};
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  hs_circuitmap_init();
+
+  /* Setup. */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+  memset(&config, 0, sizeof(config));
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 13;
+  config.intro_dos_burst_per_sec = 42;
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+  /* The INTRO2 bucket should be 0 at this point. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 0);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 0);
+
+  /* Case 1: Build encoded cell. Usable DoS parameters. */
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be set to the burst value. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 42);
+  /* Validate the config of the intro2 bucket. */
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 13);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 42);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 1);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 2: Build encoded cell. Bad DoS parameters. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = UINT_MAX;
+  config.intro_dos_burst_per_sec = 13;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 3: Build encoded cell. Burst is smaller than rate. Not allowed. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 87;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 4: Build encoded cell. Rate is 0 but burst is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 0;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 5: Build encoded cell. Burst is 0 but rate is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 45;
+  config.intro_dos_burst_per_sec = 0;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  service_intro_point_free(ip);
+  hs_circuitmap_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+static void *
+hs_subsystem_setup_fn(const struct testcase_t *tc)
+{
+  (void) tc;
+
+  return NULL;
+}
+
+static int
+hs_subsystem_cleanup_fn(const struct testcase_t *tc, void *arg)
+{
+  (void) tc;
+  (void) arg;
+
+  return 1;
+}
+
+static struct testcase_setup_t test_setup = {
+  hs_subsystem_setup_fn, hs_subsystem_cleanup_fn
+};
+
 struct testcase_t hs_intropoint_tests[] = {
   { "intro_point_registration",
-    test_intro_point_registration, TT_FORK, NULL, NULL },
+    test_intro_point_registration, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype",
-    test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype2",
-    test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype2, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_purpose",
-    test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_purpose, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig",
-    test_establish_intro_wrong_sig, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig_len",
-    test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_auth_key_len",
-    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_mac",
-    test_establish_intro_wrong_mac, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_mac, TT_FORK, NULL, &test_setup},
 
   { "introduce1_suitable_circuit",
-    test_introduce1_suitable_circuit, TT_FORK, NULL, NULL },
+    test_introduce1_suitable_circuit, TT_FORK, NULL, &test_setup},
 
   { "introduce1_is_legacy",
-    test_introduce1_is_legacy, TT_FORK, NULL, NULL },
+    test_introduce1_is_legacy, TT_FORK, NULL, &test_setup},
 
   { "introduce1_validation",
-    test_introduce1_validation, TT_FORK, NULL, NULL },
+    test_introduce1_validation, TT_FORK, NULL, &test_setup},
 
   { "received_introduce1_handling",
-    test_received_introduce1_handling, TT_FORK, NULL, NULL },
+    test_received_introduce1_handling, TT_FORK, NULL, &test_setup},
+
+  { "received_establish_intro_dos_ext",
+    test_received_establish_intro_dos_ext, TT_FORK, NULL, &test_setup},
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_ntor.c b/src/test/test_hs_ntor.c
index 1c694e6040..7867740a1a 100644
--- a/src/test/test_hs_ntor.c
+++ b/src/test/test_hs_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,7 +23,7 @@ test_hs_ntor(void *arg)
 {
   int retval;
 
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   ed25519_keypair_t service_intro_auth_keypair;
   curve25519_keypair_t service_intro_enc_keypair;
@@ -42,7 +42,7 @@ test_hs_ntor(void *arg)
   /* Generate fake data for this unittest */
   {
     /* Generate fake subcredential */
-    memset(subcredential, 'Z', DIGEST256_LEN);
+    memset(subcredential.subcred, 'Z', DIGEST256_LEN);
 
     /* service */
     curve25519_keypair_generate(&service_intro_enc_keypair, 0);
@@ -57,7 +57,7 @@ test_hs_ntor(void *arg)
     hs_ntor_client_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
                                        &service_intro_enc_keypair.pubkey,
                                        &client_ephemeral_enc_keypair,
-                                       subcredential,
+                                       &subcredential,
                                        &client_hs_ntor_intro_cell_keys);
   tt_int_op(retval, OP_EQ, 0);
 
@@ -66,7 +66,7 @@ test_hs_ntor(void *arg)
     hs_ntor_service_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
                                         &service_intro_enc_keypair,
                                         &client_ephemeral_enc_keypair.pubkey,
-                                        subcredential,
+                                        &subcredential,
                                         &service_hs_ntor_intro_cell_keys);
   tt_int_op(retval, OP_EQ, 0);
 
diff --git a/src/test/test_hs_ntor.sh b/src/test/test_hs_ntor.sh
index 8a0003d44a..ee7141cc9a 100755
--- a/src/test/test_hs_ntor.sh
+++ b/src/test/test_hs_ntor.sh
@@ -3,7 +3,7 @@
 
 exitcode=0
 
-# Run the python integration test sand return the exitcode of the python
+# Run the python integration tests and return the exitcode of the python
 # script.  The python script might ask the testsuite to skip it if not all
 # python dependencies are covered.
 "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/hs_ntor_ref.py" || exitcode=$?
diff --git a/src/test/test_hs_ntor_cl.c b/src/test/test_hs_ntor_cl.c
index 6341b96d84..3acd7ef0bc 100644
--- a/src/test/test_hs_ntor_cl.c
+++ b/src/test/test_hs_ntor_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /** This is a wrapper over the little-t-tor HS ntor functions. The wrapper is
@@ -53,7 +53,7 @@ client1(int argc, char **argv)
   curve25519_public_key_t intro_enc_pubkey;
   ed25519_public_key_t intro_auth_pubkey;
   curve25519_keypair_t client_ephemeral_enc_keypair;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   /* Output */
   hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
@@ -65,7 +65,7 @@ client1(int argc, char **argv)
   BASE16(3, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
   BASE16(4, client_ephemeral_enc_keypair.seckey.secret_key,
          CURVE25519_SECKEY_LEN);
-  BASE16(5, subcredential, DIGEST256_LEN);
+  BASE16(5, subcredential.subcred, DIGEST256_LEN);
 
   /* Generate keypair */
   curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
@@ -74,7 +74,7 @@ client1(int argc, char **argv)
   retval = hs_ntor_client_get_introduce1_keys(&intro_auth_pubkey,
                                               &intro_enc_pubkey,
                                               &client_ephemeral_enc_keypair,
-                                              subcredential,
+                                              &subcredential,
                                               &hs_ntor_intro_cell_keys);
   if (retval < 0) {
     goto done;
@@ -106,7 +106,7 @@ server1(int argc, char **argv)
   curve25519_keypair_t intro_enc_keypair;
   ed25519_public_key_t intro_auth_pubkey;
   curve25519_public_key_t client_ephemeral_enc_pubkey;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   /* Output */
   hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
@@ -119,7 +119,7 @@ server1(int argc, char **argv)
   BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
   BASE16(3, intro_enc_keypair.seckey.secret_key, CURVE25519_SECKEY_LEN);
   BASE16(4, client_ephemeral_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
-  BASE16(5, subcredential, DIGEST256_LEN);
+  BASE16(5, subcredential.subcred, DIGEST256_LEN);
 
   /* Generate keypair */
   curve25519_public_key_generate(&intro_enc_keypair.pubkey,
@@ -130,7 +130,7 @@ server1(int argc, char **argv)
   retval = hs_ntor_service_get_introduce1_keys(&intro_auth_pubkey,
                                                &intro_enc_keypair,
                                                &client_ephemeral_enc_pubkey,
-                                               subcredential,
+                                               &subcredential,
                                                &hs_ntor_intro_cell_keys);
   if (retval < 0) {
     goto done;
@@ -188,7 +188,7 @@ client2(int argc, char **argv)
   ed25519_public_key_t intro_auth_pubkey;
   curve25519_keypair_t client_ephemeral_enc_keypair;
   curve25519_public_key_t service_ephemeral_rend_pubkey;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   /* Output */
   hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
@@ -201,7 +201,7 @@ client2(int argc, char **argv)
          CURVE25519_SECKEY_LEN);
   BASE16(4, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
   BASE16(5, service_ephemeral_rend_pubkey.public_key, CURVE25519_PUBKEY_LEN);
-  BASE16(6, subcredential, DIGEST256_LEN);
+  BASE16(6, subcredential.subcred, DIGEST256_LEN);
 
   /* Generate keypair */
   curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
diff --git a/src/test/test_hs_ob.c b/src/test/test_hs_ob.c
new file mode 100644
index 0000000000..7f40187b5f
--- /dev/null
+++ b/src/test/test_hs_ob.c
@@ -0,0 +1,268 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_ob.c
+ * \brief Test hidden service onion balance functionality.
+ */
+
+#define CONFIG_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define HS_OB_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "app/config/config.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_ob.h"
+#include "feature/hs/hs_service.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+static ed25519_keypair_t onion_addr_kp_1;
+static char onion_addr_1[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+static ed25519_keypair_t onion_addr_kp_2;
+static char onion_addr_2[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+static bool config_is_good = true;
+
+static int
+helper_tor_config(const char *conf)
+{
+  int ret = -1;
+  or_options_t *options = helper_parse_options(conf);
+  tt_assert(options);
+  ret = hs_config_service_all(options, 0);
+ done:
+  or_options_free(options);
+  return ret;
+}
+
+static networkstatus_t mock_ns;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+static char *
+mock_read_file_to_str(const char *filename, int flags, struct stat *stat_out)
+{
+  char *ret = NULL;
+
+  (void) flags;
+  (void) stat_out;
+
+  if (!strcmp(filename, get_fname("hs3" PATH_SEPARATOR "ob_config"))) {
+    if (config_is_good) {
+      tor_asprintf(&ret, "MasterOnionAddress %s.onion\n"
+                         "MasterOnionAddress %s.onion\n",
+                         onion_addr_1, onion_addr_2);
+    } else {
+      tor_asprintf(&ret, "MasterOnionAddress JUNKJUNKJUNK.onion\n"
+                         "UnknownOption BLAH\n");
+    }
+    goto done;
+  }
+
+ done:
+  return ret;
+}
+
+static void
+test_parse_config_file(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  const ed25519_public_key_t *pkey;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(read_file_to_str, mock_read_file_to_str);
+
+#define fmt_conf            \
+  "HiddenServiceDir %s\n"   \
+  "HiddenServicePort 22\n"  \
+  "HiddenServiceOnionBalanceInstance 1\n"
+  tor_asprintf(&conf, fmt_conf, get_fname("hs3"));
+#undef fmt_conf
+
+  /* Build the OB frontend onion addresses. */
+  ed25519_keypair_generate(&onion_addr_kp_1, 0);
+  hs_build_address(&onion_addr_kp_1.pubkey, HS_VERSION_THREE, onion_addr_1);
+  ed25519_keypair_generate(&onion_addr_kp_2, 0);
+  hs_build_address(&onion_addr_kp_2.pubkey, HS_VERSION_THREE, onion_addr_2);
+
+  ret = helper_tor_config(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Load the keys for the service. After that, the v3 service should be
+   * registered in the global map and we'll be able to access it. */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+  hs_service_load_all_keys();
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  const hs_service_t *s = get_first_service();
+  tt_assert(s);
+  tt_assert(s->config.ob_master_pubkeys);
+  tt_assert(hs_ob_service_is_instance(s));
+  tt_assert(smartlist_len(s->config.ob_master_pubkeys) == 2);
+
+  /* Test the public keys we've added. */
+  pkey = smartlist_get(s->config.ob_master_pubkeys, 0);
+  tt_mem_op(&onion_addr_kp_1.pubkey, OP_EQ, pkey, ED25519_PUBKEY_LEN);
+  pkey = smartlist_get(s->config.ob_master_pubkeys, 1);
+  tt_mem_op(&onion_addr_kp_2.pubkey, OP_EQ, pkey, ED25519_PUBKEY_LEN);
+
+ done:
+  hs_free_all();
+
+  UNMOCK(read_file_to_str);
+}
+
+static void
+test_parse_config_file_bad(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(read_file_to_str, mock_read_file_to_str);
+
+  /* Indicate mock_read_file_to_str() to use the bad config. */
+  config_is_good = false;
+
+#define fmt_conf            \
+  "HiddenServiceDir %s\n"   \
+  "HiddenServicePort 22\n"  \
+  "HiddenServiceOnionBalanceInstance 1\n"
+  tor_asprintf(&conf, fmt_conf, get_fname("hs3"));
+#undef fmt_conf
+
+  setup_full_capture_of_logs(LOG_INFO);
+  ret = helper_tor_config(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("OnionBalance: MasterOnionAddress "
+                            "JUNKJUNKJUNK.onion is invalid");
+  expect_log_msg_containing("Found unrecognized option \'UnknownOption\'; "
+                            "saving it.");
+  teardown_capture_of_logs();
+
+ done:
+  hs_free_all();
+
+  UNMOCK(read_file_to_str);
+}
+
+static void
+test_get_subcredentials(void *arg)
+{
+  int ret;
+  hs_service_t *service = NULL;
+  hs_service_config_t config;
+  hs_subcredential_t *subcreds = NULL;
+
+  (void) arg;
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Setup consensus with proper time so we can compute the time period. */
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  config.ob_master_pubkeys = smartlist_new();
+  tt_assert(config.ob_master_pubkeys);
+
+  /* Set up an instance */
+  service = tor_malloc_zero(sizeof(hs_service_t));
+  service->config = config;
+  /* Setup the service descriptors */
+  service->desc_current = service_descriptor_new();
+  service->desc_next = service_descriptor_new();
+
+  /* First try to compute subcredentials but with no OB keys. Make sure that
+   * subcreds get NULLed. To do this check we first poison subcreds. */
+  subcreds = (void*)999;
+  tt_ptr_op(subcreds, OP_NE, NULL);
+  size_t num = compute_subcredentials(service, &subcreds);
+  tt_ptr_op(subcreds, OP_EQ, NULL);
+
+  /* Generate a keypair to add to the OB keys list. */
+  ed25519_keypair_generate(&onion_addr_kp_1, 0);
+  smartlist_add(config.ob_master_pubkeys, &onion_addr_kp_1.pubkey);
+
+  /* Set up the instance subcredentials */
+  char current_subcred[SUBCRED_LEN];
+  char next_subcred[SUBCRED_LEN];
+  memset(current_subcred, 'C', SUBCRED_LEN);
+  memset(next_subcred, 'N', SUBCRED_LEN);
+  memcpy(service->desc_current->desc->subcredential.subcred, current_subcred,
+         SUBCRED_LEN);
+  memcpy(service->desc_next->desc->subcredential.subcred, next_subcred,
+         SUBCRED_LEN);
+
+  /* See that subcreds are computed properly */
+  num = compute_subcredentials(service, &subcreds);
+  /* 5 subcredentials: 3 for the frontend, 2 for the instance */
+  tt_uint_op(num, OP_EQ, 5);
+  tt_ptr_op(subcreds, OP_NE, NULL);
+
+  /* Validate the subcredentials we just got. We'll build them oursevles with
+   * the right time period steps and compare. */
+  const uint64_t tp = hs_get_time_period_num(0);
+  const int steps[3] = {0, -1, 1};
+
+  unsigned int i;
+  for (i = 0; i < 3; i++) {
+    hs_subcredential_t subcredential;
+    ed25519_public_key_t blinded_pubkey;
+    hs_build_blinded_pubkey(&onion_addr_kp_1.pubkey, NULL, 0, tp + steps[i],
+                            &blinded_pubkey);
+    hs_get_subcredential(&onion_addr_kp_1.pubkey, &blinded_pubkey,
+                         &subcredential);
+    tt_mem_op(subcreds[i].subcred, OP_EQ, subcredential.subcred,
+              SUBCRED_LEN);
+  }
+
+  tt_mem_op(subcreds[i++].subcred, OP_EQ, current_subcred, SUBCRED_LEN);
+  tt_mem_op(subcreds[i++].subcred, OP_EQ, next_subcred, SUBCRED_LEN);
+
+ done:
+  tor_free(subcreds);
+
+  smartlist_free(config.ob_master_pubkeys);
+  if (service) {
+    memset(&service->config, 0, sizeof(hs_service_config_t));
+    hs_service_free(service);
+  }
+
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
+struct testcase_t hs_ob_tests[] = {
+  { "parse_config_file", test_parse_config_file, TT_FORK,
+    NULL, NULL },
+  { "parse_config_file_bad", test_parse_config_file_bad, TT_FORK,
+    NULL, NULL },
+
+  { "get_subcredentials", test_get_subcredentials, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c
index c60ab6c930..8b94bb6cf1 100644
--- a/src/test/test_hs_service.c
+++ b/src/test/test_hs_service.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,8 +19,9 @@
 #define MAINLOOP_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define STATEFILE_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define HS_CLIENT_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -43,13 +44,15 @@
 #include "core/or/versions.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/shared_random_state.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_ob.h"
+#include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 #include "feature/nodelist/networkstatus.h"
@@ -60,6 +63,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -86,6 +90,14 @@ mock_networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
   return &mock_ns;
 }
 
+static networkstatus_t *
+mock_networkstatus_get_reasonably_live_consensus_null(time_t now, int flavor)
+{
+  (void) now;
+  (void) flavor;
+  return NULL;
+}
+
 static or_state_t *dummy_state = NULL;
 
 /* Mock function to get fake or state (used for rev counters) */
@@ -108,6 +120,9 @@ mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
   return;
 }
 
+static size_t relay_payload_len;
+static char relay_payload[RELAY_PAYLOAD_SIZE];
+
 static int
 mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
                                   uint8_t relay_command, const char *payload,
@@ -123,6 +138,24 @@ mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
   (void) cpath_layer;
   (void) filename;
   (void) lineno;
+
+  memcpy(relay_payload, payload, payload_len);
+  relay_payload_len = payload_len;
+
+  return 0;
+}
+
+static unsigned int num_intro_points = 0;
+static unsigned int
+mock_count_desc_circuit_established(const hs_service_descriptor_t *desc)
+{
+  (void) desc;
+  return num_intro_points;
+}
+
+static int
+mock_router_have_minimum_dir_info_false(void)
+{
   return 0;
 }
 
@@ -170,8 +203,7 @@ test_e2e_rend_circuit_setup(void *arg)
     tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
     tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
 
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
 
     TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
   }
@@ -183,9 +215,8 @@ test_e2e_rend_circuit_setup(void *arg)
   /* Setup the circuit: do the ntor key exchange */
   {
     uint8_t ntor_key_seed[DIGEST256_LEN] = {2};
-    retval = hs_circuit_setup_e2e_rend_circ(or_circ,
-                                       ntor_key_seed, sizeof(ntor_key_seed),
-                                       1);
+    retval = hs_circuit_setup_e2e_rend_circ(or_circ, ntor_key_seed,
+                                            sizeof(ntor_key_seed), 1);
     tt_int_op(retval, OP_EQ, 0);
   }
 
@@ -194,12 +225,12 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
@@ -268,7 +299,7 @@ helper_clone_authorized_client(const hs_service_authorized_client_t *client)
 
 /* Helper: Return a newly allocated service object with the identity keypair
  * sets and the current descriptor. Then register it to the global map.
- * Caller should us hs_free_all() to free this service or remove it from the
+ * Caller should use hs_free_all() to free this service or remove it from the
  * global map before freeing. */
 static hs_service_t *
 helper_create_service(void)
@@ -290,6 +321,20 @@ helper_create_service(void)
   return service;
 }
 
+/* Helper: Deallocate a given service object, its child objects and
+ * remove it from onion service map.
+ * */
+static void
+helper_destroy_service(hs_service_t *service)
+{
+  if (!service)
+    return;
+
+  remove_service(get_hs_service_map(), service);
+
+  hs_service_free(service);
+}
+
 /* Helper: Return a newly allocated service object with clients. */
 static hs_service_t *
 helper_create_service_with_clients(int num_clients)
@@ -315,17 +360,18 @@ helper_create_service_with_clients(int num_clients)
 static hs_service_intro_point_t *
 helper_create_service_ip(void)
 {
-  hs_desc_link_specifier_t *ls;
-  hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+  link_specifier_t *ls;
+  hs_service_intro_point_t *ip = service_intro_point_new(NULL);
   tor_assert(ip);
   /* Add a first unused link specifier. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_IPV4;
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
   smartlist_add(ip->base.link_specifiers, ls);
   /* Add a second link specifier used by a test. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_LEGACY_ID;
-  memset(ls->u.legacy_id, 'A', sizeof(ls->u.legacy_id));
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memset(link_specifier_getarray_un_legacy_id(ls), 'A',
+         link_specifier_getlen_un_legacy_id(ls));
   smartlist_add(ip->base.link_specifiers, ls);
 
   return ip;
@@ -379,11 +425,11 @@ test_load_keys(void *arg)
   tt_assert(s);
 
   /* Ok we have the service object. Validate few things. */
-  tt_assert(!tor_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
+  tt_assert(!fast_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
   tt_int_op(hs_address_is_valid(s->onion_address), OP_EQ, 1);
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_sk.seckey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_sk.seckey,
                              ED25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_pk.pubkey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_pk.pubkey,
                              ED25519_PUBKEY_LEN));
   /* Check onion address from identity key. */
   hs_build_address(&s->keys.identity_pk, s->config.version, addr);
@@ -657,13 +703,15 @@ test_service_intro_point(void *arg)
 
   (void) arg;
 
+  update_approx_time(1481621834);
+
   /* Test simple creation of an object. */
   {
-    time_t now = time(NULL);
+    time_t now = approx_time();
     ip = helper_create_service_ip();
     tt_assert(ip);
     /* Make sure the authentication keypair is not zeroes. */
-    tt_int_op(tor_mem_is_zero((const char *) &ip->auth_key_kp,
+    tt_int_op(fast_mem_is_zero((const char *) &ip->auth_key_kp,
                               sizeof(ed25519_keypair_t)), OP_EQ, 0);
     /* The introduce2_max MUST be in that range. */
     tt_u64_op(ip->introduce2_max, OP_GE,
@@ -798,10 +846,11 @@ test_helper_functions(void *arg)
     const node_t *node = get_node_from_intro_point(ip);
     tt_ptr_op(node, OP_EQ, &mock_node);
     SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                            hs_desc_link_specifier_t *, ls) {
-      if (ls->type == LS_LEGACY_ID) {
+                            link_specifier_t *, ls) {
+      if (link_specifier_get_ls_type(ls) == LS_LEGACY_ID) {
         /* Change legacy id in link specifier which is not the mock node. */
-        memset(ls->u.legacy_id, 'B', sizeof(ls->u.legacy_id));
+        memset(link_specifier_getarray_un_legacy_id(ls), 'B',
+               link_specifier_getlen_un_legacy_id(ls));
       }
     } SMARTLIST_FOREACH_END(ls);
     node = get_node_from_intro_point(ip);
@@ -859,6 +908,10 @@ test_helper_functions(void *arg)
 
  done:
   /* This will free the service and all objects associated to it. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_service_free_all();
   UNMOCK(node_get_by_id);
 }
@@ -868,7 +921,7 @@ static void
 test_intro_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   origin_circuit_t *circ = NULL;
 
   (void) arg;
@@ -916,6 +969,10 @@ test_intro_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -930,7 +987,7 @@ test_intro_established(void *arg)
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -985,12 +1042,15 @@ test_intro_established(void *arg)
   /* Send an empty payload. INTRO_ESTABLISHED cells are basically zeroes. */
   ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
   tt_int_op(ret, OP_EQ, 0);
-  tt_u64_op(ip->circuit_established, OP_EQ, 1);
   tt_int_op(TO_CIRCUIT(circ)->purpose, OP_EQ, CIRCUIT_PURPOSE_S_INTRO);
 
  done:
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1002,7 +1062,7 @@ test_rdv_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
 
   (void) arg;
 
@@ -1033,6 +1093,10 @@ test_rdv_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -1070,8 +1134,7 @@ test_closing_intro_circs(void *arg)
 
   /* Initialize intro circuit */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Register circuit in the circuitmap . */
   hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
                                                     &ip->auth_key_kp.pubkey);
@@ -1097,8 +1160,7 @@ test_closing_intro_circs(void *arg)
   /* Now pretend that a new intro point circ was launched and opened. Check
    * that the intro point will be established correctly. */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&intro_circ->hs_ident->intro_auth_pk,
                       &ip->auth_key_kp.pubkey);
   /* Register circuit in the circuitmap . */
@@ -1120,19 +1182,23 @@ test_closing_intro_circs(void *arg)
     circuit_free_(TO_CIRCUIT(intro_circ));
   }
   /* Frees the service object. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(assert_circuit_ok);
 }
 
 /** Test sending and receiving introduce2 cells */
 static void
-test_introduce2(void *arg)
+test_bad_introduce2(void *arg)
 {
   int ret;
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -1142,7 +1208,7 @@ test_introduce2(void *arg)
   MOCK(get_or_state,
        get_or_state_replacement);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
   tt_assert(circ);
@@ -1199,6 +1265,10 @@ test_introduce2(void *arg)
   dummy_state = NULL;
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1222,6 +1292,7 @@ test_service_event(void *arg)
 
   /* Set a service for this circuit. */
   service = helper_create_service();
+  tt_assert(service);
   ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
                       &service->keys.identity_pk);
 
@@ -1253,7 +1324,6 @@ test_service_event(void *arg)
      * descriptor map so we can retry it. */
     ip = helper_create_service_ip();
     service_intro_point_add(service->desc_current->intro_points.map, ip);
-    ip->circuit_established = 1;  /* We'll test that, it MUST be 0 after. */
     run_housekeeping_event(now);
     tt_int_op(digest256map_size(service->desc_current->intro_points.map),
               OP_EQ, 1);
@@ -1283,6 +1353,10 @@ test_service_event(void *arg)
  done:
   hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1293,12 +1367,12 @@ test_rotate_descriptors(void *arg)
 {
   int ret;
   time_t next_rotation_time, now;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_descriptor_t *desc_next;
 
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   hs_init();
   MOCK(get_or_state, get_or_state_replacement);
@@ -1314,7 +1388,7 @@ test_rotate_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
   now = mock_ns.valid_after+1;
@@ -1355,7 +1429,7 @@ test_rotate_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
   now = mock_ns.valid_after+1;
@@ -1385,6 +1459,10 @@ test_rotate_descriptors(void *arg)
   tt_assert(service->desc_next);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(get_or_state);
   UNMOCK(circuit_mark_for_close_);
@@ -1397,9 +1475,8 @@ static void
 test_build_update_descriptors(void *arg)
 {
   int ret;
-  time_t now = time(NULL);
   node_t *node;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip_cur, *ip_next;
   routerinfo_t ri;
 
@@ -1412,7 +1489,7 @@ test_build_update_descriptors(void *arg)
   MOCK(networkstatus_get_reasonably_live_consensus,
        mock_networkstatus_get_reasonably_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1420,10 +1497,11 @@ test_build_update_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
-  now = mock_ns.valid_after+1;
+
+  time_t now = mock_ns.valid_after+1;
 
   /* Create a service without a current descriptor to trigger a build. */
   service = helper_create_service();
@@ -1547,9 +1625,9 @@ test_build_update_descriptors(void *arg)
   tt_int_op(smartlist_len(ip_cur->base.link_specifiers), OP_EQ, 3);
   /* Make sure we have a valid encryption keypair generated when we pick an
    * intro point in the update process. */
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
                              CURVE25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
                              CURVE25519_PUBKEY_LEN));
   tt_u64_op(ip_cur->time_to_expire, OP_GE, now +
             INTRO_POINT_LIFETIME_MIN_SECONDS);
@@ -1615,6 +1693,10 @@ test_build_update_descriptors(void *arg)
   tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   nodelist_free_all();
 }
@@ -1627,6 +1709,7 @@ test_build_descriptors(void *arg)
 {
   int ret;
   time_t now = time(NULL);
+  hs_service_t *last_service = NULL;
 
   (void) arg;
 
@@ -1637,7 +1720,7 @@ test_build_descriptors(void *arg)
   MOCK(networkstatus_get_reasonably_live_consensus,
        mock_networkstatus_get_reasonably_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1645,25 +1728,33 @@ test_build_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   /* Generate a valid number of fake auth clients when a client authorization
    * is disabled. */
   {
     hs_service_t *service = helper_create_service();
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
     build_all_descriptors(now);
+    tt_assert(service->desc_current);
+    tt_assert(service->desc_current->desc);
+
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(0);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1671,12 +1762,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is not a multiple of 16. */
   {
     hs_service_t *service = helper_create_service_with_clients(20);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1684,12 +1779,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Do not generate any fake desc client when the number of clients is
    * a multiple of 16 but not zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(32);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1697,9 +1796,13 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
  done:
+  helper_destroy_service(last_service);
   hs_free_all();
 }
 
@@ -1718,7 +1821,7 @@ test_upload_descriptors(void *arg)
   MOCK(networkstatus_get_reasonably_live_consensus,
        mock_networkstatus_get_reasonably_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1726,7 +1829,7 @@ test_upload_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
   now = mock_ns.valid_after+1;
@@ -1848,9 +1951,9 @@ test_rendezvous1_parsing(void *arg)
   }
 
   /* Send out the RENDEZVOUS1 and make sure that our mock func worked */
-  tt_assert(tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(fast_mem_is_zero(rend1_payload, 32));
   hs_circ_service_rp_has_opened(service, service_circ);
-  tt_assert(!tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(!fast_mem_is_zero(rend1_payload, 32));
   tt_int_op(rend1_payload_len, OP_EQ, HS_LEGACY_RENDEZVOUS_CELL_SIZE);
 
   /******************************/
@@ -2098,6 +2201,490 @@ test_export_client_circuit_id(void *arg)
   tor_free(cp2);
 }
 
+static smartlist_t *
+mock_node_get_link_specifier_smartlist(const node_t *node, bool direct_conn)
+{
+  (void) node;
+  (void) direct_conn;
+
+  smartlist_t *lspecs = smartlist_new();
+  link_specifier_t *ls_legacy = link_specifier_new();
+  smartlist_add(lspecs, ls_legacy);
+
+  return lspecs;
+}
+
+static node_t *fake_node = NULL;
+
+static const node_t *
+mock_build_state_get_exit_node(cpath_build_state_t *state)
+{
+  (void) state;
+
+  if (!fake_node) {
+    curve25519_secret_key_t seckey;
+    curve25519_secret_key_generate(&seckey, 0);
+
+    fake_node = tor_malloc_zero(sizeof(node_t));
+    fake_node->ri = tor_malloc_zero(sizeof(routerinfo_t));
+    fake_node->ri->onion_curve25519_pkey =
+      tor_malloc_zero(sizeof(curve25519_public_key_t));
+    curve25519_public_key_generate(fake_node->ri->onion_curve25519_pkey,
+                                   &seckey);
+  }
+
+  return fake_node;
+}
+
+static void
+mock_launch_rendezvous_point_circuit(const hs_service_t *service,
+                                     const hs_service_intro_point_t *ip,
+                                     const hs_cell_introduce2_data_t *data)
+{
+  (void) service;
+  (void) ip;
+  (void) data;
+  return;
+}
+
+/**
+ *  Test that INTRO2 cells are handled well by onion services in the normal
+ *  case and also when onionbalance is enabled.
+ */
+static void
+test_intro2_handling(void *arg)
+{
+  (void)arg;
+
+  MOCK(build_state_get_exit_node, mock_build_state_get_exit_node);
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+  MOCK(node_get_link_specifier_smartlist,
+       mock_node_get_link_specifier_smartlist);
+  MOCK(launch_rendezvous_point_circuit, mock_launch_rendezvous_point_circuit);
+
+  memset(relay_payload, 0, sizeof(relay_payload));
+
+  int retval;
+  time_t now = 0101010101;
+  update_approx_time(now);
+
+  /** OK this is the play:
+   *
+   *  In Act I, we have a standalone onion service X (without onionbalance
+   *  enabled). We test that X can properly handle INTRO2 cells sent by a
+   *  client Alice.
+   *
+   *  In Act II, we create an onionbalance setup with frontend being Z which
+   *  includes instances X and Y. We then setup onionbalance on X and test that
+   *  Alice who addresses Z can communicate with X through INTRO2 cells.
+   *
+   *  In Act III, we test that Alice can also communicate with X
+   *  directly even tho onionbalance is enabled.
+   *
+   *  And finally in Act IV, we check various cases where the INTRO2 cell
+   *  should not go through because the subcredentials don't line up
+   *  (e.g. Alice sends INTRO2 to X using Y's subcredential).
+   */
+
+  /** Let's start with some setup! Create the instances and the frontend
+      service, create Alice, etc:  */
+
+  /* Create instance X */
+  hs_service_t x_service;
+  memset(&x_service, 0, sizeof(hs_service_t));
+  /* Disable onionbalance */
+  x_service.config.ob_master_pubkeys = NULL;
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0,0);
+
+  /* Create subcredential for x: */
+  ed25519_keypair_t x_identity_keypair;
+  hs_subcredential_t x_subcred;
+  ed25519_keypair_generate(&x_identity_keypair, 0);
+  hs_helper_get_subcred_from_identity_keypair(&x_identity_keypair,
+                                              &x_subcred);
+
+  /* Create the x instance's intro point */
+  hs_service_intro_point_t *x_ip = NULL;
+  {
+    curve25519_secret_key_t seckey;
+    curve25519_public_key_t pkey;
+    curve25519_secret_key_generate(&seckey, 0);
+    curve25519_public_key_generate(&pkey, &seckey);
+
+    node_t intro_node;
+    memset(&intro_node, 0, sizeof(intro_node));
+    routerinfo_t ri;
+    memset(&ri, 0, sizeof(routerinfo_t));
+    ri.onion_curve25519_pkey = &pkey;
+    intro_node.ri = &ri;
+
+    x_ip = service_intro_point_new(&intro_node);
+  }
+
+  /* Create z frontend's subcredential */
+  ed25519_keypair_t z_identity_keypair;
+  hs_subcredential_t z_subcred;
+  ed25519_keypair_generate(&z_identity_keypair, 0);
+  hs_helper_get_subcred_from_identity_keypair(&z_identity_keypair,
+                                              &z_subcred);
+
+  /* Create y instance's subcredential */
+  ed25519_keypair_t y_identity_keypair;
+  hs_subcredential_t y_subcred;
+  ed25519_keypair_generate(&y_identity_keypair, 0);
+  hs_helper_get_subcred_from_identity_keypair(&y_identity_keypair,
+                                              &y_subcred);
+
+  /* Create Alice's intro point */
+  hs_desc_intro_point_t *alice_ip;
+  ed25519_keypair_t signing_kp;
+  ed25519_keypair_generate(&signing_kp, 0);
+  alice_ip = hs_helper_build_intro_point(&signing_kp, now, "1.2.3.4", 0,
+                                         &x_ip->auth_key_kp,
+                                         &x_ip->enc_key_kp);
+
+  /* Create Alice's intro and rend circuits */
+  origin_circuit_t *intro_circ = origin_circuit_new();
+  intro_circ->cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  intro_circ->cpath->prev = intro_circ->cpath;
+  intro_circ->hs_ident = tor_malloc_zero(sizeof(*intro_circ->hs_ident));
+  origin_circuit_t rend_circ;
+  rend_circ.hs_ident = tor_malloc_zero(sizeof(*rend_circ.hs_ident));
+  curve25519_keypair_generate(&rend_circ.hs_ident->rendezvous_client_kp, 0);
+  memset(rend_circ.hs_ident->rendezvous_cookie, 'r', HS_REND_COOKIE_LEN);
+
+  /* ************************************************************ */
+
+  /* Act I:
+   *
+   * Where Alice connects to X without onionbalance in the picture */
+
+  /* Create INTRODUCE1 */
+  tt_assert(fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &x_subcred);
+
+  /* Check that the payload was written successfully */
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  /* Handle the cell */
+  retval = hs_circ_handle_introduce2(&x_service,
+                                    intro_circ, x_ip,
+                                    &x_subcred,
+                                    (uint8_t*)relay_payload,relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* ************************************************************ */
+
+  /* Act II:
+   *
+   * We now create an onionbalance setup with Z being the frontend and X and Y
+   * being the backend instances. Make sure that Alice can talk with the
+   * backend instance X even tho she thinks she is talking to the frontend Z.
+   */
+
+  /* Now configure the X instance to do onionbalance with Z as the frontend */
+  x_service.config.ob_master_pubkeys = smartlist_new();
+  smartlist_add(x_service.config.ob_master_pubkeys,
+                &z_identity_keypair.pubkey);
+
+  /* Create descriptors for x and load next descriptor with the x's
+   * subcredential so that it can accept connections for itself. */
+  x_service.desc_current = service_descriptor_new();
+  memset(x_service.desc_current->desc->subcredential.subcred, 'C',SUBCRED_LEN);
+  x_service.desc_next = service_descriptor_new();
+  memcpy(&x_service.desc_next->desc->subcredential, &x_subcred, SUBCRED_LEN);
+
+  /* Refresh OB keys */
+  hs_ob_refresh_keys(&x_service);
+
+  /* Create INTRODUCE1 from Alice to X through Z */
+  memset(relay_payload, 0, sizeof(relay_payload));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &z_subcred);
+
+  /* Check that the payload was written successfully */
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  /* Deliver INTRODUCE1 to X even tho it carries Z's subcredential */
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &z_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  replaycache_free(x_ip->replay_cache);
+  x_ip->replay_cache = replaycache_new(0, 0);
+
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+
+  /* ************************************************************ */
+
+  /* Act III:
+   *
+   * Now send a direct INTRODUCE cell from Alice to X using X's subcredential
+   * and check that it succeeds even with onionbalance enabled.
+   */
+
+  /* Refresh OB keys (just to check for memleaks) */
+  hs_ob_refresh_keys(&x_service);
+
+  /* Create INTRODUCE1 from Alice to X using X's subcred. */
+  memset(relay_payload, 0, sizeof(relay_payload));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &x_subcred);
+
+  /* Check that the payload was written successfully */
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  /* Send INTRODUCE1 to X with X's subcredential (should succeed) */
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* ************************************************************ */
+
+  /* Act IV:
+   *
+   * Test cases where the INTRO2 cell should not be able to decode.
+   */
+
+  /* Try sending the exact same INTRODUCE2 cell again and see that the intro
+   * point replay cache triggers: */
+  setup_full_capture_of_logs(LOG_WARN);
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, -1);
+  expect_log_msg_containing("with the same ENCRYPTED section");
+  teardown_capture_of_logs();
+
+  /* Now cleanup the intro point replay cache but not the service replay cache
+     and see that this one triggers this time. */
+  replaycache_free(x_ip->replay_cache);
+  x_ip->replay_cache = replaycache_new(0, 0);
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, -1);
+  expect_log_msg_containing("with same REND_COOKIE");
+  teardown_capture_of_logs();
+
+  /* Now just to make sure cleanup both replay caches and make sure that the
+     cell gets through */
+  replaycache_free(x_ip->replay_cache);
+  x_ip->replay_cache = replaycache_new(0, 0);
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* As a final thing, create an INTRODUCE1 cell from Alice to X using Y's
+   * subcred (should fail since Y is just another instance and not the frontend
+   * service!) */
+  memset(relay_payload, 0, sizeof(relay_payload));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &y_subcred);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check that the payload was written successfully */
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &y_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  /* Start cleaning up X */
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  smartlist_free(x_service.config.ob_master_pubkeys);
+  tor_free(x_service.state.ob_subcreds);
+  service_descriptor_free(x_service.desc_current);
+  service_descriptor_free(x_service.desc_next);
+  service_intro_point_free(x_ip);
+
+  /* Clean up Alice */
+  hs_desc_intro_point_free(alice_ip);
+  tor_free(rend_circ.hs_ident);
+
+  if (fake_node) {
+    tor_free(fake_node->ri->onion_curve25519_pkey);
+    tor_free(fake_node->ri);
+    tor_free(fake_node);
+  }
+
+  UNMOCK(build_state_get_exit_node);
+  UNMOCK(relay_send_command_from_edge_);
+  UNMOCK(node_get_link_specifier_smartlist);
+  UNMOCK(launch_rendezvous_point_circuit);
+}
+
+static void
+test_cannot_upload_descriptors(void *arg)
+{
+  int ret;
+  time_t now;
+  hs_service_t *service;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  dummy_state = or_state_new();
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  update_approx_time(mock_ns.valid_after + 1);
+  now = mock_ns.valid_after + 1;
+
+  /* Create a service with no descriptor. It's added to the global map. */
+  service = hs_service_new(get_options());
+  tt_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  /* Register service to global map. */
+  ret = register_service(get_hs_service_map(), service);
+  tt_int_op(ret, OP_EQ, 0);
+  /* But first, build our descriptor. */
+  build_all_descriptors(now);
+
+  /* 1. Testing missing intro points reason. */
+  {
+    digest256map_t *cur = service->desc_current->intro_points.map;
+    digest256map_t *tmp = digest256map_new();
+    service->desc_current->intro_points.map = tmp;
+    service->desc_current->missing_intro_points = 1;
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    digest256map_free(tmp, tor_free_);
+    service->desc_current->intro_points.map = cur;
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Missing intro points");
+    teardown_capture_of_logs();
+    /* Reset. */
+    service->desc_current->missing_intro_points = 0;
+  }
+
+  /* 2. Testing non established intro points. */
+  {
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Intro circuits aren't yet all established (0/3).");
+    teardown_capture_of_logs();
+  }
+
+  /* We need to pass the established circuit tests and thus from now on, we
+   * MOCK this to return 3 intro points. */
+  MOCK(count_desc_circuit_established, mock_count_desc_circuit_established);
+  num_intro_points = 3;
+
+  /* 3. Testing non established intro points. */
+  {
+    service->desc_current->next_upload_time = now + 1000;
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Next upload time is");
+    teardown_capture_of_logs();
+    /* Reset. */
+    service->desc_current->next_upload_time = 0;
+  }
+
+  /* 4. Testing missing live consensus. */
+  {
+    MOCK(networkstatus_get_reasonably_live_consensus,
+         mock_networkstatus_get_reasonably_live_consensus_null);
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "No reasonably live consensus");
+    teardown_capture_of_logs();
+    /* Reset. */
+    MOCK(networkstatus_get_reasonably_live_consensus,
+         mock_networkstatus_get_reasonably_live_consensus);
+  }
+
+  /* 5. Test missing minimum directory information. */
+  {
+    MOCK(router_have_minimum_dir_info,
+         mock_router_have_minimum_dir_info_false);
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Not enough directory information");
+    teardown_capture_of_logs();
+
+    /* Running it again shouldn't trigger anything due to rate limitation. */
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_no_log_entry();
+    teardown_capture_of_logs();
+    UNMOCK(router_have_minimum_dir_info);
+  }
+
+  /* Increase time and redo test (5) in order to test the rate limiting. */
+  update_approx_time(mock_ns.valid_after + 61);
+  {
+    MOCK(router_have_minimum_dir_info,
+         mock_router_have_minimum_dir_info_false);
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Not enough directory information");
+    teardown_capture_of_logs();
+    UNMOCK(router_have_minimum_dir_info);
+  }
+
+ done:
+  hs_free_all();
+  UNMOCK(count_desc_circuit_established);
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+  UNMOCK(get_or_state);
+}
+
 struct testcase_t hs_service_tests[] = {
   { "e2e_rend_circuit_setup", test_e2e_rend_circuit_setup, TT_FORK,
     NULL, NULL },
@@ -2123,7 +2710,7 @@ struct testcase_t hs_service_tests[] = {
     NULL, NULL },
   { "rdv_circuit_opened", test_rdv_circuit_opened, TT_FORK,
     NULL, NULL },
-  { "introduce2", test_introduce2, TT_FORK,
+  { "bad_introduce2", test_bad_introduce2, TT_FORK,
     NULL, NULL },
   { "service_event", test_service_event, TT_FORK,
     NULL, NULL },
@@ -2135,12 +2722,15 @@ struct testcase_t hs_service_tests[] = {
     NULL, NULL },
   { "upload_descriptors", test_upload_descriptors, TT_FORK,
     NULL, NULL },
+  { "cannot_upload_descriptors", test_cannot_upload_descriptors, TT_FORK,
+    NULL, NULL },
   { "rendezvous1_parsing", test_rendezvous1_parsing, TT_FORK,
     NULL, NULL },
   { "authorized_client_config_equal", test_authorized_client_config_equal,
     TT_FORK, NULL, NULL },
   { "export_client_circuit_id", test_export_client_circuit_id, TT_FORK,
     NULL, NULL },
+  { "intro2_handling", test_intro2_handling, TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c
index 4a6d90d97e..0ae78496b2 100644
--- a/src/test/test_introduce.c
+++ b/src/test/test_introduce.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -383,8 +383,10 @@ make_intro_from_plaintext(
 
   /* Output the cell */
   *cell_out = cell;
+  cell = NULL;
 
  done:
+  tor_free(cell);
   return cell_len;
 }
 
@@ -535,4 +537,3 @@ struct testcase_t introduce_tests[] = {
   INTRODUCE_LEGACY(late_parse_v3),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh
index 3474210607..2238f7aa78 100755
--- a/src/test/test_key_expiration.sh
+++ b/src/test/test_key_expiration.sh
@@ -6,14 +6,28 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -21,11 +35,21 @@ if test "$UNAME_OS" = 'CYGWIN' || \
   exit 77
 fi
 
+# find the tor binary
 if [ $# -ge 1 ]; then
   TOR_BINARY="${1}"
   shift
 else
-  TOR_BINARY="${TESTING_TOR_BINARY}"
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "$TOR_BINARY" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
 fi
 
 if [ $# -ge 1 ]; then
@@ -47,11 +71,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_key_expiration_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -60,10 +84,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh
index 7afff271cb..6812f8883d 100755
--- a/src/test/test_keygen.sh
+++ b/src/test/test_keygen.sh
@@ -6,14 +6,28 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -21,14 +35,22 @@ if test "$UNAME_OS" = 'CYGWIN' || \
   exit 77
 fi
 
+# find the tor binary
 if [ $# -ge 1 ]; then
   TOR_BINARY="${1}"
   shift
 else
-  TOR_BINARY="${TESTING_TOR_BINARY}"
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
 fi
 
+TOR_BINARY="$(abspath "$TOR_BINARY")"
 
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "$TOR_BINARY" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
+fi
 
   if [ $# -ge 1 ]; then
       dflt=0
@@ -64,11 +86,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_keygen_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_keygen_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -77,10 +99,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
@@ -144,7 +166,9 @@ ME="${DATA_DIR}/case2a"
 SRC="${DATA_DIR}/orig"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: `cat ${ME}/stdout`" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Somehow succeeded when missing secret key, certs: $(cat "${ME}/stdout")"
+fi
 check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key"
 
 grep "We needed to load a secret key.*but couldn't find it" "${ME}/stdout" >/dev/null || die "Tor didn't declare that it was missing a secret key"
@@ -281,7 +305,9 @@ SRC="${DATA_DIR}/encrypted"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_secret_key_encrypted" "${ME}/keys/"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Tor started with encrypted secret key and no certs" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Tor started with encrypted secret key and no certs"
+fi
 check_no_file "${ME}/keys/ed25519_signing_cert"
 check_no_file "${ME}/keys/ed25519_signing_secret_key"
 
@@ -370,7 +396,9 @@ mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
 cp "${OTHER}/keys/ed25519_master_id_secret_key" "${ME}/keys/"
 
-${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout" && die "Successfully started with mismatched keys!?" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout"; then
+  die "Successfully started with mismatched keys!?"
+fi
 
 grep "public_key does not match.*secret_key" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a key mismatch"
 
@@ -386,7 +414,9 @@ ME="${DATA_DIR}/case11a"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout" && die "Successfully started with passphrase-fd but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout"; then
+  die "Successfully started with passphrase-fd but no keygen?"
+fi
 
 grep "passphrase-fd specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -402,7 +432,9 @@ ME="${DATA_DIR}/case11b"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout" && die "Successfully started with no-passphrase but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with no-passphrase but no keygen?"
+fi
 
 grep "no-passphrase specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -418,7 +450,9 @@ ME="${DATA_DIR}/case11C"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout" && die "Successfully started with newpass but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout"; then
+  die "Successfully started with newpass but no keygen?"
+fi
 
 grep "newpass specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -456,7 +490,9 @@ ME="${DATA_DIR}/case11E"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd?"
+fi
 
 grep "Invalid --passphrase-fd value" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -473,7 +509,9 @@ ME="${DATA_DIR}/case11F"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd combination?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd combination?"
+fi
 
 grep "no-passphrase specified with --passphrase-fd" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
diff --git a/src/test/test_keypin.c b/src/test/test_keypin.c
index e7beef8609..ff6397f8c7 100644
--- a/src/test/test_keypin.c
+++ b/src/test/test_keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c
index 34f59f26cd..1566b349ed 100644
--- a/src/test/test_link_handshake.c
+++ b/src/test/test_link_handshake.c
@@ -1,11 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CHANNELTLS_PRIVATE
 #define CONNECTION_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define TORTLS_PRIVATE
 
 #include "core/or/or.h"
@@ -18,6 +18,7 @@
 #include "feature/relay/routerkeys.h"
 #include "core/or/scheduler.h"
 #include "feature/nodelist/torcert.h"
+#include "feature/relay/relay_handshake.h"
 
 #include "core/or/or_connection_st.h"
 #include "core/or/or_handshake_certs_st.h"
@@ -263,7 +264,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_assert(c1->handshake_state->authenticated_rsa);
     tt_assert(! c1->handshake_state->authenticated_ed25519);
   }
-  tt_assert(! tor_mem_is_zero(
+  tt_assert(! fast_mem_is_zero(
                 (char*)c1->handshake_state->authenticated_rsa_peer_id, 20));
 
   chan2 = tor_malloc_zero(sizeof(*chan2));
@@ -290,7 +291,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_ptr_op(c2->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
   }
   tt_assert(c2->handshake_state->certs->id_cert);
-  tt_assert(tor_mem_is_zero(
+  tt_assert(fast_mem_is_zero(
               (char*)c2->handshake_state->authenticated_rsa_peer_id, 20));
   /* no authentication has happened yet, since we haen't gotten an AUTH cell.
    */
@@ -325,7 +326,7 @@ test_link_handshake_certs_ok(void *arg)
   crypto_pk_free(key2);
 }
 
-typedef struct certs_data_s {
+typedef struct certs_data_t {
   int is_ed;
   int is_link_cert;
   or_connection_t *c;
@@ -948,7 +949,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_int_op(36, OP_EQ, cell1->payload_len);
   tt_int_op(36, OP_EQ, cell2->payload_len);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_int_op(0, OP_EQ, cell1->circ_id);
   tt_int_op(0, OP_EQ, cell2->circ_id);
   tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell1->command);
@@ -960,7 +961,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 4);
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 4);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_mem_op(cell1->payload, OP_NE, cell2->payload, 32);
 
  done:
@@ -972,7 +973,7 @@ test_link_handshake_send_authchallenge(void *arg)
   crypto_pk_free(rsa1);
 }
 
-typedef struct authchallenge_data_s {
+typedef struct authchallenge_data_t {
   or_connection_t *c;
   channel_tls_t *chan;
   var_cell_t *cell;
@@ -1171,7 +1172,7 @@ mock_set_circid_type(channel_t *chan,
   (void) consider_identity;
 }
 
-typedef struct authenticate_data_s {
+typedef struct authenticate_data_t {
   int is_ed;
   or_connection_t *c1, *c2;
   channel_tls_t *chan2;
@@ -1492,6 +1493,7 @@ AUTHENTICATE_FAIL(missing_ed_auth,
                     "authentication certificate";
                 })
 
+#ifndef COCCI
 #define TEST_RSA(name, flags)                                           \
   { #name , test_link_handshake_ ## name, (flags),                      \
       &passthrough_setup, (void*)"RSA" }
@@ -1527,6 +1529,7 @@ AUTHENTICATE_FAIL(missing_ed_auth,
 #define TEST_AUTHENTICATE_ED(name)                                      \
   { "authenticate/" #name "_ed25519" , test_link_handshake_auth_ ## name, \
       TT_FORK, &setup_authenticate, (void*)3 }
+#endif /* !defined(COCCI) */
 
 struct testcase_t link_handshake_tests[] = {
   TEST_RSA(certs_ok, TT_FORK),
diff --git a/src/test/test_logging.c b/src/test/test_logging.c
index 95a2fce757..e09f7a21cd 100644
--- a/src/test/test_logging.c
+++ b/src/test/test_logging.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
@@ -9,14 +9,13 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 
 static void
-dummy_cb_fn(int severity, uint32_t domain, const char *msg)
+dummy_cb_fn(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)severity; (void)domain; (void)msg;
 }
@@ -36,7 +35,7 @@ test_get_sigsafe_err_fds(void *arg)
 
   set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug);
   set_log_severity_config(LOG_WARN, LOG_ERR, &no_bug);
-  no_bug.masks[0] &= ~(LD_BUG|LD_GENERAL);
+  no_bug.masks[SEVERITY_MASK_IDX(LOG_ERR)] &= ~(LD_BUG|LD_GENERAL);
   set_log_severity_config(LOG_INFO, LOG_NOTICE, &no_bug2);
 
   /* Add some logs; make sure the output is as expected. */
@@ -117,22 +116,27 @@ test_sigsafe_err(void *arg)
   content = read_file_to_str(fn, 0, NULL);
 
   tt_ptr_op(content, OP_NE, NULL);
-  tor_split_lines(lines, content, (int)strlen(content));
+  smartlist_split_string(lines, content, "\n", 0, 0);
   tt_int_op(smartlist_len(lines), OP_GE, 5);
 
-  if (strstr(smartlist_get(lines, 0), "opening new log file"))
+  if (strstr(smartlist_get(lines, 0), "opening new log file")) {
+    void *item = smartlist_get(lines, 0);
     smartlist_del_keeporder(lines, 0);
+    tor_free(item);
+  }
+
   tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool"));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 1), "=============="));
-  tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal."));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 3), "=============="));
-  tt_str_op(smartlist_get(lines, 4), OP_EQ,
+  tt_str_op(smartlist_get(lines, 1), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 2), "=============="));
+  tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal."));
+  tt_str_op(smartlist_get(lines, 4), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 5), "=============="));
+  tt_str_op(smartlist_get(lines, 6), OP_EQ,
             "Testing any attempt to manually log from a signal.");
 
  done:
   tor_free(content);
+  SMARTLIST_FOREACH(lines, char *, x, tor_free(x));
   smartlist_free(lines);
 }
 
diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c
index 089ea812cf..c4e60d9da5 100644
--- a/src/test/test_mainloop.c
+++ b/src/test/test_mainloop.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,11 +6,29 @@
  * \brief Tests for functions closely related to the Tor main loop
  */
 
+#define CONFIG_PRIVATE
+#define MAINLOOP_PRIVATE
+#define STATEFILE_PRIVATE
+
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
+#include "lib/confmgt/confmgt.h"
+
 #include "core/or/or.h"
+#include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_state_st.h"
+#include "core/mainloop/mainloop_sys.h"
+#include "core/mainloop/netstatus.h"
+
+#include "feature/hs/hs_service.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/config/or_state_st.h"
+
+#include "app/main/subsysmgr.h"
 
 static const uint64_t BILLION = 1000000000;
 
@@ -131,12 +149,233 @@ test_mainloop_update_time_jumps(void *arg)
   monotime_disable_test_mocking();
 }
 
+static int schedule_rescan_called = 0;
+static void
+mock_schedule_rescan_periodic_events(void)
+{
+  ++schedule_rescan_called;
+}
+
+static void
+test_mainloop_user_activity(void *arg)
+{
+  (void)arg;
+  const time_t start = 1542658829;
+  update_approx_time(start);
+
+  MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events);
+
+  reset_user_activity(start);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  set_network_participation(false);
+
+  // reset can move backwards and forwards, but does not change network
+  // participation.
+  reset_user_activity(start-10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10);
+  reset_user_activity(start+10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  tt_int_op(schedule_rescan_called, OP_EQ, 0);
+  tt_int_op(false, OP_EQ, is_participating_on_network());
+
+  // "note" can only move forward.  Calling it from a non-participating
+  // state makes us rescan the periodic callbacks and set participation.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // Calling it again will move us forward, but not call rescan again.
+  note_user_activity(start+25);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We won't move backwards.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We _will_ adjust if the clock jumps though.
+  netstatus_note_clock_jumped(500);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525);
+
+  netstatus_note_clock_jumped(-400);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125);
+
+ done:
+  UNMOCK(schedule_rescan_periodic_events);
+}
+
+static unsigned int
+mock_get_num_services(void)
+{
+  return 1;
+}
+
+static connection_t *
+mock_connection_gbtu(int type)
+{
+  (void) type;
+  return (void *)"hello fellow connections";
+}
+
+static void
+test_mainloop_check_participation(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  const time_t start = 1542658829;
+  const time_t ONE_DAY = 24*60*60;
+
+  // Suppose we've been idle for a day or two
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  check_network_participation_callback(start, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+  // suppose we've been idle for 2 days... but we are a server.
+  reset_user_activity(start - 2*ONE_DAY);
+  options->ORPort_set = 1;
+  set_network_participation(true);
+  check_network_participation_callback(start+2, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2);
+  options->ORPort_set = 0;
+
+  // idle for 2 days, but we have a hidden service.
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  MOCK(hs_service_get_num_services, mock_get_num_services);
+  check_network_participation_callback(start+3, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3);
+  UNMOCK(hs_service_get_num_services);
+
+  // idle for 2 days but we have at least one user connection
+  MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu);
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+  check_network_participation_callback(start+10, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  // as above, but DormantTimeoutDisabledByIdleStreams is not set
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 0;
+  check_network_participation_callback(start+13, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+  UNMOCK(connection_get_by_type_nonlinked);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+
+  // idle for 2 days but DormantClientTimeout is 3 days
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantClientTimeout = ONE_DAY * 3;
+  check_network_participation_callback(start+30, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+ done:
+  or_options_free(options);
+  UNMOCK(hs_service_get_num_services);
+  UNMOCK(connection_get_by_type_nonlinked);
+}
+
+static void
+test_mainloop_dormant_load_state(void *arg)
+{
+  (void)arg;
+  or_state_t *or_state = or_state_new();
+  mainloop_state_t *state;
+  {
+    int idx = subsystems_get_state_idx(&sys_mainloop);
+    tor_assert(idx >= 0);
+    state = config_mgr_get_obj_mutable(get_state_mgr(), or_state, idx);
+  }
+  const time_t start = 1543956575;
+
+  reset_user_activity(0);
+  set_network_participation(false);
+
+  // When we construct a new state, it starts out in "auto" mode.
+  tt_int_op(state->Dormant, OP_EQ, -1);
+
+  // Initializing from "auto" makes us start out (by default) non-Dormant,
+  // with activity right now.
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  // Initializing from dormant clears the last user activity time, and
+  // makes us dormant.
+  state->Dormant = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(! is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, 0);
+
+  // Initializing from non-dormant sets the last user activity time, and
+  // makes us non-dormant.
+  state->Dormant = 0;
+  state->MinutesSinceUserActivity = 123;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60);
+
+  // If we would start dormant, but DormantCanceledByStartup is set, then
+  // we start up non-dormant.
+  state->Dormant = 1;
+  get_options_mutable()->DormantCanceledByStartup = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+ done:
+  or_state_free(or_state);
+}
+
+static void
+test_mainloop_dormant_save_state(void *arg)
+{
+  (void)arg;
+  mainloop_state_t *state = tor_malloc_zero(sizeof(mainloop_state_t));
+  const time_t start = 1543956575;
+
+  // Can we save a non-dormant state correctly?
+  reset_user_activity(start - 1000);
+  set_network_participation(true);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 0);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60);
+
+  // Can we save a dormant state correctly?
+  set_network_participation(false);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 1);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0);
+
+ done:
+  tor_free(state);
+}
+
 #define MAINLOOP_TEST(name) \
   { #name, test_mainloop_## name , TT_FORK, NULL, NULL }
 
 struct testcase_t mainloop_tests[] = {
   MAINLOOP_TEST(update_time_normal),
   MAINLOOP_TEST(update_time_jumps),
+  MAINLOOP_TEST(user_activity),
+  MAINLOOP_TEST(check_participation),
+  MAINLOOP_TEST(dormant_load_state),
+  MAINLOOP_TEST(dormant_save_state),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c
index 4c4317d81a..f89025aa6c 100644
--- a/src/test/test_microdesc.c
+++ b/src/test/test_microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,6 +11,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
 
@@ -20,6 +21,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -70,6 +72,7 @@ test_md_cache(void *data)
   const char *test_md3_noannotation = strchr(test_md3, '\n')+1;
   time_t time1, time2, time3;
   char *fn = NULL, *s = NULL;
+  char *encoded_family = NULL;
   (void)data;
 
   options = get_options_mutable();
@@ -172,8 +175,9 @@ test_md_cache(void *data)
 
   tt_ptr_op(md1->family, OP_EQ, NULL);
   tt_ptr_op(md3->family, OP_NE, NULL);
-  tt_int_op(smartlist_len(md3->family), OP_EQ, 3);
-  tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX");
+
+  encoded_family = nodefamily_format(md3->family);
+  tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez");
 
   /* Now rebuild the cache! */
   tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0);
@@ -254,6 +258,7 @@ test_md_cache(void *data)
   smartlist_free(wanted);
   tor_free(s);
   tor_free(fn);
+  tor_free(encoded_family);
 }
 
 static const char truncated_md[] =
@@ -417,6 +422,28 @@ static const char test_md2_21[] =
   "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
   "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
 
+static const char test_md2_withfamily_28[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family OtherNode !Strange\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
+static const char test_md2_withfamily_29[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
 static void
 test_md_generate(void *arg)
 {
@@ -447,13 +474,24 @@ test_md_generate(void *arg)
   tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey,
                               &ri->cache_info.signing_key_cert->signing_key));
 
+  // Try family encoding.
+  microdesc_free(md);
+  ri->declared_family = smartlist_new();
+  smartlist_add_strdup(ri->declared_family, "OtherNode !Strange");
+  md = dirvote_create_microdescriptor(ri, 28);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_28);
+
+  microdesc_free(md);
+  md = dirvote_create_microdescriptor(ri, 29);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_29);
+
  done:
   microdesc_free(md);
   routerinfo_free(ri);
 }
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
@@ -611,9 +649,44 @@ static const char MD_PARSE_TEST_DATA[] =
   "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
   "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   "p6 allow 80\n"
+  /* Good 11: Normal, non-exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 12: Normal, exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 13: Normal, exit relay with only ipv6 exit policy */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p6 accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   ;
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-ENABLE_GCC_WARNING(overlength-strings)
+ENABLE_GCC_WARNING("-Woverlength-strings")
 #endif
 
 /** More tests for parsing different kinds of microdescriptors, and getting
@@ -628,7 +701,7 @@ test_md_parse(void *arg)
   smartlist_t *mds = microdescs_parse_from_string(MD_PARSE_TEST_DATA,
                                                   NULL, 1, SAVED_NOWHERE,
                                                   invalid);
-  tt_int_op(smartlist_len(mds), OP_EQ, 11);
+  tt_int_op(smartlist_len(mds), OP_EQ, 14);
   tt_int_op(smartlist_len(invalid), OP_EQ, 4);
 
   test_memeq_hex(smartlist_get(invalid,0),
@@ -675,6 +748,21 @@ test_md_parse(void *arg)
   tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
   tt_int_op(md->ipv6_orport, OP_EQ, 9090);
 
+  md = smartlist_get(mds, 11);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 1);
+
+  md = smartlist_get(mds, 12);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
+  md = smartlist_get(mds, 13);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
  done:
   SMARTLIST_FOREACH(mds, microdesc_t *, mdsc, microdesc_free(mdsc));
   smartlist_free(mds);
@@ -683,6 +771,80 @@ test_md_parse(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+static void
+test_md_parse_id_ed25519(void *arg)
+{
+  (void)arg;
+
+  /* A correct MD with an ed25519 ID ... and an unspecified ID type,
+   * which is permitted. */
+  const char GOOD_MD[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id wumpus dodecahedron\n";
+
+  smartlist_t *mds = NULL;
+  const microdesc_t *md;
+
+  mds = microdescs_parse_from_string(GOOD_MD,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 1);
+  md = smartlist_get(mds, 0);
+  tt_mem_op(md->ed25519_identity_pkey, OP_EQ,
+            "This isn't actually a public key", ED25519_PUBKEY_LEN);
+  SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key appears twice. */
+  const char DUPLICATE_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n";
+
+  setup_capture_of_logs(LOG_WARN);
+  mds = microdescs_parse_from_string(DUPLICATE_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Extra ed25519 key");
+  mock_clean_saved_logs();
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key is invalid. */
+  const char BOGUS_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyZZZZZZZZZZZ\n";
+
+  mds = microdescs_parse_from_string(BOGUS_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Bogus ed25519 key");
+
+ done:
+  if (mds) {
+    SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+    smartlist_free(mds);
+  }
+  teardown_capture_of_logs();
+}
+
 static int mock_rgsbd_called = 0;
 static routerstatus_t *mock_rgsbd_val_a = NULL;
 static routerstatus_t *mock_rgsbd_val_b = NULL;
@@ -816,6 +978,7 @@ struct testcase_t microdesc_tests[] = {
   { "broken_cache", test_md_cache_broken, TT_FORK, NULL, NULL },
   { "generate", test_md_generate, 0, NULL, NULL },
   { "parse", test_md_parse, 0, NULL, NULL },
+  { "parse_id_ed25519", test_md_parse_id_ed25519, 0, NULL, NULL },
   { "reject_cache", test_md_reject_cache, TT_FORK, NULL, NULL },
   { "corrupt_desc", test_md_corrupt_desc, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
diff --git a/src/test/test_namemap.c b/src/test/test_namemap.c
new file mode 100644
index 0000000000..e93d3fbc3c
--- /dev/null
+++ b/src/test/test_namemap.c
@@ -0,0 +1,174 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/malloc/malloc.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static void
+test_namemap_empty(void *arg)
+{
+  (void)arg;
+
+  namemap_t m;
+  namemap_init(&m);
+  namemap_t m2 = NAMEMAP_INIT();
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m2));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+ done:
+  namemap_clear(&m);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_toolong(void *arg)
+{
+  (void)arg;
+  namemap_t m;
+  char *ok = NULL;
+  char *toolong = NULL;
+  namemap_init(&m);
+
+  ok = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+1);
+  memset(ok, 'x', MAX_NAMEMAP_NAME_LEN);
+
+  toolong = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+2);
+  memset(toolong, 'x', MAX_NAMEMAP_NAME_LEN+1);
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+  unsigned u1 = namemap_get_or_create_id(&m, toolong);
+  unsigned u2 = namemap_get_or_create_id(&m, ok);
+  tt_uint_op(u1, OP_EQ, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+
+  tt_str_op(ok, OP_EQ, namemap_get_name(&m, u2));
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m, u1));
+
+ done:
+  tor_free(ok);
+  tor_free(toolong);
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_blackbox(void *arg)
+{
+  (void)arg;
+
+  namemap_t m1, m2;
+  namemap_init(&m1);
+  namemap_init(&m2);
+
+  unsigned u1 = namemap_get_or_create_id(&m1, "hello");
+  unsigned u2 = namemap_get_or_create_id(&m1, "world");
+  tt_uint_op(u1, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u1, OP_NE, u2);
+
+  tt_uint_op(u1, OP_EQ, namemap_get_id(&m1, "hello"));
+  tt_uint_op(u1, OP_EQ, namemap_get_or_create_id(&m1, "hello"));
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m1, "world"));
+  tt_uint_op(u2, OP_EQ, namemap_get_or_create_id(&m1, "world"));
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "HELLO"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+
+  unsigned u3 = namemap_get_or_create_id(&m2, "hola");
+  tt_uint_op(u3, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_or_create_id(&m2, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  unsigned int u4 = namemap_get_or_create_id(&m1, "hola");
+  tt_uint_op(u4, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u4, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  tt_str_op("hello", OP_EQ, namemap_get_name(&m1, u1));
+  tt_str_op("world", OP_EQ, namemap_get_name(&m1, u2));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m2, u3));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m1, u4));
+
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m2, u3 + 10));
+
+ done:
+  namemap_clear(&m1);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_internals(void *arg)
+{
+  (void)arg;
+  // This test actually assumes know something about the identity layout.
+  namemap_t m;
+  namemap_init(&m);
+
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+
+ done:
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_fmt(void *arg)
+{
+  (void)arg;
+  namemap_t m = NAMEMAP_INIT();
+
+  unsigned a = namemap_get_or_create_id(&m, "greetings");
+  unsigned b = namemap_get_or_create_id(&m, "earthlings");
+
+  tt_str_op(namemap_fmt_name(&m, a), OP_EQ, "greetings");
+  tt_str_op(namemap_fmt_name(&m, b), OP_EQ, "earthlings");
+  tt_int_op(a, OP_NE, 100);
+  tt_int_op(b, OP_NE, 100);
+  tt_str_op(namemap_fmt_name(&m, 100), OP_EQ, "{100}");
+
+ done:
+  namemap_clear(&m);
+}
+
+#define T(name) \
+  { #name, test_namemap_ ## name , 0, NULL, NULL }
+
+struct testcase_t namemap_tests[] = {
+  T(empty),
+  T(toolong),
+  T(blackbox),
+  T(internals),
+  T(fmt),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c
new file mode 100644
index 0000000000..93892978dc
--- /dev/null
+++ b/src/test/test_netinfo.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "trunnel/netinfo.h"
+#include "test/test.h"
+
+static void
+test_netinfo_unsupported_addr(void *arg)
+{
+  const uint8_t wire_data[] =
+    { // TIME
+      0x00, 0x00, 0x00, 0x01,
+      // OTHERADDR
+      0x04, // ATYPE
+      0x04, // ALEN
+      0x08, 0x08, 0x08, 0x08, // AVAL
+      0x01, // NMYADDR
+      0x03, // ATYPE (unsupported)
+      0x05, // ALEN
+      'a', 'd', 'r', 'r', '!' // AVAL (unsupported)
+    };
+
+  (void)arg;
+
+  netinfo_cell_t *parsed_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data,
+                  sizeof(wire_data));
+
+  tt_assert(parsed == sizeof(wire_data));
+
+  netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0);
+  tt_assert(addr);
+
+  tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr));
+  tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr));
+
+ done:
+  netinfo_cell_free(parsed_cell);
+}
+
+struct testcase_t netinfo_tests[] = {
+  { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c
index 53eb0413e5..fbbbf0a99f 100644
--- a/src/test/test_nodelist.c
+++ b/src/test/test_nodelist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,15 +6,24 @@
  * \brief Unit tests for nodelist related functions.
  **/
 
+#define NODELIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/torcert.h"
 
+#include "core/or/extend_info_st.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/nodelist/fmt_routerstatus.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodefamily_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
@@ -72,7 +81,7 @@ test_nodelist_node_get_verbose_nickname_not_named(void *arg)
 }
 
 /** A node should be considered a directory server if it has an open dirport
- * of it accepts tunnelled directory requests.
+ * or it accepts tunnelled directory requests.
  */
 static void
 test_nodelist_node_is_dir(void *arg)
@@ -231,6 +240,1176 @@ test_nodelist_ed_id(void *arg)
 #undef N_NODES
 }
 
+static void
+test_nodelist_nodefamily(void *arg)
+{
+  (void)arg;
+  /* hex ID digests */
+  const char h1[] = "5B435D6869206861206C65207363617270652070";
+  const char h2[] = "75C3B220616E6461726520696E206769726F2061";
+  const char h3[] = "2074726F766172206461206D616E67696172652C";
+  const char h4[] = "206D656E747265206E6F6E2076616C65206C2769";
+  const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669";
+
+  /* binary ID digests */
+  uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN],
+    d5[DIGEST_LEN];
+  base16_decode((char*)d1, sizeof(d1), h1, strlen(h1));
+  base16_decode((char*)d2, sizeof(d2), h2, strlen(h2));
+  base16_decode((char*)d3, sizeof(d3), h3, strlen(h3));
+  base16_decode((char*)d4, sizeof(d4), h4, strlen(h4));
+  base16_decode((char*)d5, sizeof(d5), h5, strlen(h5));
+
+  char *enc=NULL, *enc2=NULL;
+
+  nodefamily_t *nf1 = NULL;
+  nodefamily_t *nf2 = NULL;
+  nodefamily_t *nf3 = NULL;
+
+  enc = nodefamily_format(NULL);
+  tt_str_op(enc, OP_EQ, "");
+  tor_free(enc);
+
+  /* Make sure that sorting and de-duplication work. */
+  tor_asprintf(&enc, "$%s hello", h1);
+  nf1 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf1);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "hello hello $%s hello", h1);
+  nf2 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf2);
+  tt_ptr_op(nf1, OP_EQ, nf2);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "%s $%s hello", h1, h1);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(! nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_nickname(nf1, "hello"));
+  tt_assert(nodefamily_contains_nickname(nf1, "HELLO"));
+  tt_assert(! nodefamily_contains_nickname(nf1, "goodbye"));
+
+  tt_int_op(nf1->refcnt, OP_EQ, 3);
+  nodefamily_free(nf3);
+  tt_int_op(nf1->refcnt, OP_EQ, 2);
+
+  /* Try parsing with a provided self RSA digest. */
+  nf3 = nodefamily_parse("hello ", d1, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+
+  /* Do we get the expected result when we re-encode? */
+  tor_asprintf(&enc, "$%s hello", h1);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc2, OP_EQ, enc);
+  tor_free(enc2);
+  tor_free(enc);
+
+  /* Make sure that we get a different result if we give a different digest. */
+  nodefamily_free(nf3);
+  tor_asprintf(&enc, "hello $%s hello", h3);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_NE, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf3, d3));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d2));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d1));
+  tt_assert(nodefamily_contains_nickname(nf3, "hello"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "goodbye"));
+
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+
+  /* Try one with several digests, all with nicknames appended, in different
+     formats. */
+  tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4);
+  nf1 = nodefamily_parse(enc, d5, 0);
+  tt_assert(nf1);
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d3));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d5));
+  /* Nicknames aren't preserved when ids are present, since node naming is
+   * deprecated */
+  tt_assert(! nodefamily_contains_nickname(nf3, "res"));
+  tor_free(enc);
+  tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc, OP_EQ, enc2);
+  tor_free(enc);
+  tor_free(enc2);
+
+  /* Try ones where we parse the empty string. */
+  nf2 = nodefamily_parse("", NULL, 0);
+  nf3 = nodefamily_parse("", d4, 0);
+  tt_assert(nf2);
+  tt_assert(nf3);
+  tt_ptr_op(nf2, OP_NE, nf3);
+
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf3, d4));
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d5));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d5));
+  tt_assert(! nodefamily_contains_nickname(nf2, "fred"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "bosco"));
+
+  /* The NULL family should contain nothing. */
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d4));
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d5));
+
+ done:
+  tor_free(enc);
+  tor_free(enc2);
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+  nodefamily_free_all();
+}
+
+static void
+test_nodelist_nodefamily_parse_err(void *arg)
+{
+  (void)arg;
+  nodefamily_t *nf1 = NULL;
+  char *enc = NULL;
+  const char *semibogus =
+    "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname
+    "$jkASDFLkjsadfjhkl " // not hex
+    "$7468696e67732d696e2d7468656d73656c766573 " // ok
+    "reticulatogranulate "// ok
+    "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex
+    "$616273656e746d696e6465646e6573736573" // too short for hex
+    ;
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // We only get two items when we parse this.
+  for (int reject = 0; reject <= 1; ++reject) {
+    for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) {
+      unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0;
+      flags |= reject ? NF_REJECT_MALFORMED : 0;
+      nf1 = nodefamily_parse(semibogus, NULL, flags);
+      if (reject) {
+        tt_assert(nf1 == NULL);
+      } else {
+        tt_assert(nf1);
+        enc = nodefamily_format(nf1);
+        tt_str_op(enc, OP_EQ,
+                  "$7468696E67732D696E2D7468656D73656C766573 "
+                  "reticulatogranulate");
+        tor_free(enc);
+      }
+
+      if (log_at_warn) {
+        expect_log_msg_containing("$616273656e746d696e6465646e6573736573");
+        expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf");
+      } else {
+        tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0);
+      }
+      mock_clean_saved_logs();
+    }
+  }
+
+ done:
+  tor_free(enc);
+  nodefamily_free(nf1);
+  teardown_capture_of_logs();
+}
+
+static const node_t *
+mock_node_get_by_id(const char *id)
+{
+  if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  memcpy(fake_node->identity, id, DIGEST_LEN);
+  return fake_node;
+}
+
+static const node_t *
+mock_node_get_by_nickname(const char *nn, unsigned flags)
+{
+  (void)flags;
+  if (!strcmp(nn, "nonesuch"))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  strlcpy(fake_node->identity, nn, DIGEST_LEN);
+  return fake_node;
+}
+
+static void
+test_nodelist_nodefamily_lookup(void *arg)
+{
+  (void)arg;
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  smartlist_t *sl = smartlist_new();
+  nodefamily_t *nf1 = NULL;
+  char *mem_op_hex_tmp = NULL;
+
+  // 'null' is allowed.
+  nodefamily_add_nodes_to_smartlist(NULL, sl);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
+
+  // Try a real family
+  nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE "
+                         "$2121212121212121212121212121212121212121 "
+                         "$3333333333333333333333333333333333333333 "
+                         "erewhon nonesuch", NULL, 0);
+  tt_assert(nf1);
+  nodefamily_add_nodes_to_smartlist(nf1, sl);
+  // There were 5 elements; 2 were dropped because the mocked lookup failed.
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+
+  const node_t *n = smartlist_get(sl, 0);
+  test_memeq_hex(n->identity, "3333333333333333333333333333333333333333");
+  n = smartlist_get(sl, 1);
+  test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE");
+  n = smartlist_get(sl, 2);
+  tt_str_op(n->identity, OP_EQ, "erewhon");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node));
+  smartlist_free(sl);
+  nodefamily_free(nf1);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_nodelist_nickname_matches(void *arg)
+{
+  (void)arg;
+  node_t mock_node;
+  routerstatus_t mock_rs;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+
+  strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname));
+  mock_node.rs = &mock_rs;
+  memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN);
+
+#define match(x) tt_assert(node_nickname_matches(&mock_node, (x)))
+#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x)))
+
+  match("evilgeniuses");
+  match("EvilGeniuses");
+  match("EvilGeniuses");
+  match("2e666f7261626574746572746f6d6f72726f772e");
+  match("2E666F7261626574746572746F6D6F72726F772E");
+  match("$2e666f7261626574746572746f6d6f72726f772e");
+  match("$2E666F7261626574746572746F6D6F72726F772E");
+  match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses");
+  match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES");
+
+  no_match("evilgenius");
+  no_match("evilgeniuseses");
+  no_match("evil.genius");
+  no_match("$2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E~fred");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES");
+ done:
+  ;
+}
+
+static void
+test_nodelist_node_nodefamily(void *arg)
+{
+  (void)arg;
+  node_t mock_node1;
+  routerstatus_t mock_rs;
+  microdesc_t mock_md;
+
+  node_t mock_node2;
+  routerinfo_t mock_ri;
+
+  smartlist_t *nodes=smartlist_new();
+
+  memset(&mock_node1, 0, sizeof(mock_node1));
+  memset(&mock_node2, 0, sizeof(mock_node2));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+  memset(&mock_md, 0, sizeof(mock_md));
+  memset(&mock_ri, 0, sizeof(mock_ri));
+
+  mock_node1.rs = &mock_rs;
+  mock_node1.md = &mock_md;
+
+  mock_node2.ri = &mock_ri;
+
+  strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname));
+  mock_ri.nickname = tor_strdup("nodetwo");
+
+  memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN);
+  memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN);
+
+  // empty families.
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain nodes, but not these nodes
+  mock_ri.declared_family = smartlist_new();
+  smartlist_add(mock_ri.declared_family, (char*)"NodeThree");
+  mock_md.family = nodefamily_parse("NodeFour", NULL, 0);
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain one another.
+  smartlist_add(mock_ri.declared_family, (char*)
+                "4e6f64654f6e654e6f6465314e6f64654f6e6531");
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  nodefamily_free(mock_md.family);
+  mock_md.family = nodefamily_parse(
+            "NodeFour "
+            "5365636f6e644e6f64655765277265546573746e", NULL, 0);
+  tt_assert(node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  // Try looking up families now.
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  node_lookup_declared_family(nodes, &mock_node1);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  const node_t *n = smartlist_get(nodes, 0);
+  tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN);
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "nodefour");
+
+  // free, try the other one.
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_clear(nodes);
+
+  node_lookup_declared_family(nodes, &mock_node2);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  n = smartlist_get(nodes, 0);
+  // This gets a truncated hex hex ID since it was looked up by name
+  tt_str_op(n->identity, OP_EQ, "NodeThree");
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  smartlist_free(mock_ri.declared_family);
+  nodefamily_free(mock_md.family);
+  tor_free(mock_ri.nickname);
+  // use tor_free, these aren't real nodes
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_free(nodes);
+}
+
+static void
+test_nodelist_nodefamily_canonicalize(void *arg)
+{
+  (void)arg;
+  char *c = NULL;
+
+  c = nodefamily_canonicalize("", NULL, 0);
+  tt_str_op(c, OP_EQ, "");
+  tor_free(c);
+
+  uint8_t own_id[20];
+  memset(own_id, 0, sizeof(own_id));
+  c = nodefamily_canonicalize(
+           "alice BOB caroL %potrzebie !!!@#@# "
+           "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred "
+           "ffffffffffffffffffffffffffffffffffffffff "
+           "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0);
+  tt_str_op(c, OP_EQ,
+           "!!!@#@# "
+           "$0000000000000000000000000000000000000000 "
+           "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB "
+           "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC "
+           "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF "
+           "%potrzebie "
+           "alice bob carol");
+
+ done:
+  tor_free(c);
+}
+
+/** format_node_description() should return
+ * "Fingerprint~Nickname at IPv4 and [IPv6]".
+ * The nickname and addresses are optional.
+ */
+static void
+test_nodelist_format_node_description(void *arg)
+{
+  char mock_digest[DIGEST_LEN];
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_null_ip;
+  tor_addr_t mock_ipv4;
+  tor_addr_t mock_ipv6;
+
+  char ndesc[NODE_DESC_BUF_LEN];
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(ndesc, 0, sizeof(ndesc));
+  memset(mock_digest, 0, sizeof(mock_digest));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_null_ip, 0, sizeof(mock_null_ip));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ipv6, 0, sizeof(mock_ipv6));
+
+  /* Set variables */
+  memcpy(mock_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  tor_addr_parse(&mock_ipv6, "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Test function with variables */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  /* format node description should use ~ because named is deprecated */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~""TestOR7890123456789");
+
+  /* Try a null IP address, rather than NULL */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_null_ip,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789");
+
+  /* Try some real IP addresses */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               &mock_ipv4,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA at 111.222.233.244");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               tor_addr_to_ipv4h(&mock_ipv4));
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = format_node_description(NULL, NULL, NULL, NULL, 0);
+  tt_str_op(rv, OP_EQ, "<NULL BUFFER>");
+
+  rv = format_node_description(ndesc, NULL, NULL, NULL, 0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(rv, OP_EQ, "<NULL ID DIGEST>");
+
+ done:
+  return;
+}
+
+/** router_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_router_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+  routerinfo_t mock_ri_ipv4;
+  routerinfo_t mock_ri_ipv6;
+  routerinfo_t mock_ri_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_ipv4, 0, sizeof(mock_ri_ipv4));
+  memset(&mock_ri_ipv6, 0, sizeof(mock_ri_ipv6));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerinfos.
+   * mock_nickname is referenced from all 3 routerinfos.
+   * That's ok, all their memory is static. */
+  memcpy(&mock_ri_ipv4, &mock_ri_dual, sizeof(mock_ri_ipv4));
+  memcpy(&mock_ri_ipv6, &mock_ri_dual, sizeof(mock_ri_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_ri_ipv4.ipv6_addr, 0, sizeof(mock_ri_ipv4.ipv6_addr));
+  mock_ri_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = router_describe(&mock_ri_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = router_describe(&mock_ri_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = router_describe(&mock_ri_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = router_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerinfos */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.ri = &mock_ri_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.ri = &mock_ri_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = &mock_ri_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** node_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only and routerstatus-only node_describe() tests are in
+ * test_nodelist_router_describe() and test_nodelist_routerstatus_describe(),
+ * so we can re-use their mocked variables.
+ */
+static void
+test_nodelist_node_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Routerinfos */
+  routerinfo_t mock_ri_dual;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Routerstatuses */
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_dual;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB"
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "Bbb",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "2.2.2.2");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[bbbb::bbbb]");
+
+  /* Create and modify the other routerstatus. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  /* Clear the unnecessary IPv6 address */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+
+  /* Microdescs */
+  microdesc_t mock_md_null;
+  microdesc_t mock_md_ipv6;
+
+  /* Clear variables */
+  memset(&mock_md_null, 0, sizeof(mock_md_null));
+  memset(&mock_md_ipv6, 0, sizeof(mock_md_ipv6));
+
+  /* Set up the microdesc */
+  tor_addr_parse(&mock_md_ipv6.ipv6_addr,
+                 "[eeee::6000:6000]");
+
+  /* Set up the node */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC"
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC",
+         sizeof(mock_node.identity));
+
+  /* Test that the routerinfo and routerstatus work separately, but the
+   * identity comes from the node */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerstatus overrides the routerinfo */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the microdesc IPv6 is used if the routerinfo doesn't have IPv6
+   */
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerinfo doesn't change the results above
+   */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* test NULL handling */
+  rv = node_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  mock_node.ri = NULL;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "<null rs and ri>");
+
+ done:
+  return;
+}
+
+/** routerstatus_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ *
+ * The routerstatus-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_routerstatus_describe(void *arg)
+{
+  tor_addr_t mock_ipv4;
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_ipv6;
+  routerstatus_t mock_rs_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_ipv6, 0, sizeof(mock_rs_ipv6));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "TestOR7890123456789",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerstatuses. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  memcpy(&mock_rs_ipv6, &mock_rs_dual, sizeof(mock_rs_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+  mock_rs_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = routerstatus_describe(&mock_rs_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = routerstatus_describe(&mock_rs_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = routerstatus_describe(&mock_rs_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = routerstatus_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerstatuses */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.rs = &mock_rs_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.rs = &mock_rs_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.rs = &mock_rs_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** extend_info_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ */
+static void
+test_nodelist_extend_info_describe(void *arg)
+{
+  extend_info_t mock_ei_ipv4;
+  extend_info_t mock_ei_ipv6;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ei_ipv4, 0, sizeof(mock_ei_ipv4));
+  memset(&mock_ei_ipv6, 0, sizeof(mock_ei_ipv6));
+
+  /* Set up the IPv4 extend info */
+  memcpy(mock_ei_ipv4.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ei_ipv4.identity_digest));
+  strlcpy(mock_ei_ipv4.nickname, "TestOR7890123456789",
+          sizeof(mock_ei_ipv4.nickname));
+  tor_addr_parse(&mock_ei_ipv4.addr, "111.222.233.244");
+
+  /* Create and modify the other extend info. */
+  memcpy(&mock_ei_ipv6, &mock_ei_ipv4, sizeof(mock_ei_ipv6));
+  tor_addr_parse(&mock_ei_ipv6.addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = extend_info_describe(&mock_ei_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = extend_info_describe(&mock_ei_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Extend infos only have one IP address, so there is no dual case */
+
+  /* test NULL handling */
+  rv = extend_info_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+ done:
+  return;
+}
+
+/** router_get_verbose_nickname() should return "Fingerprint~Nickname"
+ */
+static void
+test_nodelist_router_get_verbose_nickname(void *arg)
+{
+  routerinfo_t mock_ri;
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+
+  char vname[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  (void) arg;
+
+  memset(&mock_ri, 0, sizeof(routerinfo_t));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  mock_ri.nickname = mock_nickname;
+
+  /* verbose nickname should use ~ because named is deprecated */
+  strlcpy(mock_nickname, "TestOR", sizeof(mock_nickname));
+  memcpy(mock_ri.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         DIGEST_LEN);
+  router_get_verbose_nickname(vname, &mock_ri);
+  tt_str_op(vname, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR");
+
+  /* test NULL router handling */
+  router_get_verbose_nickname(vname, NULL);
+  tt_str_op(vname, OP_EQ, "<null>");
+
+  router_get_verbose_nickname(NULL, &mock_ri);
+  router_get_verbose_nickname(NULL, NULL);
+
+ done:
+  return;
+}
+
+static void
+test_nodelist_routerstatus_has_visibly_changed(void *arg)
+{
+  (void)arg;
+  routerstatus_t rs_orig, rs;
+  char *fmt_orig = NULL, *fmt = NULL;
+  memset(&rs_orig, 0, sizeof(rs_orig));
+  strlcpy(rs_orig.nickname, "friendly", sizeof(rs_orig.nickname));
+  memcpy(rs_orig.identity_digest, "abcdefghijklmnopqrst", 20);
+  memcpy(rs_orig.descriptor_digest, "abcdefghijklmnopqrst", 20);
+  rs_orig.addr = 0x7f000001;
+  rs_orig.or_port = 3;
+  rs_orig.published_on = time(NULL);
+  rs_orig.has_bandwidth = 1;
+  rs_orig.bandwidth_kb = 20;
+
+#define COPY() memcpy(&rs, &rs_orig, sizeof(rs))
+#define FORMAT() \
+  STMT_BEGIN \
+    tor_free(fmt_orig);                                                   \
+    tor_free(fmt);                                                        \
+    fmt_orig = routerstatus_format_entry(&rs_orig, NULL, NULL,            \
+                          NS_CONTROL_PORT,                                \
+                          NULL);                                          \
+    fmt = routerstatus_format_entry(&rs, NULL, NULL, NS_CONTROL_PORT,     \
+                          NULL);                                          \
+    tt_assert(fmt_orig);                                                  \
+    tt_assert(fmt);                                                       \
+  STMT_END
+#define ASSERT_SAME() \
+  STMT_BEGIN                                                    \
+    tt_assert(! routerstatus_has_visibly_changed(&rs_orig, &rs));       \
+    FORMAT();                                                   \
+    tt_str_op(fmt_orig, OP_EQ, fmt);                            \
+    COPY();                                                     \
+  STMT_END
+#define ASSERT_CHANGED() \
+  STMT_BEGIN                                                    \
+    tt_assert(routerstatus_has_visibly_changed(&rs_orig, &rs));         \
+    FORMAT();                                                   \
+    tt_str_op(fmt_orig, OP_NE, fmt);                            \
+    COPY();                                                     \
+  STMT_END
+#define ASSERT_CHANGED_NO_FORMAT() \
+  STMT_BEGIN                                                    \
+    tt_assert(routerstatus_has_visibly_changed(&rs_orig, &rs));         \
+    COPY();                                                     \
+  STMT_END
+
+  COPY();
+  ASSERT_SAME();
+
+  rs.addr = 0x7f000002;
+  ASSERT_CHANGED();
+
+  strlcpy(rs.descriptor_digest, "hello world", sizeof(rs.descriptor_digest));
+  ASSERT_CHANGED();
+
+  strlcpy(rs.nickname, "fr1end1y", sizeof(rs.nickname));
+  ASSERT_CHANGED();
+
+  rs.published_on += 3600;
+  ASSERT_CHANGED();
+
+  rs.or_port = 55;
+  ASSERT_CHANGED();
+
+  rs.dir_port = 9999;
+  ASSERT_CHANGED();
+
+  tor_addr_parse(&rs.ipv6_addr, "1234::56");
+  ASSERT_CHANGED();
+
+  tor_addr_parse(&rs_orig.ipv6_addr, "1234::56");
+  rs_orig.ipv6_orport = 99;
+  COPY();
+  rs.ipv6_orport = 22;
+  ASSERT_CHANGED();
+
+  rs.is_authority = 1;
+  ASSERT_CHANGED();
+
+  rs.is_exit = 1;
+  ASSERT_CHANGED();
+
+  rs.is_stable = 1;
+  ASSERT_CHANGED();
+
+  rs.is_fast = 1;
+  ASSERT_CHANGED();
+
+  rs.is_flagged_running = 1;
+  ASSERT_CHANGED();
+
+  // This option is obsolete and not actually formatted.
+  rs.is_named = 1;
+  ASSERT_CHANGED_NO_FORMAT();
+
+  // This option is obsolete and not actually formatted.
+  rs.is_unnamed = 1;
+  ASSERT_CHANGED_NO_FORMAT();
+
+  rs.is_valid = 1;
+  ASSERT_CHANGED();
+
+  rs.is_possible_guard = 1;
+  ASSERT_CHANGED();
+
+  rs.is_bad_exit = 1;
+  ASSERT_CHANGED();
+
+  rs.is_hs_dir = 1;
+  ASSERT_CHANGED();
+
+  rs.is_v2_dir = 1;
+  ASSERT_CHANGED();
+
+  rs.is_staledesc = 1;
+  ASSERT_CHANGED();
+
+  // Setting this to zero crashes us with an assertion failure in
+  // routerstatus_format_entry() if we don't have a descriptor.
+  rs.has_bandwidth = 0;
+  ASSERT_CHANGED_NO_FORMAT();
+
+  // Does not actually matter; not visible to controller.
+  rs.has_exitsummary = 1;
+  ASSERT_SAME();
+
+  // Does not actually matter; not visible to the controller.
+  rs.bw_is_unmeasured = 1;
+  ASSERT_SAME();
+
+  rs.bandwidth_kb = 2000;
+  ASSERT_CHANGED();
+
+  // not visible to the controller.
+  rs.has_guardfraction = 1;
+  rs.guardfraction_percentage = 22;
+  ASSERT_SAME();
+
+  // not visible to the controller.
+  rs_orig.has_guardfraction = 1;
+  rs_orig.guardfraction_percentage = 20;
+  COPY();
+  rs.guardfraction_percentage = 25;
+  ASSERT_SAME();
+
+  // not visible to the controller.
+  rs.exitsummary = (char*)"accept 1-2";
+  ASSERT_SAME();
+
+ done:
+#undef COPY
+#undef ASSERT_SAME
+#undef ASSERT_CHANGED
+  tor_free(fmt_orig);
+  tor_free(fmt);
+  return;
+}
+
 #define NODE(name, flags) \
   { #name, test_nodelist_##name, (flags), NULL, NULL }
 
@@ -239,6 +1418,18 @@ struct testcase_t nodelist_tests[] = {
   NODE(node_get_verbose_nickname_not_named, TT_FORK),
   NODE(node_is_dir, TT_FORK),
   NODE(ed_id, TT_FORK),
+  NODE(nodefamily, TT_FORK),
+  NODE(nodefamily_parse_err, TT_FORK),
+  NODE(nodefamily_lookup, TT_FORK),
+  NODE(nickname_matches, 0),
+  NODE(node_nodefamily, TT_FORK),
+  NODE(nodefamily_canonicalize, 0),
+  NODE(format_node_description, 0),
+  NODE(router_describe, 0),
+  NODE(node_describe, 0),
+  NODE(routerstatus_describe, 0),
+  NODE(extend_info_describe, 0),
+  NODE(router_get_verbose_nickname, 0),
+  NODE(routerstatus_has_visibly_changed, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c
index 68b6927f56..a1508d0afc 100644
--- a/src/test/test_ntor_cl.c
+++ b/src/test/test_ntor_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_oom.c b/src/test/test_oom.c
index b813fa43a3..51c237ec2e 100644
--- a/src/test/test_oom.c
+++ b/src/test/test_oom.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOM handling logic */
@@ -8,7 +8,7 @@
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "lib/evloop/compat_libevent.h"
 #include "core/mainloop/connection.h"
diff --git a/src/test/test_oos.c b/src/test/test_oos.c
index 815feda7ce..f8c712a6b6 100644
--- a/src/test/test_oos.c
+++ b/src/test/test_oos.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOS handler */
diff --git a/src/test/test_options.c b/src/test/test_options.c
index 0e52967a23..8e0d19f126 100644
--- a/src/test/test_options.c
+++ b/src/test/test_options.c
@@ -1,19 +1,29 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
+#define RELAY_CONFIG_PRIVATE
+#define LOG_PRIVATE
+#define ROUTERSET_PRIVATE
 #include "core/or/or.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "app/config/config.h"
+#include "feature/dirauth/dirauth_config.h"
+#include "feature/dirauth/dirauth_options_st.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/relay/relay_config.h"
 #include "test/test.h"
 #include "lib/geoip/geoip.h"
 
-#define ROUTERSET_PRIVATE
 #include "feature/nodelist/routerset.h"
 #include "core/mainloop/mainloop.h"
+#include "app/main/subsysmgr.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
+#include "lib/crypt_ops/crypto_options_st.h"
+#include "lib/crypt_ops/crypto_sys.h"
 
 #include "lib/sandbox/sandbox.h"
 #include "lib/memarea/memarea.h"
@@ -21,24 +31,23 @@
 #include "lib/encoding/confline.h"
 #include "core/or/policies.h"
 #include "test/test_helpers.h"
+#include "test/opts_test_helpers.h"
 #include "lib/net/resolve.h"
 
 #ifdef HAVE_SYS_PARAM_H
 #include <sys/param.h>
 #endif
 
-#define NS_MODULE test_options
-
 typedef struct {
   int severity;
-  uint32_t domain;
+  log_domain_mask_t domain;
   char *msg;
 } logmsg_t;
 
 static smartlist_t *messages = NULL;
 
 static void
-log_cback(int severity, uint32_t domain, const char *msg)
+log_cback(int severity, log_domain_mask_t domain, const char *msg)
 {
   logmsg_t *x = tor_malloc(sizeof(*x));
   x->severity = severity;
@@ -54,9 +63,9 @@ setup_log_callback(void)
 {
   log_severity_list_t lst;
   memset(&lst, 0, sizeof(lst));
-  lst.masks[LOG_ERR - LOG_ERR] = ~0;
-  lst.masks[LOG_WARN - LOG_ERR] = ~0;
-  lst.masks[LOG_NOTICE - LOG_ERR] = ~0;
+  lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = LD_ALL_DOMAINS;
   add_callback_log(&lst, log_cback);
   mark_logs_temp();
 }
@@ -90,16 +99,57 @@ clear_log_messages(void)
   messages = NULL;
 }
 
-#define setup_options(opt,dflt)              \
+#define setup_options(opt)                   \
   do {                                       \
     opt = options_new();                     \
     opt->command = CMD_RUN_TOR;              \
     options_init(opt);                       \
-                                             \
-    dflt = config_dup(&options_format, opt); \
-    clear_log_messages();                    \
   } while (0)
 
+#ifdef COCCI
+
+#define ENABLE_AUTHORITY_MIN ""
+#define ENABLE_AUTHORITY_V3_MIN ""
+#define ENABLE_AUTHORITY_BRIDGE_MIN ""
+#define AUTHORITY_OPT_REQ_ ""
+#define ENABLE_AUTHORITY ""
+#define ENABLE_AUTHORITY_V3 ""
+#define ENABLE_AUTHORITY_BRIDGE ""
+
+#else /* !defined(COCCI) */
+
+#define ENABLE_AUTHORITY_MIN \
+  "AuthoritativeDirectory 1\n"
+
+#define ENABLE_AUTHORITY_V3_MIN \
+  ENABLE_AUTHORITY_MIN \
+  "V3AuthoritativeDir 1\n"
+
+#define ENABLE_AUTHORITY_BRIDGE_MIN \
+  ENABLE_AUTHORITY_MIN \
+  "BridgeAuthoritativeDir 1\n"
+
+#define AUTHORITY_OPT_REQ_ \
+  "Address 192.0.2.111\n" \
+  "ContactInfo a@example.org\n" \
+  "DirPort 1025\n" \
+  "ORPort 1026\n"
+
+/* Not actually valid: requires v3 / bridge */
+#define ENABLE_AUTHORITY \
+  ENABLE_AUTHORITY_MIN \
+  AUTHORITY_OPT_REQ_
+
+#define ENABLE_AUTHORITY_V3 \
+  ENABLE_AUTHORITY_V3_MIN \
+  AUTHORITY_OPT_REQ_
+
+#define ENABLE_AUTHORITY_BRIDGE \
+  ENABLE_AUTHORITY_BRIDGE_MIN \
+  AUTHORITY_OPT_REQ_
+
+#endif /* defined(COCCI) */
+
 #define VALID_DIR_AUTH "DirAuthority dizum orport=443 v3ident=E8A9C45"  \
   "EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212:80 7EA6 EAD6 FD83" \
   " 083C 538F 4403 8BBF A077 587D D755\n"
@@ -179,12 +229,11 @@ test_options_validate_impl(const char *configuration,
                            int phase)
 {
   or_options_t *opt=NULL;
-  or_options_t *dflt;
   config_line_t *cl=NULL;
   char *msg=NULL;
   int r;
 
-  setup_options(opt, dflt);
+  setup_options(opt);
 
   r = config_get_lines(configuration, &cl, 1);
   if (phase == PH_GETLINES) {
@@ -196,7 +245,7 @@ test_options_validate_impl(const char *configuration,
   if (r)
     goto done;
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   if (phase == PH_ASSIGN) {
     if (test_options_checkmsgs(configuration, expect_errmsg,
                                expect_log_severity,
@@ -207,7 +256,7 @@ test_options_validate_impl(const char *configuration,
   if (r)
     goto done;
 
-  r = options_validate(NULL, opt, dflt, 0, &msg);
+  r = options_validate(NULL, opt, &msg);
   if (phase == PH_VALIDATE) {
     if (test_options_checkmsgs(configuration, expect_errmsg,
                                expect_log_severity,
@@ -221,7 +270,6 @@ test_options_validate_impl(const char *configuration,
   policies_free_all();
   config_free_lines(cl);
   or_options_free(opt);
-  or_options_free(dflt);
   tor_free(msg);
   clear_log_messages();
 }
@@ -241,6 +289,7 @@ test_options_validate(void *arg)
   (void)arg;
   setup_log_callback();
   sandbox_disable_getaddrinfo_cache();
+  mock_hostname_resolver();
 
   WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE);
 
@@ -258,14 +307,10 @@ test_options_validate(void *arg)
   WANT_ERR("BridgeRelay 1\nDirCache 0",
            "We're a bridge but DirCache is disabled.", PH_VALIDATE);
 
-  WANT_ERR_LOG("HeartbeatPeriod 21 snarks",
-               "Interval 'HeartbeatPeriod 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
-               PH_ASSIGN);
-  WANT_ERR_LOG("LogTimeGranularity 21 snarks",
-               "Msec interval 'LogTimeGranularity 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
-               PH_ASSIGN);
+  WANT_ERR("HeartbeatPeriod 21 snarks",
+           "Unknown unit in 21 snarks", PH_ASSIGN);
+  WANT_ERR("LogTimeGranularity 21 snarks",
+           "Unknown unit in 21 snarks", PH_ASSIGN);
   OK("HeartbeatPeriod 1 hour", PH_VALIDATE);
   OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE);
 
@@ -278,6 +323,7 @@ test_options_validate(void *arg)
 
   close_temp_logs();
   clear_log_messages();
+  unmock_hostname_resolver();
   return;
 }
 
@@ -287,20 +333,18 @@ test_have_enough_mem_for_dircache(void *arg)
 {
   (void)arg;
   or_options_t *opt=NULL;
-  or_options_t *dflt=NULL;
   config_line_t *cl=NULL;
   char *msg=NULL;
   int r;
   const char *configuration = "ORPort 8080\nDirCache 1", *expect_errmsg;
 
-  setup_options(opt, dflt);
+  setup_options(opt);
   setup_log_callback();
-  (void)dflt;
 
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled */
@@ -323,7 +367,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled, Bridge */
@@ -346,7 +390,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 200 MB RAM available, DirCache disabled */
@@ -369,7 +413,6 @@ test_have_enough_mem_for_dircache(void *arg)
  done:
   if (msg)
     tor_free(msg);
-  or_options_free(dflt);
   or_options_free(opt);
   config_free_lines(cl);
   return;
@@ -383,33 +426,8 @@ fixed_get_uname(void)
   return fixed_get_uname_result;
 }
 
-#define TEST_OPTIONS_OLD_VALUES   "TestingV3AuthInitialVotingInterval 1800\n" \
-  "ClientBootstrapConsensusMaxInProgressTries 3\n" \
-  "TestingV3AuthInitialVoteDelay 300\n"   \
-  "TestingV3AuthInitialDistDelay 300\n" \
-  "TestingClientMaxIntervalWithoutRequest 600\n" \
-  "TestingDirConnectionMaxStall 600\n" \
-
-#define TEST_OPTIONS_DEFAULT_VALUES TEST_OPTIONS_OLD_VALUES \
-  "MaxClientCircuitsPending 1\n"                                        \
-  "RendPostPeriod 1000\n"                                               \
-  "KeepAlivePeriod 1\n"                                                 \
-  "ConnLimit 1\n"                                                       \
-  "V3AuthVotingInterval 300\n"                                          \
-  "V3AuthVoteDelay 20\n"                                                \
-  "V3AuthDistDelay 20\n"                                                \
-  "V3AuthNIntervalsValid 3\n"                                           \
-  "ClientUseIPv4 1\n"                                                   \
-  "VirtualAddrNetworkIPv4 127.192.0.0/10\n"                             \
-  "VirtualAddrNetworkIPv6 [FE80::]/10\n"                                \
-  "UseEntryGuards 1\n"                                                  \
-  "Schedulers Vanilla\n"                                                \
-  "ClientDNSRejectInternalAddresses 1\n"
-
 typedef struct {
-  or_options_t *old_opt;
   or_options_t *opt;
-  or_options_t *def_opt;
 } options_test_data_t;
 
 static void free_options_test_data(options_test_data_t *td);
@@ -422,17 +440,12 @@ get_options_test_data(const char *conf)
   config_line_t *cl=NULL;
   options_test_data_t *result = tor_malloc(sizeof(options_test_data_t));
   result->opt = options_new();
-  result->old_opt = options_new();
-  result->def_opt = options_new();
 
-  // XXX: Really, all of these options should be set to defaults
-  // with options_init(), but about a dozen tests break when I do that.
-  // Being kinda lame and just fixing the immedate breakage for now..
-  result->opt->ConnectionPadding = -1; // default must be "auto"
+  options_init(result->opt);
 
   rv = config_get_lines(conf, &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->opt, cl, 0, &msg);
+  rv = config_assign(get_options_mgr(), result->opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
@@ -441,13 +454,7 @@ get_options_test_data(const char *conf)
   config_free_lines(cl);
   result->opt->LogTimeGranularity = 1;
   result->opt->TokenBucketRefillInterval = 1;
-  rv = config_get_lines(TEST_OPTIONS_OLD_VALUES, &cl, 1);
-  tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->def_opt, cl, 0, &msg);
-  if (msg) {
-    /* Display the parse error message by comparing it with an empty string */
-    tt_str_op(msg, OP_EQ, "");
-  }
+  rv = config_get_lines("", &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
 
  done:
@@ -466,9 +473,7 @@ static void
 free_options_test_data(options_test_data_t *td)
 {
   if (!td) return;
-  or_options_free(td->old_opt);
   or_options_free(td->opt);
-  or_options_free(td->def_opt);
   tor_free(td);
 }
 
@@ -491,7 +496,7 @@ test_options_validate__uname_for_server(void *ignored)
 
   MOCK(get_uname, fixed_get_uname);
   fixed_get_uname_result = "Windows 95";
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Tor is running as a server, but you"
            " are running Windows 95; this probably won't work. See https://www"
            ".torproject.org/docs/faq.html#BestOSForRelay for details.\n");
@@ -499,7 +504,7 @@ test_options_validate__uname_for_server(void *ignored)
 
   fixed_get_uname_result = "Windows 98";
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Tor is running as a server, but you"
            " are running Windows 98; this probably won't work. See https://www"
            ".torproject.org/docs/faq.html#BestOSForRelay for details.\n");
@@ -507,7 +512,7 @@ test_options_validate__uname_for_server(void *ignored)
 
   fixed_get_uname_result = "Windows Me";
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Tor is running as a server, but you"
            " are running Windows Me; this probably won't work. See https://www"
            ".torproject.org/docs/faq.html#BestOSForRelay for details.\n");
@@ -515,8 +520,8 @@ test_options_validate__uname_for_server(void *ignored)
 
   fixed_get_uname_result = "Windows 2000";
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  expect_no_log_entry();
+  options_validate(NULL, tdata->opt, &msg);
+  expect_no_log_msg("Tor is running as a server, but you ");
   tor_free(msg);
 
  done:
@@ -539,7 +544,7 @@ test_options_validate__outbound_addresses(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                     "OutboundBindAddress xxyy!!!sdfaf");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Multiple outbound bind addresses configured: "
                         "xxyy!!!sdfaf");
@@ -576,7 +581,7 @@ test_options_validate__data_directory(void *ignored)
                                                 "ONGLONGlongreallylongLONG"
                                                 "LONG"); // 440 characters
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Invalid DataDirectory");
 
@@ -594,7 +599,7 @@ test_options_validate__nickname(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                         "Nickname ThisNickNameIsABitTooLong");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Nickname 'ThisNickNameIsABitTooLong', nicknames must be between "
@@ -604,16 +609,14 @@ test_options_validate__nickname(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("Nickname AMoreValidNick");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("DataDirectory /tmp/somewhere");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
 
  done:
   free_options_test_data(tdata);
@@ -631,8 +634,8 @@ test_options_validate__contactinfo(void *ignored)
   setup_capture_of_logs(LOG_DEBUG);
   tdata->opt->ContactInfo = NULL;
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
            "Your ContactInfo config option is not set. Please strongly "
            "consider setting it, so we can contact you if your relay is "
@@ -645,8 +648,8 @@ test_options_validate__contactinfo(void *ignored)
   tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "ContactInfo hella@example.org");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
            "Your ContactInfo config option is not set. Please strongly "
            "consider setting it, so we can contact you if your relay is "
@@ -673,50 +676,53 @@ test_options_validate__logs(void *ignored)
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
-  tt_str_op(tdata->opt->Logs->value, OP_EQ, "notice stdout");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
   quiet_level = 1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
-  tt_str_op(tdata->opt->Logs->value, OP_EQ, "warn stdout");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
   quiet_level = 2;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 1, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+#ifdef _WIN32
+  /* Can't RunAsDaemon on Windows. */
   tt_int_op(ret, OP_EQ, -1);
+#else
+  tt_int_op(ret, OP_EQ, 0);
+#endif /* defined(_WIN32) */
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -724,7 +730,7 @@ test_options_validate__logs(void *ignored)
   config_line_t *cl=NULL;
   config_get_lines("Log foo", &cl, 1);
   tdata->opt->Logs = cl;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op((intptr_t)tdata->opt->Logs, OP_EQ, (intptr_t)cl);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -752,13 +758,14 @@ test_options_validate__authdir(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_INFO);
   options_test_data_t *tdata = get_options_test_data(
-                                 "AuthoritativeDirectory 1\n"
+                                 ENABLE_AUTHORITY_V3_MIN
                                  "Address this.should.not!exist!.example.org");
+  const dirauth_options_t *da_opt;
 
   sandbox_disable_getaddrinfo_cache();
 
   MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Failed to resolve/guess local address. See logs for"
@@ -768,246 +775,217 @@ test_options_validate__authdir(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Authoritative directory servers must set "
                         "ContactInfo");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Authoritative directory servers must set ContactInfo");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_MIN
                                 "Address 100.200.10.1\n"
                                 "TestingTorNetwork 1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
             "AuthoritativeDir is set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY);
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
             "AuthoritativeDir is set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "RecommendedVersions 1.2, 3.14\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "RecommendedVersions 1.2, 3.14\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "1.2, 3.14");
-  tt_str_op(tdata->opt->RecommendedServerVersions->value, OP_EQ, "1.2, 3.14");
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
+  tt_str_op(da_opt->RecommendedClientVersions->value, OP_EQ, "1.2, 3.14");
+  tt_str_op(da_opt->RecommendedServerVersions->value, OP_EQ, "1.2, 3.14");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
-                                "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedServerVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "25");
-  tt_str_op(tdata->opt->RecommendedServerVersions->value, OP_EQ, "4.18");
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
+  tt_str_op(da_opt->RecommendedClientVersions->value, OP_EQ, "25");
+  tt_str_op(da_opt->RecommendedServerVersions->value, OP_EQ, "4.18");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY
                                 "VersioningAuthoritativeDirectory 1\n"
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
-                                "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedServerVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
             "AuthoritativeDir is set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "VersioningAuthoritativeDirectory 1\n"
-                                "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedServerVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
             "Recommended*Versions.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "VersioningAuthoritativeDirectory 1\n"
-                                "RecommendedClientVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedClientVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
             "Recommended*Versions.");
   tor_free(msg);
+  da_opt = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "UseEntryGuards 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "UseEntryGuards 1\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Authoritative directory servers "
             "can't set UseEntryGuards. Disabling.\n");
   tt_int_op(tdata->opt->UseEntryGuards, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "DownloadExtraInfo 0\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Authoritative directories always try"
             " to download extra-info documents. Setting DownloadExtraInfo.\n");
   tt_int_op(tdata->opt->DownloadExtraInfo, OP_EQ, 1);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "DownloadExtraInfo 1\n"
-                                "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3BandwidthsFile non-existent-file\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  expect_no_log_msg("Authoritative directories always try"
-            " to download extra-info documents. Setting DownloadExtraInfo.\n");
-  tt_int_op(tdata->opt->DownloadExtraInfo, OP_EQ, 1);
+  options_validate(NULL, tdata->opt, &msg);
+  expect_log_msg("Can't open bandwidth file at configured location: "
+                 "non-existent-file\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "GuardfractionFile non-existent-file\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
-            "AuthoritativeDir is set.");
+  options_validate(NULL, tdata->opt, &msg);
+  expect_log_msg("Cannot open guardfraction file 'non-existent-file'. "
+                 "Failing.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existent-file\n");
+                                "ORPort 2000\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Running as authoritative directory, but no DirPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_BRIDGE_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existent-file\n");
+                                "ORPort 2000\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Running as authoritative directory, but no DirPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existent-file\n");
+                                "DirPort 999\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no DirPort set.");
+            "Running as authoritative directory, but no ORPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_BRIDGE_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existent-file\n");
+                                "DirPort 999\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no DirPort set.");
+            "Running as authoritative directory, but no ORPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "ClientOnly 1\n");
+  /* We have to call the dirauth-specific function, and fake port parsing,
+   * to hit this case */
+  tdata->opt->DirPort_set = 1;
+  tdata->opt->ORPort_set = 1;
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate_dirauth_mode(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no DirPort set.");
+  tt_str_op(msg, OP_EQ, "Running as authoritative directory, "
+            "but ClientOnly also set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "DirPort 999\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_BRIDGE
+                                "ClientOnly 1\n");
+  /* We have to call the dirauth-specific function, and fake port parsing,
+   * to hit this case */
+  tdata->opt->DirPort_set = 1;
+  tdata->opt->ORPort_set = 1;
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate_dirauth_mode(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no ORPort set.");
+  tt_str_op(msg, OP_EQ, "Running as authoritative directory, "
+            "but ClientOnly also set.");
   tor_free(msg);
 
-  // TODO: This case can't be reached, since clientonly is used to
-  // check when parsing port lines as well.
-  /* free_options_test_data(tdata); */
-  /* tdata = get_options_test_data("AuthoritativeDirectory 1\n" */
-  /*                               "Address 100.200.10.1\n" */
-  /*                               "DirPort 999\n" */
-  /*                               "ORPort 888\n" */
-  /*                               "ClientOnly 1\n" */
-  /*                               "BridgeAuthoritativeDir 1\n" */
-  /*                               "ContactInfo hello@hello.com\n" ); */
-  /* mock_clean_saved_logs(); */
-  /* ret = options_validate(tdata->old_opt, tdata->opt, */
-  /*                        tdata->def_opt, 0, &msg); */
-  /* tt_int_op(ret, OP_EQ, -1); */
-  /* tt_str_op(msg, OP_EQ, "Running as authoritative directory, " */
-  /*           "but ClientOnly also set."); */
-
  done:
   teardown_capture_of_logs();
   //  sandbox_free_getaddrinfo_cache();
@@ -1020,6 +998,7 @@ test_options_validate__relay_with_hidden_services(void *ignored)
 {
   (void)ignored;
   char *msg;
+  int ret;
   setup_capture_of_logs(LOG_DEBUG);
   options_test_data_t *tdata = get_options_test_data(
                                   "ORPort 127.0.0.1:5555\n"
@@ -1028,7 +1007,8 @@ test_options_validate__relay_with_hidden_services(void *ignored)
                                   "HiddenServicePort 80 127.0.0.1:8080\n"
                                                      );
 
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "Tor is currently configured as a relay and a hidden service. "
             "That's not very secure: you should probably run your hidden servi"
@@ -1041,27 +1021,25 @@ test_options_validate__relay_with_hidden_services(void *ignored)
   tor_free(msg);
 }
 
-// TODO: it doesn't seem possible to hit the case of having no port lines at
-// all, since there will be a default created for SocksPort
-/* static void */
-/* test_options_validate__ports(void *ignored) */
-/* { */
-/*   (void)ignored; */
-/*   int ret; */
-/*   char *msg; */
-/*   setup_capture_of_logs(LOG_WARN); */
-/*   options_test_data_t *tdata = get_options_test_data(""); */
-/*   ret = options_validate(tdata->old_opt, tdata->opt, */
-/*                          tdata->def_opt, 0, &msg); */
-/*   expect_log_msg("SocksPort, TransPort, NATDPort, DNSPort, and ORPort " */
-/*           "are all undefined, and there aren't any hidden services " */
-/*           "configured. " */
-/*           " Tor will still run, but probably won't do anything.\n"); */
-/*  done: */
-/*   teardown_capture_of_logs(); */
-/*   free_options_test_data(tdata); */
-/*   tor_free(msg); */
-/* } */
+static void
+test_options_validate__listen_ports(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  char *msg;
+  setup_capture_of_logs(LOG_WARN);
+  options_test_data_t *tdata = get_options_test_data("SOCKSPort 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg("SocksPort, TransPort, NATDPort, DNSPort, and ORPort "
+                 "are all undefined, and there aren't any hidden services "
+                 "configured. "
+                 " Tor will still run, but probably won't do anything.\n");
+ done:
+  teardown_capture_of_logs();
+  free_options_test_data(tdata);
+  tor_free(msg);
+}
 
 static void
 test_options_validate__transproxy(void *ignored)
@@ -1075,31 +1053,31 @@ test_options_validate__transproxy(void *ignored)
   // Test default trans proxy
   tdata = get_options_test_data("TransProxyType default\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_DEFAULT);
   tor_free(msg);
 
   // Test pf-divert trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType pf-divert\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
-#if !defined(OpenBSD) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined(DARWIN)
   tt_str_op(msg, OP_EQ,
           "pf-divert is a OpenBSD-specific and OS X/Darwin-specific feature.");
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_PF_DIVERT);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without "
             "any valid TransPort.");
-#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
+#endif /* !defined(OpenBSD) && !defined(DARWIN) */
   tor_free(msg);
 
   // Test tproxy trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType tproxy\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
 #if !defined(__linux__)
@@ -1114,7 +1092,7 @@ test_options_validate__transproxy(void *ignored)
   // Test ipfw trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType ipfw\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
 #ifndef KERNEL_MAY_SUPPORT_IPFW
@@ -1130,7 +1108,7 @@ test_options_validate__transproxy(void *ignored)
   // Test unknown trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType non-existent\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value for TransProxyType");
   tor_free(msg);
@@ -1142,39 +1120,35 @@ test_options_validate__transproxy(void *ignored)
 #if defined(__linux__)
   tdata = get_options_test_data("TransProxyType tproxy\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
 #elif defined(KERNEL_MAY_SUPPORT_IPFW)
   tdata = get_options_test_data("TransProxyType ipfw\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 #elif defined(OpenBSD)
   tdata = get_options_test_data("TransProxyType pf-divert\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 #elif defined(__NetBSD__)
   tdata = get_options_test_data("TransProxyType default\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 #endif /* defined(__linux__) || ... */
 
   // Assert that a test has run for some TransProxyType
   tt_assert(tdata);
 
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   tdata = get_options_test_data("TransPort 127.0.0.1:555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TransPort is disabled in this build.");
   tor_free(msg);
@@ -1185,13 +1159,14 @@ test_options_validate__transproxy(void *ignored)
   tor_free(msg);
 }
 
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static country_t opt_tests_geoip_get_country(const char *country);
+ATTR_UNUSED static int opt_tests_geoip_get_country_called = 0;
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+opt_tests_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  opt_tests_geoip_get_country_called++;
 
   return 1;
 }
@@ -1201,7 +1176,8 @@ test_options_validate__exclude_nodes(void *ignored)
 {
   (void)ignored;
 
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       opt_tests_geoip_get_country);
 
   int ret;
   char *msg;
@@ -1209,8 +1185,8 @@ test_options_validate__exclude_nodes(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                                   "ExcludeExitNodes {us}\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(tdata->opt->ExcludeExitNodesUnion_->list), OP_EQ, 1);
   tt_str_op((char *)
             (smartlist_get(tdata->opt->ExcludeExitNodesUnion_->list, 0)),
@@ -1219,8 +1195,8 @@ test_options_validate__exclude_nodes(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(tdata->opt->ExcludeExitNodesUnion_->list), OP_EQ, 1);
   tt_str_op((char *)
             (smartlist_get(tdata->opt->ExcludeExitNodesUnion_->list, 0)),
@@ -1230,8 +1206,8 @@ test_options_validate__exclude_nodes(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n"
                                 "ExcludeExitNodes {us} {cn}\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(tdata->opt->ExcludeExitNodesUnion_->list), OP_EQ, 2);
   tt_str_op((char *)
             (smartlist_get(tdata->opt->ExcludeExitNodesUnion_->list, 0)),
@@ -1245,8 +1221,8 @@ test_options_validate__exclude_nodes(void *ignored)
   tdata = get_options_test_data("ExcludeNodes {cn}\n"
                                 "StrictNodes 1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "You have asked to exclude certain relays from all positions "
             "in your circuits. Expect hidden services and other Tor "
@@ -1256,8 +1232,8 @@ test_options_validate__exclude_nodes(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "You have asked to exclude certain relays from all positions "
             "in your circuits. Expect hidden services and other Tor "
@@ -1265,7 +1241,7 @@ test_options_validate__exclude_nodes(void *ignored)
   tor_free(msg);
 
  done:
-  NS_UNMOCK(geoip_get_country);
+  UNMOCK(geoip_get_country);
   teardown_capture_of_logs();
   free_options_test_data(tdata);
   tor_free(msg);
@@ -1281,8 +1257,8 @@ test_options_validate__node_families(void *ignored)
                                      "NodeFamily flux, flax\n"
                                      "NodeFamily somewhere\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->NodeFamilySets);
   tt_int_op(smartlist_len(tdata->opt->NodeFamilySets), OP_EQ, 2);
   tt_str_op((char *)(smartlist_get(
@@ -1299,15 +1275,15 @@ test_options_validate__node_families(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(!tdata->opt->NodeFamilySets);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("NodeFamily !flux\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_assert(tdata->opt->NodeFamilySets);
   tt_int_op(smartlist_len(tdata->opt->NodeFamilySets), OP_EQ, 0);
@@ -1327,14 +1303,14 @@ test_options_validate__token_bucket(void *ignored)
   options_test_data_t *tdata = get_options_test_data("");
 
   tdata->opt->TokenBucketRefillInterval = 0;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TokenBucketRefillInterval must be between 1 and 1000 inclusive.");
   tor_free(msg);
 
   tdata->opt->TokenBucketRefillInterval = 1001;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TokenBucketRefillInterval must be between 1 and 1000 inclusive.");
@@ -1346,29 +1322,6 @@ test_options_validate__token_bucket(void *ignored)
 }
 
 static void
-test_options_validate__recommended_packages(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_WARN);
-  options_test_data_t *tdata = get_options_test_data(
-            "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n"
-            "RecommendedPackages invalid-package-line\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg("Invalid RecommendedPackage line "
-            "invalid-package-line will be ignored\n");
-
- done:
-  escaped(NULL); // This will free the leaking memory from the previous escaped
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__fetch_dir(void *ignored)
 {
   (void)ignored;
@@ -1378,7 +1331,7 @@ test_options_validate__fetch_dir(void *ignored)
                                             "FetchDirInfoExtraEarly 1\n"
                                             "FetchDirInfoEarly 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "FetchDirInfoExtraEarly requires that you"
             " also set FetchDirInfoEarly");
@@ -1388,10 +1341,8 @@ test_options_validate__fetch_dir(void *ignored)
   tdata = get_options_test_data("FetchDirInfoExtraEarly 1\n"
                                 "FetchDirInfoEarly 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_NE, "FetchDirInfoExtraEarly requires that you"
-            " also set FetchDirInfoEarly");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1408,7 +1359,7 @@ test_options_validate__conn_limit(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                             "ConnLimit 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
   tor_free(msg);
@@ -1416,10 +1367,8 @@ test_options_validate__conn_limit(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("ConnLimit 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "MaxClientCircuitsPending must be between 1 and 1024, "
-            "but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1442,11 +1391,10 @@ test_options_validate__paths_needed(void *ignored)
 
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
-                                      "PathsNeededToBuildCircuits 0.1\n"
-                                      "ConnLimit 1\n");
+                                      "PathsNeededToBuildCircuits 0.1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->PathsNeededToBuildCircuits > 0.24 &&
             tdata->opt->PathsNeededToBuildCircuits < 0.26);
   expect_log_msg("PathsNeededToBuildCircuits is too low. "
@@ -1455,11 +1403,10 @@ test_options_validate__paths_needed(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.99\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.99\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->PathsNeededToBuildCircuits > 0.94 &&
             tdata->opt->PathsNeededToBuildCircuits < 0.96);
   expect_log_msg("PathsNeededToBuildCircuits is "
@@ -1468,14 +1415,13 @@ test_options_validate__paths_needed(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.91\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.91\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->PathsNeededToBuildCircuits > 0.90 &&
             tdata->opt->PathsNeededToBuildCircuits < 0.92);
-  expect_no_log_entry();
+  expect_no_log_msg_containing("PathsNeededToBuildCircuits");
   tor_free(msg);
 
  done:
@@ -1495,32 +1441,28 @@ test_options_validate__max_client_circuits(void *ignored)
   int ret;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                           "MaxClientCircuitsPending 0\n"
-                                           "ConnLimit 1\n");
+                                             "MaxClientCircuitsPending 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "MaxClientCircuitsPending must be between 1 and 1024,"
             " but was set to 0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("MaxClientCircuitsPending 1025\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("MaxClientCircuitsPending 1025\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "MaxClientCircuitsPending must be between 1 and 1024,"
             " but was set to 1025");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("MaxClientCircuitsPending 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "KeepalivePeriod option must be positive.");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1534,23 +1476,18 @@ test_options_validate__ports(void *ignored)
   (void)ignored;
   int ret;
   char *msg;
-  options_test_data_t *tdata = get_options_test_data(
-                                      "FirewallPorts 65537\n"
-                                      "MaxClientCircuitsPending 1\n"
-                                      "ConnLimit 1\n");
+  options_test_data_t *tdata = get_options_test_data("FirewallPorts 65537\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '65537' out of range in FirewallPorts");
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("FirewallPorts 1\n"
-                                "LongLivedPorts 124444\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "LongLivedPorts 124444\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '124444' out of range in LongLivedPorts");
   tor_free(msg);
@@ -1558,11 +1495,9 @@ test_options_validate__ports(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 2\n"
-                                "RejectPlaintextPorts 112233\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "RejectPlaintextPorts 112233\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '112233' out of range in RejectPlaintextPorts");
   tor_free(msg);
@@ -1571,11 +1506,9 @@ test_options_validate__ports(void *ignored)
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 2\n"
                                 "RejectPlaintextPorts 3\n"
-                                "WarnPlaintextPorts 65536\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "WarnPlaintextPorts 65536\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '65536' out of range in WarnPlaintextPorts");
   tor_free(msg);
@@ -1584,13 +1517,10 @@ test_options_validate__ports(void *ignored)
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 2\n"
                                 "RejectPlaintextPorts 3\n"
-                                "WarnPlaintextPorts 4\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "WarnPlaintextPorts 4\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "KeepalivePeriod option must be positive.");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1605,13 +1535,10 @@ test_options_validate__reachable_addresses(void *ignored)
   int ret;
   char *msg;
   setup_capture_of_logs(LOG_NOTICE);
-  options_test_data_t *tdata = get_options_test_data(
-                                     "FascistFirewall 1\n"
-                                     "MaxClientCircuitsPending 1\n"
-                                     "ConnLimit 1\n");
+  options_test_data_t *tdata = get_options_test_data("FascistFirewall 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Converting FascistFirewall config "
             "option to new format: \"ReachableDirAddresses *:80\"\n");
   tt_str_op(tdata->opt->ReachableDirAddresses->value, OP_EQ, "*:80");
@@ -1624,13 +1551,17 @@ test_options_validate__reachable_addresses(void *ignored)
   mock_clean_saved_logs();
   tdata = get_options_test_data("FascistFirewall 1\n"
                                 "ReachableDirAddresses *:81\n"
-                                "ReachableORAddresses *:444\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
-  tdata->opt->FirewallPorts = smartlist_new();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+                                "ReachableORAddresses *:444\n");
+  tt_assert(tdata->opt->FirewallPorts);
+  SMARTLIST_FOREACH(tdata->opt->FirewallPorts, char *, cp, tor_free(cp));
+  smartlist_clear(tdata->opt->FirewallPorts);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+#if 0
+  /* This does not actually produce any logs, and did not produce any relevant
+   * logs before. */
   expect_log_entry();
+#endif
   tt_str_op(tdata->opt->ReachableDirAddresses->value, OP_EQ, "*:81");
   tt_str_op(tdata->opt->ReachableORAddresses->value, OP_EQ, "*:444");
   tor_free(msg);
@@ -1638,12 +1569,10 @@ test_options_validate__reachable_addresses(void *ignored)
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("FascistFirewall 1\n"
-                                "FirewallPort 123\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "FirewallPort 123\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Converting FascistFirewall and "
             "FirewallPorts config options to new format: "
             "\"ReachableAddresses *:123\"\n");
@@ -1655,25 +1584,25 @@ test_options_validate__reachable_addresses(void *ignored)
   tdata = get_options_test_data("FascistFirewall 1\n"
                                 "ReachableAddresses *:82\n"
                                 "ReachableAddresses *:83\n"
-                                "ReachableAddresses reject *:*\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ReachableAddresses reject *:*\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+#if 0
+  /* This does not actually produce any logs, and did not produce any relevant
+   * logs before. */
   expect_log_entry();
+#endif
   tt_str_op(tdata->opt->ReachableAddresses->value, OP_EQ, "*:82");
   tor_free(msg);
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("FascistFirewall 1\n"
-                                "ReachableAddresses *:82\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ReachableAddresses *:82\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(tdata->opt->ReachableAddresses->next, OP_EQ, NULL);
   tor_free(msg);
 
@@ -1683,44 +1612,36 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableAddresses *:82\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableORAddresses *:82\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableDirAddresses *:82\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ClientUseIPv4 0\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
@@ -1728,74 +1649,68 @@ test_options_validate__reachable_addresses(void *ignored)
   /* Test IPv4-only clients setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 1\n"
+  tdata = get_options_test_data("ClientUseIPv4 1\n"
                                 "ClientUseIPv6 0\n"
                                 "UseBridges 0\n"
                                 "ClientPreferIPv6ORPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 1\n"
+  tdata = get_options_test_data("ClientUseIPv4 1\n"
                                 "ClientUseIPv6 0\n"
                                 "UseBridges 0\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   /* Now test an IPv4/IPv6 client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 1\n"
+  tdata = get_options_test_data("ClientUseIPv4 1\n"
                                 "ClientUseIPv6 1\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
   /* Now test an IPv6 client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv6 1\n"
+  tdata = get_options_test_data("ClientUseIPv6 1\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
   /* And an implicit (IPv4 disabled) IPv6 client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 0\n"
+  tdata = get_options_test_data("ClientUseIPv4 0\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
   /* And an implicit (bridge) client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n"
+  tdata = get_options_test_data("UseBridges 1\n"
                                 "Bridge 127.0.0.1:12345\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
@@ -1814,76 +1729,68 @@ test_options_validate__use_bridges(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                    "UseBridges 1\n"
                                    "ClientUseIPv4 1\n"
-                                   "ORPort 127.0.0.1:5555\n"
-                                   "MaxClientCircuitsPending 1\n"
-                                   "ConnLimit 1\n");
+                                   "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Servers must be able to freely connect to the rest of"
             " the Internet, so they must not set UseBridges.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("UseBridges 1\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("UseBridges 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_NE, "Servers must be able to freely connect to the rest of"
             " the Internet, so they must not set UseBridges.");
   tor_free(msg);
 
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       opt_tests_geoip_get_country);
   free_options_test_data(tdata);
   tdata = get_options_test_data("UseBridges 1\n"
-                                "EntryNodes {cn}\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "EntryNodes {cn}\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "You cannot set both UseBridges and EntryNodes.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n");
+  tdata = get_options_test_data("UseBridges 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "If you set UseBridges, you must specify at least one bridge.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n"
+  tdata = get_options_test_data("UseBridges 1\n"
                                 "Bridge 10.0.0.1\n"
                                 "UseEntryGuards 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Setting UseBridges requires also setting UseEntryGuards.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n"
+  tdata = get_options_test_data("UseBridges 1\n"
                                 "Bridge 10.0.0.1\n"
                                 "Bridge !!!\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bridge line did not parse. See logs for details.");
   tor_free(msg);
 
  done:
-  NS_UNMOCK(geoip_get_country);
+  UNMOCK(geoip_get_country);
   policies_free_all();
   free_options_test_data(tdata);
   tor_free(msg);
@@ -1895,14 +1802,13 @@ test_options_validate__entry_nodes(void *ignored)
   (void)ignored;
   int ret;
   char *msg;
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       opt_tests_geoip_get_country);
   options_test_data_t *tdata = get_options_test_data(
                                          "EntryNodes {cn}\n"
-                                         "UseEntryGuards 0\n"
-                                         "MaxClientCircuitsPending 1\n"
-                                         "ConnLimit 1\n");
+                                         "UseEntryGuards 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "If EntryNodes is set, UseEntryGuards must be enabled.");
@@ -1910,17 +1816,14 @@ test_options_validate__entry_nodes(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("EntryNodes {cn}\n"
-                                "UseEntryGuards 1\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "UseEntryGuards 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "KeepalivePeriod option must be positive.");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
-  NS_UNMOCK(geoip_get_country);
+  UNMOCK(geoip_get_country);
   free_options_test_data(tdata);
   tor_free(msg);
 }
@@ -1931,51 +1834,41 @@ test_options_validate__safe_logging(void *ignored)
   (void)ignored;
   int ret;
   char *msg;
-  options_test_data_t *tdata = get_options_test_data(
-                                            "MaxClientCircuitsPending 1\n"
-                                            "ConnLimit 1\n");
+  options_test_data_t *tdata = get_options_test_data("SafeLogging 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_NONE);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging 0\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_NONE);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging Relay\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging Relay\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_RELAY);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging 1\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_ALL);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging stuffy\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging stuffy\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value '\"stuffy\"' in SafeLogging");
   tor_free(msg);
@@ -1994,27 +1887,24 @@ test_options_validate__publish_server_descriptor(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
-             "PublishServerDescriptor bridge\n" TEST_OPTIONS_DEFAULT_VALUES
-                                                     );
+             "PublishServerDescriptor bridge\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("PublishServerDescriptor humma\n"
-                                TEST_OPTIONS_DEFAULT_VALUES);
+  tdata = get_options_test_data("PublishServerDescriptor humma\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value in PublishServerDescriptor");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("PublishServerDescriptor bridge, v3\n"
-                                TEST_OPTIONS_DEFAULT_VALUES);
+  tdata = get_options_test_data("PublishServerDescriptor bridge, v3\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bridges are not supposed to publish router "
             "descriptors to the directory authorities. Please correct your "
@@ -2023,10 +1913,9 @@ test_options_validate__publish_server_descriptor(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("BridgeRelay 1\n"
-                                "PublishServerDescriptor v3\n"
-                                TEST_OPTIONS_DEFAULT_VALUES);
+                                "PublishServerDescriptor v3\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bridges are not supposed to publish router "
             "descriptors to the directory authorities. Please correct your "
@@ -2034,9 +1923,9 @@ test_options_validate__publish_server_descriptor(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("BridgeRelay 1\n" TEST_OPTIONS_DEFAULT_VALUES);
+  tdata = get_options_test_data("BridgeRelay 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_NE, "Bridges are not supposed to publish router "
             "descriptors to the directory authorities. Please correct your "
@@ -2045,10 +1934,10 @@ test_options_validate__publish_server_descriptor(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("BridgeRelay 1\n"
-                                "DirPort 999\n" TEST_OPTIONS_DEFAULT_VALUES);
+                                "DirPort 999\n");
 
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   expect_log_msg("Can't set a DirPort on a bridge "
             "relay; disabling DirPort\n");
@@ -2073,21 +1962,19 @@ test_options_validate__testing(void *ignored)
 #define ENSURE_DEFAULT(varname, varval)                     \
   STMT_BEGIN                                                \
     free_options_test_data(tdata);                          \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES \
-                                #varname " " #varval "\n"); \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  tdata = get_options_test_data(#varname " " #varval "\n"); \
+  ret = options_validate(NULL, tdata->opt, &msg); \
   tt_str_op(msg, OP_EQ, \
             #varname " may only be changed in testing Tor networks!");  \
   tt_int_op(ret, OP_EQ, -1);                                            \
   tor_free(msg);                                                        \
                                                 \
   free_options_test_data(tdata); \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES \
-                                #varname " " #varval "\n"               \
+  tdata = get_options_test_data(#varname " " #varval "\n"               \
                                 VALID_DIR_AUTH                          \
                                 "TestingTorNetwork 1\n");               \
                                                                         \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  ret = options_validate(NULL, tdata->opt, &msg);             \
   if (msg) { \
     tt_str_op(msg, OP_NE, \
               #varname " may only be changed in testing Tor networks!"); \
@@ -2095,11 +1982,10 @@ test_options_validate__testing(void *ignored)
   } \
                                                                         \
   free_options_test_data(tdata);          \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES \
-                                #varname " " #varval "\n"           \
+  tdata = get_options_test_data(#varname " " #varval "\n"           \
                                 "___UsingTestNetworkDefaults 1\n"); \
                                                                         \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  ret = options_validate(NULL, tdata->opt, &msg);\
   if (msg) { \
     tt_str_op(msg, OP_NE, \
               #varname " may only be changed in testing Tor networks!"); \
@@ -2112,7 +1998,6 @@ test_options_validate__testing(void *ignored)
   ENSURE_DEFAULT(TestingV3AuthInitialDistDelay, 3000);
   ENSURE_DEFAULT(TestingV3AuthVotingStartOffset, 3000);
   ENSURE_DEFAULT(TestingAuthDirTimeToLearnReachability, 3000);
-  ENSURE_DEFAULT(TestingEstimatedDescriptorPropagationTime, 3000);
   ENSURE_DEFAULT(TestingServerDownloadInitialDelay, 3000);
   ENSURE_DEFAULT(TestingClientDownloadInitialDelay, 3000);
   ENSURE_DEFAULT(TestingServerConsensusDownloadInitialDelay, 3000);
@@ -2142,21 +2027,12 @@ test_options_validate__hidserv(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_WARN);
 
-  options_test_data_t *tdata = get_options_test_data(
-                                                TEST_OPTIONS_DEFAULT_VALUES);
-  tdata->opt->MinUptimeHidServDirectoryV2 = -1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("MinUptimeHidServDirectoryV2 "
-            "option must be at least 0 seconds. Changing to 0.\n");
-  tt_int_op(tdata->opt->MinUptimeHidServDirectoryV2, OP_EQ, 0);
-  tor_free(msg);
+  options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RendPostPeriod 1\n" );
+  tdata = get_options_test_data("RendPostPeriod 1\n" );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("RendPostPeriod option is too short;"
             " raising to 600 seconds.\n");
@@ -2164,10 +2040,9 @@ test_options_validate__hidserv(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RendPostPeriod 302401\n" );
+  tdata = get_options_test_data("RendPostPeriod 302401\n" );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("RendPostPeriod is too large; "
             "clipping to 302400s.\n");
@@ -2189,45 +2064,40 @@ test_options_validate__path_bias(void *ignored)
   char *msg;
 
   options_test_data_t *tdata = get_options_test_data(
-                                            TEST_OPTIONS_DEFAULT_VALUES
                                             "PathBiasNoticeRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasNoticeRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasWarnRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasWarnRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasWarnRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasExtremeRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasExtremeRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasExtremeRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasNoticeUseRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasNoticeUseRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasNoticeUseRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasExtremeUseRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasExtremeUseRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
            "PathBiasExtremeUseRate is too high. It must be between 0 and 1.0");
@@ -2246,130 +2116,141 @@ test_options_validate__bandwidth(void *ignored)
   char *msg;
   options_test_data_t *tdata = NULL;
 
-#define ENSURE_BANDWIDTH_PARAM(p) \
-  STMT_BEGIN                                                \
+#define ENSURE_BANDWIDTH_PARAM(p, EXTRA_OPT_STR) \
+  STMT_BEGIN \
   free_options_test_data(tdata); \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES #p " 3Gb\n"); \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  tdata = get_options_test_data(EXTRA_OPT_STR \
+                                #p " 3Gb\n"); \
+  ret = options_validate(NULL, tdata->opt, &msg); \
   tt_int_op(ret, OP_EQ, -1); \
   tt_mem_op(msg, OP_EQ, #p " (3221225471) must be at most 2147483647", 40); \
   tor_free(msg); \
   STMT_END
 
-  ENSURE_BANDWIDTH_PARAM(BandwidthRate);
-  ENSURE_BANDWIDTH_PARAM(BandwidthBurst);
-  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth);
-  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate);
-  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst);
-  ENSURE_BANDWIDTH_PARAM(PerConnBWRate);
-  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst);
-  ENSURE_BANDWIDTH_PARAM(AuthDirFastGuarantee);
-  ENSURE_BANDWIDTH_PARAM(AuthDirGuardBWGuarantee);
+  ENSURE_BANDWIDTH_PARAM(BandwidthRate, "");
+  ENSURE_BANDWIDTH_PARAM(BandwidthBurst, "");
+
+  ENSURE_BANDWIDTH_PARAM(BandwidthRate, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(BandwidthBurst, ENABLE_AUTHORITY_V3);
+
+  ENSURE_BANDWIDTH_PARAM(BandwidthRate, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(BandwidthBurst, ENABLE_AUTHORITY_BRIDGE);
+
+  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth, "");
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate, "");
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst, "");
+  ENSURE_BANDWIDTH_PARAM(PerConnBWRate, "");
+  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst, "");
+
+  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWRate, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst, ENABLE_AUTHORITY_V3);
+
+  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWRate, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst, ENABLE_AUTHORITY_BRIDGE);
+
+  ENSURE_BANDWIDTH_PARAM(AuthDirFastGuarantee, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(AuthDirGuardBWGuarantee, ENABLE_AUTHORITY_V3);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1000\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("RelayBandwidthRate 1000\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->RelayBandwidthBurst, OP_EQ, 1000);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthBurst 1001\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("RelayBandwidthBurst 1001\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->RelayBandwidthRate, OP_EQ, 1001);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1001\n"
+  tdata = get_options_test_data("RelayBandwidthRate 1001\n"
                                 "RelayBandwidthBurst 1000\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "RelayBandwidthBurst must be at least equal to "
             "RelayBandwidthRate.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "BandwidthRate 1001\n"
+  tdata = get_options_test_data("BandwidthRate 1001\n"
                                 "BandwidthBurst 1000\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "BandwidthBurst must be at least equal to BandwidthRate.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1001\n"
+  tdata = get_options_test_data("RelayBandwidthRate 1001\n"
                                 "BandwidthRate 1000\n"
                                 "BandwidthBurst 1000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->BandwidthRate, OP_EQ, 1001);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1001\n"
+  tdata = get_options_test_data("RelayBandwidthRate 1001\n"
                                 "BandwidthRate 1000\n"
                                 "RelayBandwidthBurst 1001\n"
                                 "BandwidthBurst 1000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->BandwidthBurst, OP_EQ, 1001);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "BandwidthRate is set to 1 bytes/second. For servers,"
             " it must be at least 76800.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "MaxAdvertisedBandwidth 30000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "MaxAdvertisedBandwidth is set to 30000 bytes/second."
             " For servers, it must be at least 38400.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "RelayBandwidthRate 1\n"
                                 "MaxAdvertisedBandwidth 38400\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "RelayBandwidthRate is set to 1 bytes/second. For "
             "servers, it must be at least 76800.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "BandwidthBurst 76800\n"
                                 "RelayBandwidthRate 76800\n"
                                 "MaxAdvertisedBandwidth 38400\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
@@ -2388,9 +2269,8 @@ test_options_validate__circuits(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MaxCircuitDirtiness 2592001\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("MaxCircuitDirtiness 2592001\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("MaxCircuitDirtiness option is too "
             "high; setting to 30 days.\n");
   tt_int_op(tdata->opt->MaxCircuitDirtiness, OP_EQ, 2592000);
@@ -2398,9 +2278,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CircuitStreamTimeout 1\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("CircuitStreamTimeout 1\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("CircuitStreamTimeout option is too"
             " short; raising to 10 seconds.\n");
   tt_int_op(tdata->opt->CircuitStreamTimeout, OP_EQ, 10);
@@ -2408,9 +2287,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CircuitStreamTimeout 111\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("CircuitStreamTimeout 111\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_no_log_msg("CircuitStreamTimeout option is too"
             " short; raising to 10 seconds.\n");
   tt_int_op(tdata->opt->CircuitStreamTimeout, OP_EQ, 111);
@@ -2418,9 +2296,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HeartbeatPeriod 1\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HeartbeatPeriod 1\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("HeartbeatPeriod option is too short;"
             " raising to 1800 seconds.\n");
   tt_int_op(tdata->opt->HeartbeatPeriod, OP_EQ, 1800);
@@ -2428,9 +2305,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HeartbeatPeriod 1982\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HeartbeatPeriod 1982\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_no_log_msg("HeartbeatPeriod option is too short;"
             " raising to 1800 seconds.\n");
   tt_int_op(tdata->opt->HeartbeatPeriod, OP_EQ, 1982);
@@ -2438,10 +2314,10 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data("LearnCircuitBuildTimeout 0\n"
                                 "CircuitBuildTimeout 1\n"
                                 );
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("CircuitBuildTimeout is shorter (1"
             " seconds) than the recommended minimum (10 seconds), and "
             "LearnCircuitBuildTimeout is disabled.  If tor isn't working, "
@@ -2450,10 +2326,9 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CircuitBuildTimeout 11\n"
+  tdata = get_options_test_data("CircuitBuildTimeout 11\n"
                                 );
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_no_log_msg("CircuitBuildTimeout is shorter (1 "
             "seconds) than the recommended minimum (10 seconds), and "
             "LearnCircuitBuildTimeout is disabled.  If tor isn't working, "
@@ -2477,51 +2352,46 @@ test_options_validate__rend(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                  "UseEntryGuards 0\n"
                  "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                  "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("UseEntryGuards is disabled, but you"
             " have configured one or more hidden services on this Tor "
             "instance.  Your hidden services will be very easy to locate using"
-            " a well-known attack -- see http://freehaven.net/anonbib/#hs-"
+            " a well-known attack -- see https://freehaven.net/anonbib/#hs-"
             "attack06 for details.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-            TEST_OPTIONS_DEFAULT_VALUES
             "UseEntryGuards 1\n"
             "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
             "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("UseEntryGuards is disabled, but you"
             " have configured one or more hidden services on this Tor "
             "instance.  Your hidden services will be very easy to locate using"
-            " a well-known attack -- see http://freehaven.net/anonbib/#hs-"
+            " a well-known attack -- see https://freehaven.net/anonbib/#hs-"
             "attack06 for details.\n");
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HiddenServicePort 80 127.0.0.1:8080\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HiddenServicePort 80 127.0.0.1:8080\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Failed to configure rendezvous options. See logs for details.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HidServAuth failed\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HidServAuth failed\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Failed to configure client authorization for hidden "
             "services. See logs for details.");
@@ -2545,11 +2415,10 @@ test_options_validate__single_onion(void *ignored)
 
   /* Test that HiddenServiceSingleHopMode must come with
    * HiddenServiceNonAnonymousMode */
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceSingleHopMode does not provide any "
             "server anonymity. It must be used with "
@@ -2557,12 +2426,11 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 0\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceSingleHopMode does not provide any "
             "server anonymity. It must be used with "
@@ -2570,23 +2438,21 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
   /* Test that SOCKSPort if HiddenServiceSingleHopMode is 1 */
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 5000\n"
+  tdata = get_options_test_data("SOCKSPort 5000\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode is incompatible with "
             "using Tor as an anonymous client. Please set "
@@ -2595,32 +2461,30 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 5000\n"
+  tdata = get_options_test_data("SOCKSPort 5000\n"
                                 "HiddenServiceSingleHopMode 0\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
   /* Test that a hidden service can't be run in non anonymous mode. */
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceNonAnonymousMode 1\n"
                   "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                   "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode does not provide any "
             "server anonymity. It must be used with "
@@ -2628,10 +2492,10 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode does not provide any "
             "server anonymity. It must be used with "
@@ -2639,23 +2503,23 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                   "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceNonAnonymousMode 1\n"
                   "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                   "HiddenServicePort 80 127.0.0.1:8080\n"
                   "HiddenServiceSingleHopMode 1\n"
                   "SOCKSPort 0\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
@@ -2676,53 +2540,56 @@ test_options_validate__accounting(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingRule something_bad\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule something_bad\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "AccountingRule must be 'sum', 'max', 'in', or 'out'");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingRule sum\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule sum\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_SUM);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingRule max\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule max\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_MAX);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingStart fail\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule in\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_IN);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data("AccountingRule out\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_OUT);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data("AccountingStart fail\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Failed to parse accounting options. See logs for details.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingMax 10\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingMax 10\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-           TEST_OPTIONS_DEFAULT_VALUES
            "ORPort 127.0.0.1:5555\n"
            "BandwidthRate 76800\n"
            "BandwidthBurst 76800\n"
@@ -2732,7 +2599,7 @@ test_options_validate__accounting(void *ignored)
            "AccountingMax 10\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Using accounting with a hidden "
             "service and an ORPort is risky: your hidden service(s) and "
@@ -2743,13 +2610,12 @@ test_options_validate__accounting(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-                TEST_OPTIONS_DEFAULT_VALUES
                 "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                 "HiddenServicePort 80 127.0.0.1:8080\n"
                 "AccountingMax 10\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("Using accounting with a hidden "
             "service and an ORPort is risky: your hidden service(s) and "
@@ -2760,7 +2626,6 @@ test_options_validate__accounting(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-             TEST_OPTIONS_DEFAULT_VALUES
              "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
              "HiddenServicePort 80 127.0.0.1:8080\n"
              "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service2/\n"
@@ -2768,7 +2633,7 @@ test_options_validate__accounting(void *ignored)
              "AccountingMax 10\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Using accounting with multiple "
             "hidden services is risky: they will all turn off at the same"
@@ -2795,36 +2660,29 @@ test_options_validate__proxy(void *ignored)
   MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPProxyPort, OP_EQ, 80);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPProxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxyAuthenticator "
+  tdata = get_options_test_data("HttpProxyAuthenticator "
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
                                 "othreonetwothreonetwothreonetwothreonetwothre"
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
@@ -2837,52 +2695,41 @@ test_options_validate__proxy(void *ignored)
                                 "othreonetwothreonetwothreonetwothreonetwothre"
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
                                 "othreonetwothreeonetwothreeonetwothree"
-
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPProxyAuthenticator is too long (>= 512 chars).");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxyAuthenticator validauth\n"
-
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxyAuthenticator validauth\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPSProxyPort, OP_EQ, 443);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPSProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPSProxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxyAuthenticator "
+  tdata = get_options_test_data("HttpsProxyAuthenticator "
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
                                 "othreonetwothreonetwothreonetwothreonetwothre"
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
@@ -2897,103 +2744,86 @@ test_options_validate__proxy(void *ignored)
                                 "othreonetwothreeonetwothreeonetwothree"
 
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPSProxyAuthenticator is too long (>= 512 chars).");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxyAuthenticator validauth\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxyAuthenticator validauth\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks4Proxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks4ProxyPort, OP_EQ, 1080);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks4Proxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks4ProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks4Proxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks4Proxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5Proxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5Proxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks5ProxyPort, OP_EQ, 1080);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5Proxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5Proxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks5ProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5Proxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5Proxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks5Proxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy 215.1.1.1\n"
+  tdata = get_options_test_data("Socks4Proxy 215.1.1.1\n"
                                 "Socks5Proxy 215.1.1.2\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "You have configured more than one proxy type. "
-            "(Socks4Proxy|Socks5Proxy|HTTPSProxy)");
+            "(Socks4Proxy|Socks5Proxy|HTTPSProxy|TCPProxy)");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
-                                );
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("HTTPProxy configured, but no SOCKS "
-            "proxy or HTTPS proxy configured. Watch out: this configuration "
-            "will proxy unencrypted directory connections only.\n");
+  expect_log_msg("HTTPProxy configured, but no SOCKS proxy, "
+            "HTTPS proxy, or any other TCP proxy configured. Watch out: "
+            "this configuration will proxy unencrypted directory "
+            "connections only.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n"
                                 "Socks4Proxy 215.1.1.1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("HTTPProxy configured, but no SOCKS "
             "proxy or HTTPS proxy configured. Watch out: this configuration "
@@ -3001,12 +2831,11 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n"
                                 "Socks5Proxy 215.1.1.1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("HTTPProxy configured, but no SOCKS "
             "proxy or HTTPS proxy configured. Watch out: this configuration "
@@ -3014,12 +2843,11 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n"
                                 "HttpsProxy 215.1.1.1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "HTTPProxy configured, but no SOCKS proxy or HTTPS proxy "
@@ -3028,81 +2856,69 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                );
+  tdata = get_options_test_data("");
   tdata->opt->Socks5ProxyUsername = tor_strdup("");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyUsername must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                );
+  tdata = get_options_test_data("");
   tdata->opt->Socks5ProxyUsername =
     tor_strdup("ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789AB"
                "CDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCD"
                "EABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEA"
                "BCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABC"
                "DE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyUsername must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks5ProxyPassword must be included with "
             "Socks5ProxyUsername.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                );
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n");
   tdata->opt->Socks5ProxyPassword = tor_strdup("");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyPassword must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                );
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n");
   tdata->opt->Socks5ProxyPassword =
     tor_strdup("ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789AB"
                "CDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCD"
                "EABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEA"
                "BCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABC"
                "DE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyPassword must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                "Socks5ProxyPassword world_hello\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n"
+                                "Socks5ProxyPassword world_hello\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyPassword hello_world\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5ProxyPassword hello_world\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks5ProxyPassword must be included with "
             "Socks5ProxyUsername.");
@@ -3127,69 +2943,62 @@ test_options_validate__control(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HashedControlPassword something_incorrect\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data(
+                         "HashedControlPassword something_incorrect\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Bad HashedControlPassword: wrong length or bad encoding");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HashedControlPassword 16:872860B76453A77D60CA"
+  tdata = get_options_test_data("HashedControlPassword 16:872860B76453A77D60CA"
                                 "2BB8C1A7042072093276A3D701AD684053EC4C\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-                   TEST_OPTIONS_DEFAULT_VALUES
                    "__HashedControlSessionPassword something_incorrect\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bad HashedControlSessionPassword: wrong length or "
             "bad encoding");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "__HashedControlSessionPassword 16:872860B7645"
+  tdata = get_options_test_data("__HashedControlSessionPassword 16:872860B7645"
                                 "3A77D60CA2BB8C1A7042072093276A3D701AD684053EC"
                                 "4C\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-                           TEST_OPTIONS_DEFAULT_VALUES
                            "__OwningControllerProcess something_incorrect\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bad OwningControllerProcess: invalid PID");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "__OwningControllerProcess 123\n"
+  tdata = get_options_test_data("__OwningControllerProcess 123\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3199,13 +3008,12 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 "HashedControlPassword 16:872860B76453A77D60CA"
                                 "2BB8C1A7042072093276A3D701AD684053EC4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3215,14 +3023,13 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 "__HashedControlSessionPassword 16:872860B7645"
                                 "3A77D60CA2BB8C1A7042072093276A3D701AD684053EC"
                                 "4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3232,12 +3039,11 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 "CookieAuthentication 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3248,11 +3054,9 @@ test_options_validate__control(void *ignored)
 
 #ifdef HAVE_SYS_UN_H
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
-                                );
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3262,13 +3066,12 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n"
                                 "HashedControlPassword 16:872860B76453A77D60CA"
                                 "2BB8C1A7042072093276A3D701AD684053EC4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3278,14 +3081,13 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n"
                                 "__HashedControlSessionPassword 16:872860B7645"
                                 "3A77D60CA2BB8C1A7042072093276A3D701AD684053EC"
                                 "4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3295,12 +3097,11 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n"
                                 "CookieAuthentication 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3311,11 +3112,10 @@ test_options_validate__control(void *ignored)
 #endif /* defined(HAVE_SYS_UN_H) */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CookieAuthFileGroupReadable 1\n"
+  tdata = get_options_test_data("CookieAuthFileGroupReadable 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "CookieAuthFileGroupReadable is set, but will have no effect: you "
@@ -3324,12 +3124,11 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CookieAuthFileGroupReadable 1\n"
+  tdata = get_options_test_data("CookieAuthFileGroupReadable 1\n"
                                 "CookieAuthFile /tmp/somewhere\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "CookieAuthFileGroupReadable is set, but will have no effect: you "
@@ -3354,8 +3153,7 @@ test_options_validate__families(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MyFamily home\n"
+  tdata = get_options_test_data("MyFamily home\n"
                                 "BridgeRelay 1\n"
                                 "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 51300\n"
@@ -3364,7 +3162,7 @@ test_options_validate__families(void *ignored)
                                 "DirCache 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "Listing a family for a bridge relay is not supported: it can "
@@ -3374,11 +3172,9 @@ test_options_validate__families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MyFamily home\n"
-                                );
+  tdata = get_options_test_data("MyFamily home\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "Listing a family for a bridge relay is not supported: it can "
@@ -3388,22 +3184,18 @@ test_options_validate__families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MyFamily !\n"
-                                );
+  tdata = get_options_test_data("MyFamily !\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Invalid nickname '!' in MyFamily line");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "NodeFamily foo\n"
-                                "NodeFamily !\n"
-                                );
+  tdata = get_options_test_data("NodeFamily foo\n"
+                                "NodeFamily !\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_assert(!msg);
   tor_free(msg);
@@ -3424,11 +3216,10 @@ test_options_validate__addr_policies(void *ignored)
   options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ExitPolicy !!!\n"
+  tdata = get_options_test_data("ExitPolicy !!!\n"
                                 "ExitRelay 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Error in ExitPolicy entry.");
   tor_free(msg);
@@ -3449,12 +3240,11 @@ test_options_validate__dir_auth(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                VALID_DIR_AUTH
+  tdata = get_options_test_data(VALID_DIR_AUTH
                                 VALID_ALT_DIR_AUTH
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Directory authority/fallback line did not parse. See logs for "
@@ -3464,10 +3254,8 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingTorNetwork 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("TestingTorNetwork 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TestingTorNetwork may only be configured in combination with a "
@@ -3476,20 +3264,18 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                VALID_DIR_AUTH
+  tdata = get_options_test_data(VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingTorNetwork 1\n"
+  tdata = get_options_test_data("TestingTorNetwork 1\n"
                                 VALID_ALT_DIR_AUTH
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TestingTorNetwork may only be configured in combination with a "
@@ -3498,11 +3284,10 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingTorNetwork 1\n"
+  tdata = get_options_test_data("TestingTorNetwork 1\n"
                                 VALID_ALT_BRIDGE_AUTH
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingTorNetwork may only be configured in "
             "combination with a non-default set of DirAuthority or both of "
@@ -3510,12 +3295,11 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                VALID_ALT_DIR_AUTH
+  tdata = get_options_test_data(VALID_ALT_DIR_AUTH
                                 VALID_ALT_BRIDGE_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
@@ -3536,11 +3320,10 @@ test_options_validate__transport(void *ignored)
   setup_capture_of_logs(LOG_NOTICE);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientTransportPlugin !!\n"
+  tdata = get_options_test_data("ClientTransportPlugin !!\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Invalid client transport line. See logs for details.");
@@ -3549,20 +3332,17 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientTransportPlugin foo exec bar\n"
+  tdata = get_options_test_data("ClientTransportPlugin foo exec bar\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportPlugin !!\n"
-                                );
+  tdata = get_options_test_data("ServerTransportPlugin !!\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Invalid server transport line. See logs for details.");
@@ -3571,11 +3351,9 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportPlugin foo exec bar\n"
-                                );
+  tdata = get_options_test_data("ServerTransportPlugin foo exec bar\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "Tor is not configured as a relay but you specified a "
@@ -3584,15 +3362,14 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportPlugin foo exec bar\n"
+  tdata = get_options_test_data("ServerTransportPlugin foo exec bar\n"
                                 "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "Tor is not configured as a relay but you specified a "
@@ -3601,22 +3378,19 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportListenAddr foo 127.0.0.42:55\n"
-                                "ServerTransportListenAddr !\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("ServerTransportListenAddr foo 127.0.0.42:55\n"
+                                "ServerTransportListenAddr !\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "ServerTransportListenAddr did not parse. See logs for details.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportListenAddr foo 127.0.0.42:55\n"
+  tdata = get_options_test_data("ServerTransportListenAddr foo 127.0.0.42:55\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "You need at least a single managed-proxy to specify a transport "
@@ -3625,8 +3399,7 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportListenAddr foo 127.0.0.42:55\n"
+  tdata = get_options_test_data("ServerTransportListenAddr foo 127.0.0.42:55\n"
                                 "ServerTransportPlugin foo exec bar\n"
                                 "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
@@ -3634,7 +3407,7 @@ test_options_validate__transport(void *ignored)
                                 "MaxAdvertisedBandwidth 38500\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "You need at least a single managed-proxy to specify a transport "
@@ -3659,50 +3432,46 @@ test_options_validate__constrained_sockets(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 0\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConstrainedSockSize is invalid.  Must be a value "
             "between 2048 and 262144 in 1024 byte increments.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 263168\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConstrainedSockSize is invalid.  Must be a value "
             "between 2048 and 262144 in 1024 byte increments.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 2047\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConstrainedSockSize is invalid.  Must be a value "
             "between 2048 and 262144 in 1024 byte increments.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 2048\n"
                                 "DirPort 999\n"
                                 "DirCache 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("You have requested constrained "
             "socket buffers while also serving directory entries via DirPort."
@@ -3711,12 +3480,11 @@ test_options_validate__constrained_sockets(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 2048\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "You have requested constrained socket buffers while also serving"
@@ -3742,12 +3510,12 @@ test_options_validate__v3_auth(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 1000\n"
                                 "V3AuthDistDelay 1000\n"
                                 "V3AuthVotingInterval 1000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
@@ -3755,20 +3523,18 @@ test_options_validate__v3_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "V3AuthVoteDelay 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3AuthVoteDelay 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVoteDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 1\n"
-                                "TestingTorNetwork 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+                                "TestingTorNetwork 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVoteDelay is way too low.");
   tor_free(msg);
@@ -3778,87 +3544,127 @@ test_options_validate__v3_auth(void *ignored)
   // since they are the same
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "V3AuthDistDelay 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3AuthDistDelay 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthDistDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthDistDelay 1\n"
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthDistDelay is way too low.");
   tor_free(msg);
 
-  // TODO: we can't reach the case of v3authdistdelay lower than
+  // We can't reach the case of v3authdistdelay lower than
   // MIN_DIST_SECONDS but not lower than MIN_DIST_SECONDS_TESTING,
   // since they are the same
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthNIntervalsValid 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthNIntervalsValid must be at least 2.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 200\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVotingInterval is insanely low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "V3AuthVoteDelay 49\n"
+                                "V3AuthDistDelay 49\n"
+                                "V3AuthVotingInterval 200\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "V3AuthVoteDelay 2\n"
+                                "V3AuthDistDelay 2\n"
+                                "V3AuthVotingInterval 9\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
+            "V3AuthVotingInterval");
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "V3AuthVoteDelay 2\n"
+                                "V3AuthDistDelay 2\n"
+                                "V3AuthVotingInterval 10\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 200000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVotingInterval is insanely high.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 1441\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("V3AuthVotingInterval does not divide"
             " evenly into 24 hours.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 1440\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("V3AuthVotingInterval does not divide"
             " evenly into 24 hours.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 299\n"
@@ -3866,84 +3672,125 @@ test_options_validate__v3_auth(void *ignored)
                                 "TestingTorNetwork 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("V3AuthVotingInterval is very low. "
             "This may lead to failure to synchronise for a consensus.\n");
   tor_free(msg);
 
-  // TODO: It is impossible to reach the case of testingtor network, with
-  // v3authvotinginterval too low
-  /* free_options_test_data(tdata); */
-  /* tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES */
-  /*                               "V3AuthVoteDelay 1\n" */
-  /*                               "V3AuthDistDelay 1\n" */
-  /*                               "V3AuthVotingInterval 9\n" */
-                                   /* VALID_DIR_AUTH */
-  /*                               "TestingTorNetwork 1\n" */
-  /*                               ); */
-  /* ret = options_validate(tdata->old_opt, tdata->opt, */
-  /*                        tdata->def_opt, 0, &msg); */
-  /* tt_int_op(ret, OP_EQ, -1); */
-  /* tt_str_op(msg, OP_EQ, "V3AuthVotingInterval is insanely low."); */
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3AuthVoteDelay 1\n"
+                                "V3AuthDistDelay 1\n"
+                                "V3AuthVotingInterval 9\n"
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                );
+  /* We have to call the dirauth-specific function to reach this case */
+  ret = options_validate_dirauth_schedule(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "V3AuthVoteDelay is way too low.");
+  tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "TestingV3AuthInitialVoteDelay 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthInitialVoteDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "TestingV3AuthInitialDistDelay 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthInitialDistDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
   tdata->opt->TestingV3AuthVotingStartOffset = 100000;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthVotingStartOffset is higher than the "
             "voting interval.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
   tdata->opt->TestingV3AuthVotingStartOffset = -1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TestingV3AuthVotingStartOffset must be non-negative.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "TestingV3AuthInitialVotingInterval 4\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthInitialVotingInterval is insanely low.");
   tor_free(msg);
 
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "TestingV3AuthInitialVoteDelay 2\n"
+                                "TestingV3AuthInitialDistDelay 2\n"
+                                "TestingV3AuthInitialVotingInterval 5\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "TestingV3AuthInitialVotingInterval 7\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "TestingV3AuthInitialVotingInterval does not divide evenly into "
+            "30 minutes.");
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "TestingV3AuthInitialVoteDelay 3\n"
+                                "TestingV3AuthInitialDistDelay 3\n"
+                                "TestingV3AuthInitialVotingInterval 5\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "TestingV3AuthInitialVoteDelay plus "
+            "TestingV3AuthInitialDistDelay must be less than "
+            "TestingV3AuthInitialVotingInterval");
+  tor_free(msg);
+
  done:
   policies_free_all();
   teardown_capture_of_logs();
@@ -3960,19 +3807,16 @@ test_options_validate__virtual_addr(void *ignored)
   options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "VirtualAddrNetworkIPv4 !!"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("VirtualAddrNetworkIPv4 !!");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Error parsing VirtualAddressNetwork !!");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "VirtualAddrNetworkIPv6 !!"
+  tdata = get_options_test_data("VirtualAddrNetworkIPv6 !!"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Error parsing VirtualAddressNetworkIPv6 !!");
   tor_free(msg);
@@ -3993,135 +3837,133 @@ test_options_validate__testing_options(void *ignored)
   options_test_data_t *tdata = NULL;
   setup_capture_of_logs(LOG_WARN);
 
-#define TEST_TESTING_OPTION(name, low_val, high_val, err_low)           \
+#define TEST_TESTING_OPTION(name, accessor, \
+                            low_val, high_val, err_low, EXTRA_OPT_STR)  \
   STMT_BEGIN                                                            \
     free_options_test_data(tdata);                                      \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES             \
+  tdata = get_options_test_data(EXTRA_OPT_STR                           \
                                 VALID_DIR_AUTH                          \
                                 "TestingTorNetwork 1\n"                 \
                                 );                                      \
-  tdata->opt-> name = low_val;                                       \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  accessor(tdata->opt)->name = low_val;                                 \
+  ret = options_validate(NULL, tdata->opt,  &msg);            \
   tt_int_op(ret, OP_EQ, -1);                                            \
   tt_str_op(msg, OP_EQ, #name " " err_low);                \
   tor_free(msg); \
                                                                         \
   free_options_test_data(tdata);                                        \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES             \
+  tdata = get_options_test_data(EXTRA_OPT_STR                           \
                                 VALID_DIR_AUTH                          \
                                 "TestingTorNetwork 1\n"                 \
                                 );                                      \
-  tdata->opt->  name = high_val;                                      \
+  accessor(tdata->opt)->name = high_val;                                \
   mock_clean_saved_logs();                                              \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  ret = options_validate(NULL, tdata->opt,  &msg);            \
   tt_int_op(ret, OP_EQ, 0);                                             \
+  tt_ptr_op(msg, OP_EQ, NULL);                                          \
   expect_log_msg( #name " is insanely high.\n"); \
   tor_free(msg); \
   STMT_END
 
-  TEST_TESTING_OPTION(TestingAuthDirTimeToLearnReachability, -1, 8000,
-                      "must be non-negative.");
-  TEST_TESTING_OPTION(TestingEstimatedDescriptorPropagationTime, -1, 3601,
-                      "must be non-negative.");
-  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, -1, 3601,
-                      "is way too low.");
-  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, 1, 3601,
-                      "is way too low.");
+  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, , -1, 3601,
+                      "is way too low.", "");
+  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, , 1, 3601,
+                      "is way too low.", "");
+
+  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, , -1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_V3);
+  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, , 1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_V3);
+
+  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, , -1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_BRIDGE);
+  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, , 1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_BRIDGE);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableConnBwEvent 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("TestingEnableConnBwEvent 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingEnableConnBwEvent may only be changed in "
             "testing Tor networks!");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableConnBwEvent 1\n"
+  tdata = get_options_test_data("TestingEnableConnBwEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableConnBwEvent 1\n"
+  tdata = get_options_test_data("TestingEnableConnBwEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 0\n"
                                 "___UsingTestNetworkDefaults 1\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableCellStatsEvent 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("TestingEnableCellStatsEvent 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingEnableCellStatsEvent may only be changed in "
             "testing Tor networks!");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableCellStatsEvent 1\n"
+  tdata = get_options_test_data("TestingEnableCellStatsEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableCellStatsEvent 1\n"
+  tdata = get_options_test_data("TestingEnableCellStatsEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 0\n"
                                 "___UsingTestNetworkDefaults 1\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableTbEmptyEvent 1\n"
+  tdata = get_options_test_data("TestingEnableTbEmptyEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableTbEmptyEvent 1\n"
+  tdata = get_options_test_data("TestingEnableTbEmptyEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 0\n"
                                 "___UsingTestNetworkDefaults 1\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
@@ -4142,40 +3984,32 @@ test_options_validate__accel(void *ignored)
   options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelName foo\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelName foo\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tdata->opt->HardwareAccel, OP_EQ, 1);
+  tt_int_op(get_crypto_options(tdata->opt)->HardwareAccel, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelName foo\n"
-                                );
-  tdata->opt->HardwareAccel = 2;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelName foo\n");
+  get_crypto_options(tdata->opt)->HardwareAccel = 2;
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tdata->opt->HardwareAccel, OP_EQ, 2);
+  tt_int_op(get_crypto_options(tdata->opt)->HardwareAccel, OP_EQ, 2);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelDir 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelDir 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Can't use hardware crypto accelerator dir without engine name.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelDir 1\n"
-                                "AccelName something\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelDir 1\n"
+                                "AccelName something\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
@@ -4185,8 +4019,280 @@ test_options_validate__accel(void *ignored)
   tor_free(msg);
 }
 
+static int mocked_granularity;
+
+static void
+mock_set_log_time_granularity(int g)
+{
+  mocked_granularity = g;
+}
+
+static void
+test_options_init_logs_granularity(void *arg)
+{
+  options_test_data_t *tdata = get_options_test_data("");
+  int rv;
+  (void) arg;
+
+  MOCK(set_log_time_granularity, mock_set_log_time_granularity);
+
+  /* Reasonable value. */
+  tdata->opt->LogTimeGranularity = 100;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 100);
+
+  /* Doesn't divide 1000. */
+  tdata->opt->LogTimeGranularity = 249;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 250);
+
+  /* Doesn't divide 1000. */
+  tdata->opt->LogTimeGranularity = 3;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 4);
+
+  /* Not a multiple of 1000. */
+  tdata->opt->LogTimeGranularity = 1500;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 2000);
+
+  /* Reasonable value. */
+  tdata->opt->LogTimeGranularity = 3000;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 3000);
+
+  /* Negative. (Shouldn't be allowed by rest of config parsing.) */
+  tdata->opt->LogTimeGranularity = -1;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, -1);
+
+  /* Very big */
+  tdata->opt->LogTimeGranularity = 3600 * 1000;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 3600 * 1000);
+
+ done:
+  free_options_test_data(tdata);
+  UNMOCK(set_log_time_granularity);
+}
+
+typedef struct {
+  char *name;
+  log_severity_list_t sev;
+  int fd;
+  bool stream;
+} added_log_t;
+
+static smartlist_t *added_logs = NULL;
+
+static void
+mock_add_stream_log_impl(const log_severity_list_t *sev, const char *name,
+                         int fd)
+{
+  added_log_t *a = tor_malloc_zero(sizeof(added_log_t));
+  a->name = tor_strdup(name);
+  memcpy(&a->sev, sev, sizeof(log_severity_list_t));
+  a->fd = fd;
+  a->stream = true;
+  smartlist_add(added_logs, a);
+}
+
+static int
+mock_add_file_log(const log_severity_list_t *sev, const char *name, int fd)
+{
+  added_log_t *a = tor_malloc_zero(sizeof(added_log_t));
+  a->name = tor_strdup(name);
+  memcpy(&a->sev, sev, sizeof(log_severity_list_t));
+  a->fd = fd;
+  smartlist_add(added_logs, a);
+  return 0;
+}
+
+static void
+clear_added_logs(void)
+{
+  SMARTLIST_FOREACH(added_logs, added_log_t *, a,
+                    { tor_free(a->name); tor_free(a); });
+  smartlist_clear(added_logs);
+}
+
+static void
+test_options_init_logs_quiet(void *arg)
+{
+  (void)arg;
+  char *cfg = NULL;
+  options_test_data_t *tdata = get_options_test_data("");
+  char *fn1 = tor_strdup(get_fname_rnd("log"));
+  const added_log_t *a;
+  int rv;
+  tdata->opt->RunAsDaemon = 0;
+
+  added_logs = smartlist_new();
+  MOCK(add_stream_log_impl, mock_add_stream_log_impl);
+  MOCK(add_file_log, mock_add_file_log);
+
+  tt_ptr_op(tdata->opt->Logs, OP_EQ, NULL);
+
+  /* First, try with no configured logs, and make sure that our configured
+     logs match the quiet level. */
+  quiet_level = QUIET_SILENT;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 0);
+
+  quiet_level = QUIET_HUSH;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 1);
+  a = smartlist_get(added_logs, 0);
+  tt_assert(a);
+  tt_assert(a->stream);
+  tt_int_op(a->fd, OP_EQ, fileno(stdout));
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_INFO)], OP_EQ, 0);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_NOTICE)], OP_EQ, 0);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_WARN)], OP_EQ, LD_ALL_DOMAINS);
+  clear_added_logs();
+
+  quiet_level = QUIET_NONE;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 1);
+  a = smartlist_get(added_logs, 0);
+  tt_assert(a);
+  tt_assert(a->stream);
+  tt_int_op(a->fd, OP_EQ, fileno(stdout));
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_INFO)], OP_EQ, 0);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_NOTICE)], OP_EQ,
+            LD_ALL_DOMAINS);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_WARN)], OP_EQ, LD_ALL_DOMAINS);
+  clear_added_logs();
+
+  /* Make sure that adding a configured log makes the default logs go away. */
+  tor_asprintf(&cfg, "Log info file %s\n", fn1);
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(cfg);
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 1);
+  a = smartlist_get(added_logs, 0);
+  tt_assert(a);
+  tt_assert(! a->stream);
+  tt_int_op(a->fd, OP_NE, fileno(stdout));
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_INFO)], OP_EQ, LD_ALL_DOMAINS);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_NOTICE)], OP_EQ,
+            LD_ALL_DOMAINS);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_WARN)], OP_EQ, LD_ALL_DOMAINS);
+
+ done:
+  free_options_test_data(tdata);
+  tor_free(fn1);
+  tor_free(cfg);
+  clear_added_logs();
+  smartlist_free(added_logs);
+  UNMOCK(add_stream_log_impl);
+  UNMOCK(add_file_log);
+}
+
+static int mock_options_act_status = 0;
+static int
+mock_options_act(const or_options_t *old_options)
+{
+  (void)old_options;
+  return mock_options_act_status;
+}
+static int
+mock_options_act_reversible(const or_options_t *old_options, char **msg_out)
+{
+  (void)old_options;
+  (void)msg_out;
+  return 0;
+}
+
+static void
+test_options_trial_assign(void *arg)
+{
+  (void)arg;
+  setopt_err_t v;
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+  int r;
+
+  // replace options_act*() so that we don't actually launch tor here.
+  MOCK(options_act, mock_options_act);
+  MOCK(options_act_reversible, mock_options_act_reversible);
+
+  // Try assigning nothing; that should work.
+  v = options_trial_assign(lines, 0, &msg);
+  if (msg)
+    puts(msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(v, OP_EQ, SETOPT_OK);
+
+  // Assigning a nickname is okay
+  r = config_get_lines("Nickname Hemiramphinae", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(v, OP_EQ, SETOPT_OK);
+  tt_str_op(get_options()->Nickname, OP_EQ, "Hemiramphinae");
+  config_free_lines(lines);
+
+  // We can't change the User; that's a transition error.
+  r = config_get_lines("User Heraclitus", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_int_op(v, OP_EQ, SETOPT_ERR_TRANSITION);
+  tt_str_op(msg, OP_EQ,  "While Tor is running, changing User is not allowed");
+  tor_free(msg);
+  config_free_lines(lines);
+
+  // We can't set the ORPort to nonsense: that's a validation error.
+  r = config_get_lines("ORPort fractabling planished", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_int_op(v, OP_EQ, SETOPT_ERR_PARSE); // (same error code for now)
+  tt_str_op(msg, OP_EQ, "Invalid ORPort configuration");
+  tor_free(msg);
+  config_free_lines(lines);
+
+  // We can't set UseBridges to a non-boolean: that's a parse error.
+  r = config_get_lines("UseBridges ambidextrous", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_int_op(v, OP_EQ, SETOPT_ERR_PARSE);
+  tt_str_op(msg, OP_EQ,
+            "Could not parse UseBridges: Unrecognized value ambidextrous. "
+            "Allowed values are 0 and 1.");
+  tor_free(msg);
+  config_free_lines(lines);
+
+  // this didn't change.
+  tt_str_op(get_options()->Nickname, OP_EQ, "Hemiramphinae");
+
+ done:
+  config_free_lines(lines);
+  tor_free(msg);
+  UNMOCK(options_act);
+  UNMOCK(options_act_reversible);
+}
+
+#ifndef COCCI
 #define LOCAL_VALIDATE_TEST(name) \
   { "validate__" #name, test_options_validate__ ## name, TT_FORK, NULL, NULL }
+#endif
 
 struct testcase_t options_tests[] = {
   { "validate", test_options_validate, TT_FORK, NULL, NULL },
@@ -4199,11 +4305,11 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(logs),
   LOCAL_VALIDATE_TEST(authdir),
   LOCAL_VALIDATE_TEST(relay_with_hidden_services),
+  LOCAL_VALIDATE_TEST(listen_ports),
   LOCAL_VALIDATE_TEST(transproxy),
   LOCAL_VALIDATE_TEST(exclude_nodes),
   LOCAL_VALIDATE_TEST(node_families),
   LOCAL_VALIDATE_TEST(token_bucket),
-  LOCAL_VALIDATE_TEST(recommended_packages),
   LOCAL_VALIDATE_TEST(fetch_dir),
   LOCAL_VALIDATE_TEST(conn_limit),
   LOCAL_VALIDATE_TEST(paths_needed),
@@ -4233,5 +4339,10 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(virtual_addr),
   LOCAL_VALIDATE_TEST(testing_options),
   LOCAL_VALIDATE_TEST(accel),
+  { "init_logs/granularity", test_options_init_logs_granularity, TT_FORK,
+    NULL, NULL },
+  { "init_logs/quiet", test_options_init_logs_quiet, TT_FORK,
+    NULL, NULL },
+  { "trial_assign", test_options_trial_assign, TT_FORK, NULL, NULL },
   END_OF_TESTCASES              /*  */
 };
diff --git a/src/test/test_options_act.c b/src/test/test_options_act.c
new file mode 100644
index 0000000000..942584bffd
--- /dev/null
+++ b/src/test/test_options_act.c
@@ -0,0 +1,272 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONFIG_PRIVATE
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/encoding/confline.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "test/test_helpers.h"
+
+#ifndef _WIN32
+#include <sys/stat.h>
+
+/**
+ * Check whether fname is readable. On success set
+ * *<b>is_group_readable_out</b> to as appropriate and return 0. On failure
+ * return -1.
+ */
+static int
+get_file_mode(const char *fname, unsigned *permissions_out)
+{
+  struct stat st;
+  int r = stat(fname, &st);
+  if (r < 0)
+    return -1;
+  *permissions_out = (unsigned) st.st_mode;
+  return 0;
+}
+#define assert_mode(fn,mask,expected) STMT_BEGIN                 \
+  unsigned mode_;                                                \
+  int tmp_ = get_file_mode((fn), &mode_);                        \
+  if (tmp_ < 0) {                                                \
+    TT_DIE(("Couldn't stat %s: %s", (fn), strerror(errno)));     \
+  }                                                              \
+  if ((mode_ & (mask)) != (expected)) {                          \
+    TT_DIE(("Bad mode %o on %s", mode_, (fn)));                  \
+  }                                                              \
+  STMT_END
+#else /* defined(_WIN32) */
+/* "group-readable" isn't meaningful on windows */
+#define assert_mode(fn,mask,expected) STMT_NIL
+#endif /* !defined(_WIN32) */
+
+static or_options_t *mock_opts;
+static const or_options_t *
+mock_get_options(void)
+{
+  return mock_opts;
+}
+
+static void
+test_options_act_create_dirs(void *arg)
+{
+  (void)arg;
+  MOCK(get_options, mock_get_options);
+  char *msg = NULL;
+  or_options_t *opts = mock_opts = options_new();
+
+  /* We're testing options_create_directories(), which assumes that
+     validate_data_directories() has already been called, and all of
+     KeyDirectory, DataDirectory, and CacheDirectory are set. */
+
+  /* Success case 1: all directories are the default */
+  char *fn;
+  fn = tor_strdup(get_fname_rnd("ddir"));
+  opts->DataDirectory = tor_strdup(fn);
+  opts->CacheDirectory = tor_strdup(fn);
+  tor_asprintf(&opts->KeyDirectory, "%s/keys", fn);
+  opts->DataDirectoryGroupReadable = 1;
+  opts->CacheDirectoryGroupReadable = -1; /* default. */
+  int r = options_create_directories(&msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->DataDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->CacheDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->KeyDirectory));
+  assert_mode(opts->DataDirectory, 0777, 0750);
+  assert_mode(opts->KeyDirectory, 0777, 0700);
+  tor_free(fn);
+  tor_free(opts->KeyDirectory);
+  or_options_free(opts);
+
+  /* Success case 2: all directories are different. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  opts->CacheDirectoryGroupReadable = 1; // cache directory group readable
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->DataDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->CacheDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->KeyDirectory));
+  assert_mode(opts->DataDirectory, 0777, 0700);
+  assert_mode(opts->KeyDirectory, 0777, 0700);
+  assert_mode(opts->CacheDirectory, 0777, 0750);
+  tor_free(fn);
+  or_options_free(opts);
+
+  /* Success case 3: all directories are the same. */
+  opts = mock_opts = options_new();
+  fn = tor_strdup(get_fname_rnd("ddir"));
+  opts->DataDirectory = tor_strdup(fn);
+  opts->CacheDirectory = tor_strdup(fn);
+  opts->KeyDirectory = tor_strdup(fn);
+  opts->DataDirectoryGroupReadable = 1;
+  opts->CacheDirectoryGroupReadable = -1; /* default. */
+  opts->KeyDirectoryGroupReadable = -1; /* default */
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->DataDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->CacheDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->KeyDirectory));
+  assert_mode(opts->DataDirectory, 0777, 0750);
+  assert_mode(opts->KeyDirectory, 0777, 0750);
+  assert_mode(opts->CacheDirectory, 0777, 0750);
+  tor_free(fn);
+  or_options_free(opts);
+
+  /* Failure case 1: Can't make datadir. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  write_str_to_file(opts->DataDirectory, "foo", 0);
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_assert(!strcmpstart(msg, "Couldn't create private data directory"));
+  or_options_free(opts);
+  tor_free(msg);
+
+  /* Failure case 2: Can't make keydir. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  write_str_to_file(opts->KeyDirectory, "foo", 0);
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_assert(!strcmpstart(msg, "Couldn't create private data directory"));
+  or_options_free(opts);
+  tor_free(msg);
+
+  /* Failure case 3: Can't make cachedir. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  write_str_to_file(opts->CacheDirectory, "foo", 0);
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_assert(!strcmpstart(msg, "Couldn't create private data directory"));
+  tor_free(fn);
+  or_options_free(opts);
+  tor_free(msg);
+
+ done:
+  UNMOCK(get_options);
+  or_options_free(opts);
+  mock_opts = NULL;
+  tor_free(fn);
+  tor_free(msg);
+}
+
+static void
+test_options_act_log_transition(void *arg)
+{
+  (void)arg;
+  or_options_t *opts = mock_opts = options_new();
+  or_options_t *old_opts = NULL;
+  opts->LogTimeGranularity = 1000;
+  opts->SafeLogging_ = SAFELOG_SCRUB_ALL;
+  struct log_transaction_t *lt = NULL;
+  char *msg = NULL;
+  MOCK(get_options, mock_get_options);
+
+  tt_ptr_op(opts->Logs, OP_EQ, NULL);
+  config_line_append(&opts->Logs, "Log", "notice stdout");
+  lt = options_start_log_transaction(NULL, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+
+  // commit, see that there is a change.
+  options_commit_log_transaction(lt);
+  lt=NULL;
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_NOTICE);
+
+  // Now drop to debug.
+  old_opts = opts;
+  opts = mock_opts = options_new();
+  opts->LogTimeGranularity = 1000;
+  opts->SafeLogging_ = SAFELOG_SCRUB_ALL;
+  config_line_append(&opts->Logs, "Log", "debug stdout");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+
+  setup_full_capture_of_logs(LOG_NOTICE);
+  options_commit_log_transaction(lt);
+  lt=NULL;
+  expect_single_log_msg_containing("may contain sensitive information");
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+  // Turn off SafeLogging
+  or_options_free(old_opts);
+  mock_clean_saved_logs();
+  old_opts = opts;
+  opts = mock_opts = options_new();
+  opts->SafeLogging_ = SAFELOG_SCRUB_NONE;
+  opts->LogTimeGranularity = 1000;
+  config_line_append(&opts->Logs, "Log", "debug stdout");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+  options_commit_log_transaction(lt);
+  lt=NULL;
+  expect_single_log_msg_containing("may contain sensitive information");
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+  // Try rolling back.
+  or_options_free(old_opts);
+  mock_clean_saved_logs();
+  old_opts = opts;
+  opts = mock_opts = options_new();
+  opts->SafeLogging_ = SAFELOG_SCRUB_NONE;
+  opts->LogTimeGranularity = 1000;
+  config_line_append(&opts->Logs, "Log", "notice stdout");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+  options_rollback_log_transaction(lt);
+  expect_no_log_entry();
+  lt = NULL;
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+  // Now try some bad options.
+  or_options_free(opts);
+  mock_clean_saved_logs();
+  opts = mock_opts = options_new();
+  opts->LogTimeGranularity = 1000;
+  config_line_append(&opts->Logs, "Log", "warn blaznert");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(!lt);
+  tt_str_op(msg, OP_EQ, "Failed to init Log options. See logs for details.");
+  expect_single_log_msg_containing("Couldn't parse");
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+ done:
+  UNMOCK(get_options);
+  or_options_free(opts);
+  or_options_free(old_opts);
+  tor_free(msg);
+  if (lt)
+    options_rollback_log_transaction(lt);
+  teardown_capture_of_logs();
+}
+
+#ifndef COCCI
+#define T(name) { #name, test_options_act_##name, TT_FORK, NULL, NULL }
+#endif
+
+struct testcase_t options_act_tests[] = {
+  T(create_dirs),
+  T(log_transition),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c
new file mode 100644
index 0000000000..9c22266da1
--- /dev/null
+++ b/src/test/test_parsecommon.c
@@ -0,0 +1,594 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/memarea/memarea.h"
+#include "lib/encoding/binascii.h"
+#include "feature/dirparse/parsecommon.h"
+#include "test/log_test_helpers.h"
+
+static void
+test_parsecommon_tokenize_string_null(void *arg)
+{
+
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  const char *str_with_null = "a\0bccccccccc";
+
+  int retval =
+  tokenize_string(area, str_with_null,
+                  str_with_null + 3,
+                  tokens, NULL, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_multiple_lines(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, GE(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup(
+          "hibernating 0\nuptime 1024\n"
+          "published 2018-10-15 10:00:00\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(smartlist_len(tokens), OP_EQ, 3);
+  directory_token_t *token = smartlist_get(tokens, 0);
+
+  tt_int_op(token->tp, OP_EQ, K_HIBERNATING);
+
+  token = smartlist_get(tokens, 1);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+
+  token = smartlist_get(tokens, 2);
+
+  tt_int_op(token->tp, OP_EQ, K_PUBLISHED);
+
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_min_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(2), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // Missing "uptime"
+  char *str = tor_strdup("uptime 1024\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_max_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "uptime" expected once, but occurs twice in input.
+  char *str = tor_strdup(
+                  "uptime 1024\nuptime 2048\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_start(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the first line.
+  char *str = tor_strdup(
+                  "uptime 1024\nclient-name Alice\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_end(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the last line.
+  char *str = tor_strdup(
+                  "client-name Alice\nuptime 1024\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_no_annotations(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_success(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "1024");
+  tt_assert(!token->object_type);
+  tt_int_op(token->object_size, OP_EQ, 0);
+  tt_assert(!token->object_body);
+
+  tt_ptr_op(*s, OP_EQ, end);
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_concat_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "proto A=1 B=2";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_PROTO);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "A=1 B=2");
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_parse_keys(void *arg)
+{
+  (void)arg;
+
+  memarea_t *area = memarea_new();
+  const char *str =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n"
+    "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n"
+    "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = (const char **)&str;
+  directory_token_t *token = NULL;
+  directory_token_t *token2 = NULL;
+
+  token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024);
+
+  token = get_next_token(area, s, end, &rule);
+  tt_assert(token);
+
+  tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY);
+  tt_int_op(token->n_args, OP_EQ, 0);
+  tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY");
+  tt_int_op(token->object_size, OP_EQ, 140);
+  tt_assert(token->object_body);
+  tt_assert(token->key);
+  tt_assert(!token->error);
+
+  const char *str2 =
+    "client-key\n"
+    "-----BEGIN RSA PRIVATE KEY-----\n"
+    "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n"
+    "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n"
+    "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n"
+    "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n"
+    "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n"
+    "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n"
+    "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n"
+    "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n"
+    "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n"
+    "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n"
+    "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n"
+    "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n"
+    "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"
+    "-----END RSA PRIVATE KEY-----\n";
+
+  const char *end2 = str2 + strlen(str2);
+  const char **s2 = (const char **)&str2;
+
+  token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS,
+                           NEED_SKEY_1024);
+
+  token2 = get_next_token(area, s2, end2, &rule2);
+  tt_assert(token2);
+
+  tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY);
+  tt_int_op(token2->n_args, OP_EQ, 0);
+  tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY");
+  tt_int_op(token2->object_size, OP_EQ, 608);
+  tt_assert(token2->object_body);
+  tt_assert(token2->key);
+  tt_assert(!token->error);
+
+ done:
+  if (token) token_clear(token);
+  if (token2) token_clear(token2);
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_object(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE);
+  tt_int_op(token->n_args, OP_EQ, 2);
+  tt_str_op(token->args[0], OP_EQ,
+                  "0232AF901C31A04EE9848595AF9BB7620D4C5B2E");
+  tt_str_op(token->args[1], OP_EQ,
+                  "CD1FD971855430880D3C31E0331C5C55800C2F79");
+
+  tt_assert(!token->error);
+
+  char decoded[256];
+  const char *signature =
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+  tt_assert(signature);
+  size_t signature_len = strlen(signature);
+  base64_decode(decoded, sizeof(decoded), signature, signature_len);
+
+  tt_str_op(token->object_type, OP_EQ, "SIGNATURE");
+  tt_int_op(token->object_size, OP_EQ, 256);
+  tt_mem_op(token->object_body, OP_EQ, decoded, 256);
+
+  tt_assert(!token->key);
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_err_too_many_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024 1024 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too many arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_too_few_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too few arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_obj_missing_endline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_beginline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-Z---\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_tag_mismatch(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SOMETHINGELSE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ,
+            "Malformed object: mismatched end tag SIGNATURE");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_base64(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "%%@%%%%%%%!!!'\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+#define PARSECOMMON_TEST(name) \
+  { #name, test_parsecommon_ ## name, 0, NULL, NULL }
+
+struct testcase_t parsecommon_tests[] = {
+  PARSECOMMON_TEST(tokenize_string_null),
+  PARSECOMMON_TEST(tokenize_string_multiple_lines),
+  PARSECOMMON_TEST(tokenize_string_min_cnt),
+  PARSECOMMON_TEST(tokenize_string_max_cnt),
+  PARSECOMMON_TEST(tokenize_string_at_start),
+  PARSECOMMON_TEST(tokenize_string_at_end),
+  PARSECOMMON_TEST(tokenize_string_no_annotations),
+  PARSECOMMON_TEST(get_next_token_success),
+  PARSECOMMON_TEST(get_next_token_concat_args),
+  PARSECOMMON_TEST(get_next_token_parse_keys),
+  PARSECOMMON_TEST(get_next_token_object),
+  PARSECOMMON_TEST(get_next_token_err_too_many_args),
+  PARSECOMMON_TEST(get_next_token_err_too_few_args),
+  PARSECOMMON_TEST(get_next_token_err_obj_missing_endline),
+  PARSECOMMON_TEST(get_next_token_err_bad_beginline),
+  PARSECOMMON_TEST(get_next_token_err_tag_mismatch),
+  PARSECOMMON_TEST(get_next_token_err_bad_base64),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_parseconf.sh b/src/test/test_parseconf.sh
new file mode 100755
index 0000000000..4fe27d9f5d
--- /dev/null
+++ b/src/test/test_parseconf.sh
@@ -0,0 +1,655 @@
+#!/bin/sh
+# Copyright 2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+# Integration test script for verifying that Tor configurations are parsed as
+# we expect.
+#
+# Valid configurations are tested with --dump-config, which parses and
+# validates the configuration before writing it out.  We then make sure that
+# the result is what we expect, before parsing and dumping it again to make
+# sure that there is no change. Optionally, we can also test the log messages
+# with --verify-config.
+#
+# Invalid configurations are tested with --verify-config, which parses
+# and validates the configuration. We capture its output and make sure that
+# it contains the error message we expect.
+#
+# When tor is compiled with different libraries or modules, some
+# configurations may have different results. We can specify these result
+# variants using additional result files.
+
+# This script looks for its test cases as individual directories in
+# src/test/conf_examples/.  Each test may have these files:
+#
+# Configuration Files
+#
+# torrc -- Usually needed. This file is passed to Tor on the command line
+#      with the "-f" flag. (If you omit it, you'll test Tor's behavior when
+#      it receives a nonexistent configuration file.)
+#
+# torrc.defaults -- Optional. If present, it is passed to Tor on the command
+#      line with the --defaults-torrc option. If this file is absent, an empty
+#      file is passed instead to prevent Tor from reading the system defaults.
+#
+# cmdline -- Optional. If present, it contains command-line arguments that
+#      will be passed to Tor.
+#
+# (included torrc files or directories) -- Optional. Additional files can be
+#      included in configuration, using the "%include" directive. Files or
+#      directories can be included in any of the config files listed above.
+#      Include paths should be specified relative to the test case directory.
+#
+# Result Files
+#
+# expected -- If this file is present, then it should be the expected result
+#      of "--dump-config short" for this test case.  Exactly one of
+#      "expected" or "error" must be present, or the test will fail.
+#
+# expected_log -- Optional. If this file is present, then it contains a regex
+#      that must be matched by some line in the output of "--verify-config",
+#      which must succeed. Only used if "expected" is also present.
+#
+# error -- If this file is present, then it contains a regex that must be
+#      matched by some line in the output of "--verify-config", which must
+#      fail. Exactly one of "expected" or "error" must be present, or the
+#      test will fail.
+#
+# {expected,expected_log,error}_${TOR_LIBS_ENABLED}* -- If this file is
+#      present, then the outcome is different when some optional libraries are
+#      enabled. If there is no result file matching the exact list of enabled
+#      libraries, the script searches for result files with one or more of
+#      those libraries disabled. The search terminates at the standard result
+#      file. If expected* is present, the script also searches for
+#      expected_log*.
+#
+#      For example:
+#      A test that succeeds, regardless of any enabled libraries:
+#       - expected
+#      A test that has a different result if the nss library is enabled
+#      (but the same result if any other library is enabled). We also check
+#      the log output in this test:
+#       - expected
+#       - expected_log
+#       - expected_nss
+#       - expected_log_nss
+#      A test that fails if the lzma and zstd modules are *not* enabled:
+#       - error
+#       - expected_lzma_zstd
+#
+# {expected,expected_log,error}*_no_${TOR_MODULES_DISABLED} -- If this file is
+#      present, then the outcome is different when some modules are disabled.
+#      If there is no result file matching the exact list of disabled modules,
+#      the standard result file is used. If expected* is present, the script
+#      also searches for expected_log*.
+#
+#      For example:
+#      A test that succeeds, regardless of any disabled modules:
+#       - expected
+#      A test that has a different result if the relay module is disabled
+#      (but the same result if just the dirauth module is disabled):
+#       - expected
+#       - expected_no_relay_dirauth
+#      A test that fails if the dirauth module is disabled:
+#       - expected
+#       - error_no_dirauth
+#       - error_no_relay_dirauth
+#      (Disabling the relay module also disables dirauth module. But we don't
+#      want to encode that knowledge in this test script, so we supply a
+#      separate result file for every combination of disabled modules that
+#      has a different result.)
+
+umask 077
+set -e
+
+MYNAME="$0"
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if test -d "$f"; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if test $# -ge 1; then
+  TOR_BINARY="$1"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "Using Tor binary '$TOR_BINARY'."
+
+# make a safe space for temporary files
+DATA_DIR=$(mktemp -d -t tor_parseconf_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# This is where we look for examples
+EXAMPLEDIR="$(dirname "$0")"/conf_examples
+
+case "$(uname -s)" in
+    CYGWIN*) WINDOWS=1;;
+    MINGW*) WINDOWS=1;;
+    MSYS*) WINDOWS=1;;
+    *) WINDOWS=0;;
+esac
+
+####
+# BUG WORKAROUND FOR 31757:
+#  On Appveyor, it seems that Tor sometimes randomly fails to produce
+#  output with --dump-config.  Whil we are figuring this out, do not treat
+#  windows errors as hard failures.
+####
+if test "$WINDOWS" = 1; then
+    EXITCODE=0
+else
+    EXITCODE=1
+fi
+
+FINAL_EXIT=0
+NEXT_TEST=
+
+# Log a failure message to stderr, using $@ as a printf string and arguments
+# Set NEXT_TEST to "yes" and FINAL_EXIT to $EXITCODE.
+fail_printf()
+{
+    printf "FAIL: " >&2
+    # The first argument is a printf string, so this warning is spurious
+    # shellcheck disable=SC2059
+    printf "$@" >&2
+    NEXT_TEST="yes"
+    FINAL_EXIT=$EXITCODE
+}
+
+# Log a failure message to stderr, using $@ as a printf string and arguments
+# Exit with status $EXITCODE.
+die_printf()
+{
+    printf "FAIL: CRITICAL error in '%s':" "$MYNAME" >&2
+    # The first argument is a printf string, so this warning is spurious
+    # shellcheck disable=SC2059
+    printf "$@" >&2
+    exit $EXITCODE
+}
+
+if test "$WINDOWS" = 1; then
+    FILTER="dos2unix"
+else
+    FILTER="cat"
+fi
+
+EMPTY="${DATA_DIR}/EMPTY"
+touch "$EMPTY" || die_printf "Couldn't create empty file '%s'.\\n" \
+                             "$EMPTY"
+NON_EMPTY="${DATA_DIR}/NON_EMPTY"
+echo "This pattern should not match any log messages" \
+     > "$NON_EMPTY" || die_printf "Couldn't create non-empty file '%s'.\\n" \
+                                  "$NON_EMPTY"
+
+STANDARD_LIBS="libevent\\|openssl\\|zlib"
+# Lib names are restricted to [a-z0-9]* at the moment
+# We don't actually want to support foreign accents here
+# shellcheck disable=SC2018,SC2019
+TOR_LIBS_ENABLED="$("$TOR_BINARY" --verify-config \
+                      -f "$EMPTY" --defaults-torrc "$EMPTY" \
+                    | sed -n 's/.* Tor .* running on .* with\(.*\)\./\1/p' \
+                    | tr 'A-Z' 'a-z' | tr ',' '\n' \
+                    | grep -v "$STANDARD_LIBS" | grep -v "n/a" \
+                    | sed 's/\( and\)* \(lib\)*\([a-z0-9]*\) .*/\3/' \
+                    | sort | tr '\n' '_')"
+# Remove the last underscore, if there is one
+TOR_LIBS_ENABLED=${TOR_LIBS_ENABLED%_}
+
+# If we ever have more than 3 optional libraries, we'll need more code here
+TOR_LIBS_ENABLED_COUNT="$(echo "$TOR_LIBS_ENABLED_SEARCH" \
+                          | tr ' ' '\n' | wc -l)"
+if test "$TOR_LIBS_ENABLED_COUNT" -gt 3; then
+    die_printf "Can not handle more than 3 optional libraries.\\n"
+fi
+# Brute-force the combinations of libraries
+TOR_LIBS_ENABLED_SEARCH_3="$(echo "$TOR_LIBS_ENABLED" \
+    | sed -n \
+      's/^\([^_]*\)_\([^_]*\)_\([^_]*\)$/_\1_\2 _\1_\3 _\2_\3 _\1 _\2 _\3/p')"
+TOR_LIBS_ENABLED_SEARCH_2="$(echo "$TOR_LIBS_ENABLED" \
+    | sed -n 's/^\([^_]*\)_\([^_]*\)$/_\1 _\2/p')"
+TOR_LIBS_ENABLED_SEARCH="_$TOR_LIBS_ENABLED \
+                           $TOR_LIBS_ENABLED_SEARCH_3 \
+                           $TOR_LIBS_ENABLED_SEARCH_2"
+TOR_LIBS_ENABLED_SEARCH="$(echo "$TOR_LIBS_ENABLED_SEARCH" | tr ' ' '\n' \
+                           | grep -v '^_*$' | tr '\n' ' ')"
+
+TOR_MODULES_DISABLED="$("$TOR_BINARY" --list-modules | grep ': no' \
+                        | cut -d ':' -f1 | sort | tr '\n' '_')"
+# Remove the last underscore, if there is one
+TOR_MODULES_DISABLED=${TOR_MODULES_DISABLED%_}
+
+echo "Tor is configured with:"
+echo "Optional Libraries: ${TOR_LIBS_ENABLED:-(None)}"
+if test "$TOR_LIBS_ENABLED"; then
+    echo "Optional Library Search List: $TOR_LIBS_ENABLED_SEARCH"
+fi
+echo "Disabled Modules: ${TOR_MODULES_DISABLED:-(None)}"
+
+# Yes, unix uses "0" for a successful command
+TRUE=0
+FALSE=1
+
+# Run tor --verify-config on the torrc $1, and defaults torrc $2, which may
+# be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted.
+# Send tor's standard output to stderr.
+log_verify_config()
+{
+    # show the command we're about to execute
+    # log_verify_config() is only called when we've failed
+    printf "Tor --verify-config said:\\n" >&2
+    printf "$ %s %s %s %s %s %s %s\\n" \
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  "$3" \
+                  >&2
+    # We need cmdline unquoted
+    # shellcheck disable=SC2086
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  $3 \
+                  >&2 \
+        || true
+}
+
+# Run "tor --dump-config short" on the torrc $1, and defaults torrc $2, which
+# may be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4.
+#
+# Set $FULL_TOR_CMD to the tor command line that was executed.
+#
+# If tor fails, fail_printf() using the file name $5, and context $6,
+# which may be an empty string. Then run log_verify_config().
+dump_config()
+{
+    if test "$6"; then
+        CONTEXT=" $6"
+    else
+        CONTEXT=""
+    fi
+
+    # keep the command we're about to execute, and show if it we fail
+    FULL_TOR_CMD=$(printf "$ %s %s %s %s %s %s %s %s" \
+         "$TOR_BINARY" --dump-config short \
+                       -f "$1" \
+                       --defaults-torrc "$2" \
+                       "$3"
+             )
+    # We need cmdline unquoted
+    # shellcheck disable=SC2086
+    if ! "$TOR_BINARY" --dump-config short \
+                       -f "$1" \
+                       --defaults-torrc "$2" \
+                       $3 \
+                       > "$4"; then
+        fail_printf "'%s': Tor --dump-config reported an error%s:\\n%s\\n" \
+                    "$5" \
+                    "$CONTEXT" \
+                    "$FULL_TOR_CMD"
+        log_verify_config "$1" \
+                          "$2" \
+                          "$3"
+    fi
+}
+
+# Run "$FILTER" on the input $1.
+# Send the standard output to $2.
+# If tor fails, log a failure message using the file name $3, and context $4,
+# which may be an empty string.
+filter()
+{
+    if test "$4"; then
+        CONTEXT=" $4"
+    else
+        CONTEXT=""
+    fi
+
+    "$FILTER" "$1" \
+              > "$2" \
+        || fail_printf "'%s': Filter '%s' reported an error%s.\\n" \
+                       "$3" \
+                       "$FILTER" \
+                       "$CONTEXT"
+}
+
+# Compare the expected file $1, and output file $2.
+#
+# If they are different, fail. Log the differences between the files.
+# Run log_verify_config() with torrc $3, defaults torrc $4, and command
+# line $5, to log Tor's error messages.
+#
+# If the file contents are identical, returns true. Otherwise, return false.
+#
+# Log failure messages using fail_printf(), with the expected file name,
+# context $6, which may be an empty string, and the tor command line $7.
+check_diff()
+{
+    if test "$6"; then
+        CONTEXT=" $6"
+    else
+        CONTEXT=""
+    fi
+
+    if cmp "$1" "$2" > /dev/null; then
+        return "$TRUE"
+    else
+        fail_printf "'%s': Tor --dump-config said%s:\\n%s\\n" \
+                    "$1" \
+                    "$CONTEXT" \
+                    "$7"
+        diff -u "$1" "$2" >&2 \
+            || true
+        log_verify_config "$3" \
+                          "$4" \
+                          "$5"
+        return "$FALSE"
+    fi
+}
+
+# Run "tor --dump-config short" on the torrc $1, and defaults torrc $2, which
+# may be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4, after running $FILTER
+# on it.
+#
+# If tor fails, run log_verify_config().
+#
+# Compare the expected file $5, and output file. If they are different, fail.
+# If this is the first step that failed in this test, run log_verify_config().
+#
+# If the file contents are identical, returns true. Otherwise, return false,
+# and log the differences between the files.
+#
+# Log failure messages using fail_printf(), with the expected file name, and
+# context $6, which may be an empty string.
+check_dump_config()
+{
+    OUTPUT="$4"
+    OUTPUT_RAW="${OUTPUT}_raw"
+
+    FULL_TOR_CMD=
+    dump_config "$1" \
+                "$2" \
+                "$3" \
+                "$OUTPUT_RAW" \
+                "$5" \
+                "$6"
+
+    filter "$OUTPUT_RAW" \
+           "$OUTPUT" \
+           "$5" \
+           "$6"
+
+    if check_diff "$5" \
+                  "$OUTPUT" \
+                  "$1" \
+                  "$2" \
+                  "$3" \
+                  "$6" \
+                  "$FULL_TOR_CMD"; then
+        return "$TRUE"
+    else
+        return "$FALSE"
+    fi
+}
+
+# Check if $1 is an empty file.
+# If it is, fail_printf() using $2 as the type of file.
+# Returns true if the file is empty, false otherwise.
+check_empty_pattern()
+{
+    if ! test -s "$1"; then
+        fail_printf "%s file '%s' is empty, and will match any output.\\n" \
+                    "$2" \
+                    "$1"
+        return "$TRUE"
+    else
+        return "$FALSE"
+    fi
+}
+
+# Run tor --verify-config on the torrc $1, and defaults torrc $2, which may
+# be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4.
+#
+# Set $FULL_TOR_CMD to the tor command line that was executed.
+#
+# If tor's exit status does not match the boolean $5, fail_printf()
+# using the file name $6, and context $7, which is required.
+verify_config()
+{
+    RESULT=$TRUE
+
+    # keep the command we're about to execute, and show if it we fail
+    FULL_TOR_CMD=$(printf "$ %s %s %s %s %s %s %s" \
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  "$3"
+             )
+    # We need cmdline unquoted
+    # shellcheck disable=SC2086
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  $3 \
+                  > "$4" || RESULT=$FALSE
+
+    # Convert the actual and expected results to boolean, and compare
+    if test $((! (! RESULT))) -ne $((! (! $5))); then
+        fail_printf "'%s': Tor --verify-config did not %s:\\n%s\\n" \
+                    "$6" \
+                    "$7" \
+                    "$FULL_TOR_CMD"
+        cat "$4" >&2
+    fi
+}
+
+# Check for the patterns in the match file $1, in the output file $2.
+# Uses grep with the entire contents of the match file as the pattern.
+# (Not "grep -f".)
+#
+# If the pattern does not match any lines in the output file, fail.
+# Log the pattern, and the entire contents of the output file.
+#
+# Log failure messages using fail_printf(), with the match file name,
+# context $3, and tor command line $4, which are required.
+check_pattern()
+{
+    expect_log="$(cat "$1")"
+    if ! grep "$expect_log" "$2" > /dev/null; then
+        fail_printf "Expected %s '%s':\\n%s\\n" \
+                    "$3" \
+                    "$1" \
+                    "$expect_log"
+        printf "Tor --verify-config said:\\n%s\\n" \
+               "$4" >&2
+        cat "$2" >&2
+    fi
+}
+
+# Run tor --verify-config on the torrc $1, and defaults torrc $2, which may
+# be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4.
+#
+# If tor's exit status does not match the boolean $5, fail.
+#
+# Check for the patterns in the match file $6, in the output file.
+# Uses grep with the entire contents of the match file as the pattern.
+# (Not "grep -f".) The match file must not be empty.
+#
+# If the pattern does not match any lines in the output file, fail.
+# Log the pattern, and the entire contents of the output file.
+#
+# Log failure messages using fail_printf(), with the match file name,
+# and context $7, which is required.
+check_verify_config()
+{
+    if check_empty_pattern "$6" "$7"; then
+        return
+    fi
+
+    FULL_TOR_CMD=
+    verify_config "$1" \
+                  "$2" \
+                  "$3" \
+                  "$4" \
+                  "$5" \
+                  "$6" \
+                  "$7"
+
+    check_pattern "$6" \
+                  "$4" \
+                  "$7" \
+                  "$FULL_TOR_CMD"
+}
+
+for dir in "${EXAMPLEDIR}"/*; do
+    NEXT_TEST=
+
+    if ! test -d "$dir"; then
+       # Only count directories.
+       continue
+    fi
+
+    testname="$(basename "${dir}")"
+    # We use printf since "echo -n" is not standard
+    printf "%s: " \
+           "$testname"
+
+    PREV_DIR="$(pwd)"
+    cd "$dir"
+
+    if test -f "./torrc.defaults"; then
+        DEFAULTS="./torrc.defaults"
+    else
+        DEFAULTS="${DATA_DIR}/EMPTY"
+    fi
+
+    if test -f "./cmdline"; then
+        CMDLINE="$(cat ./cmdline)"
+    else
+        CMDLINE=""
+    fi
+
+    EXPECTED=
+    EXPECTED_LOG=
+    ERROR=
+    # Search for a custom result file for any combination of enabled optional
+    # libraries
+    # The libs in the list are [A-Za-z0-9_]* and space-separated.
+    # shellcheck disable=SC2086
+    for lib_suffix in $TOR_LIBS_ENABLED_SEARCH ""; do
+        # Search for a custom result file for any disabled modules
+        for mod_suffix in "_no_${TOR_MODULES_DISABLED}" ""; do
+            suffix="${lib_suffix}${mod_suffix}"
+
+            if test -f "./expected${suffix}"; then
+
+                # Check for broken configs
+                if test -f "./error${suffix}"; then
+                    fail_printf "Found both '%s' and '%s'.%s\\n" \
+                                "${dir}/expected${suffix}" \
+                                "${dir}/error${suffix}" \
+                                "(Only one of these files should exist.)"
+                    break
+                fi
+
+                EXPECTED="./expected${suffix}"
+                if test -f "./expected_log${suffix}"; then
+                    EXPECTED_LOG="./expected_log${suffix}"
+                fi
+                break
+
+            elif test -f "./error${suffix}"; then
+                ERROR="./error${suffix}"
+                break
+            fi
+        done
+
+        # Exit as soon as the inner loop finds a file, or fails
+        if test -f "$EXPECTED" || test -f "$ERROR" || test "$NEXT_TEST"; then
+            break
+        fi
+    done
+
+    if test "$NEXT_TEST"; then
+        # The test failed inside the file search loop: go to the next test
+        continue
+    elif test -f "$EXPECTED"; then
+        # This case should succeed: run dump-config and see if it does.
+
+        if check_dump_config "./torrc" \
+                             "$DEFAULTS" \
+                             "$CMDLINE" \
+                             "${DATA_DIR}/output.${testname}" \
+                             "$EXPECTED" \
+                             ""; then
+            # Check round-trip.
+            check_dump_config "${DATA_DIR}/output.${testname}" \
+                              "$EMPTY" \
+                              "" \
+                              "${DATA_DIR}/output_2.${testname}" \
+                              "$EXPECTED" \
+                              "on round-trip" || true
+        fi
+
+        if test -f "$EXPECTED_LOG"; then
+            # This case should succeed: run verify-config and see if it does.
+
+            check_verify_config "./torrc" \
+                                "$DEFAULTS" \
+                                "$CMDLINE" \
+                                "${DATA_DIR}/output_log.${testname}" \
+                                "$TRUE" \
+                                "$EXPECTED_LOG" \
+                                "log success"
+        else
+            printf "\\nNOTICE: Missing '%s_log' file:\\n" \
+                   "$EXPECTED" >&2
+            log_verify_config "./torrc" \
+                              "$DEFAULTS" \
+                              "$CMDLINE"
+        fi
+
+   elif test -f "$ERROR"; then
+        # This case should fail: run verify-config and see if it does.
+
+        check_verify_config "./torrc" \
+                            "$DEFAULTS" \
+                            "$CMDLINE" \
+                            "${DATA_DIR}/output.${testname}" \
+                            "$FALSE" \
+                            "$ERROR" \
+                            "log error"
+    else
+        # This case is not actually configured with a success or a failure.
+        # call that an error.
+        fail_printf "Did not find ${dir}/*expected or ${dir}/*error.\\n"
+    fi
+
+    if test -z "$NEXT_TEST"; then
+        echo "OK"
+    fi
+
+    cd "$PREV_DIR"
+
+done
+
+exit "$FINAL_EXIT"
diff --git a/src/test/test_pem.c b/src/test/test_pem.c
index 9eb99181e2..9772be124b 100644
--- a/src/test/test_pem.c
+++ b/src/test/test_pem.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c
index 4a53639dad..b7f1785805 100644
--- a/src/test/test_periodic_event.c
+++ b/src/test/test_periodic_event.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,6 +19,7 @@
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_service.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/periodic.h"
 
 /** Helper function: This is replaced in some tests for the event callbacks so
@@ -50,16 +51,21 @@ test_pe_initialize(void *arg)
    * need to run the main loop and then wait for a second delaying the unit
    * tests. Instead, we'll test the callback work indepedently elsewhere. */
   initialize_periodic_events();
+  periodic_events_connect_all();
+  set_network_participation(false);
+  rescan_periodic_events(get_options());
 
   /* Validate that all events have been set up. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     tt_assert(item->ev);
     tt_assert(item->fn);
     tt_u64_op(item->last_action_time, OP_EQ, 0);
     /* Every event must have role(s) assign to it. This is done statically. */
     tt_u64_op(item->roles, OP_NE, 0);
-    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
  done:
@@ -79,17 +85,20 @@ test_pe_launch(void *arg)
    * network gets enabled. */
   consider_hibernation(time(NULL));
 
+  set_network_participation(true);
+
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
   options = get_options_mutable();
   options->SocksPort_set = 1;
   periodic_events_on_new_options(options);
+
 #if 0
   /* Lets make sure that before intialization, we can't scan the periodic
    * events list and launch them. Lets try by being a Client. */
@@ -99,20 +108,20 @@ test_pe_launch(void *arg)
     periodic_event_item_t *item = &periodic_events[i];
     tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
   }
-#endif
+#endif /* 0 */
 
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Now that we've initialized, rescan the list to launch. */
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
-    } else {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
-    }
+  int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL|
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = !!(item->roles & mask);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
     // enabled or not, the event has not yet been run.
     tt_u64_op(item->last_action_time, OP_EQ, 0);
   }
@@ -124,10 +133,11 @@ test_pe_launch(void *arg)
 
   unsigned roles = get_my_roles(options);
   tt_uint_op(roles, OP_EQ,
-             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER);
+             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER|
+             PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     /* Only Client role should be disabled. */
     if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
@@ -144,17 +154,23 @@ test_pe_launch(void *arg)
   /* Disable everything and we'll enable them ALL. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
+  options->DisableNetwork = 1;
+  set_network_participation(false);
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
   /* Enable everything. */
   options->SocksPort_set = 1; options->ORPort_set = 1;
   options->BridgeRelay = 1; options->AuthoritativeDir = 1;
   options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1;
+  options->DisableNetwork = 0;
+  set_network_participation(true);
   register_dummy_hidden_service(&service);
   periodic_events_on_new_options(options);
   /* Note down the reference because we need to remove this service from the
@@ -163,9 +179,10 @@ test_pe_launch(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ,
+              (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV));
   }
 
  done:
@@ -187,42 +204,49 @@ test_pe_get_roles(void *arg)
 
   or_options_t *options = get_options_mutable();
   tt_assert(options);
+  set_network_participation(true);
+
+  const int ALL = PERIODIC_EVENT_ROLE_ALL |
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
 
   /* Nothing configured, should be no roles. */
+  tt_assert(net_is_disabled());
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Indicate we have a SocksPort, roles should be come Client. */
   options->SocksPort_set = 1;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT);
+  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL);
 
   /* Now, we'll add a ORPort so should now be a Relay + Client. */
   options->ORPort_set = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_DIRSERVER | ALL));
 
   /* Now add a Bridge. */
   options->BridgeRelay = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER);
   /* Unset client so we can solely test Router role. */
   options->SocksPort_set = 0;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER |
+            ALL);
 
   /* Reset options so we can test authorities. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
   options->BridgeRelay = 0;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Now upgrade to Dirauth. */
   options->DirPort_set = 1;
@@ -230,7 +254,7 @@ test_pe_get_roles(void *arg)
   options->V3AuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Now Bridge Authority. */
@@ -238,7 +262,7 @@ test_pe_get_roles(void *arg)
   options->BridgeAuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Move that bridge auth to become a relay. */
@@ -246,7 +270,7 @@ test_pe_get_roles(void *arg)
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY
-             | PERIODIC_EVENT_ROLE_DIRSERVER));
+             | PERIODIC_EVENT_ROLE_DIRSERVER|ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* And now an Hidden service. */
@@ -257,7 +281,8 @@ test_pe_get_roles(void *arg)
   remove_service(get_hs_service_map(), &service);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
  done:
@@ -277,12 +302,13 @@ test_pe_hs_service(void *arg)
   consider_hibernation(time(NULL));
   /* Initialize the events so we can enable them */
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
@@ -295,8 +321,8 @@ test_pe_hs_service(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
     }
@@ -306,8 +332,8 @@ test_pe_hs_service(void *arg)
   /* Remove the service from the global map, it should trigger a rescan and
    * disable the HS service events. */
   remove_service(get_hs_service_map(), &service);
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
     }
diff --git a/src/test/test_policy.c b/src/test/test_policy.c
index 9c001c294a..7949e90e9e 100644
--- a/src/test/test_policy.c
+++ b/src/test/test_policy.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
@@ -6,13 +6,18 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/circuitbuild.h"
 #include "core/or/policies.h"
 #include "feature/dirparse/policy_parse.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_descriptor.h"
 #include "feature/relay/router.h"
 #include "lib/encoding/confline.h"
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #include "core/or/addr_policy_st.h"
+#include "core/or/extend_info_st.h"
 #include "core/or/port_cfg_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -57,8 +62,8 @@ test_policy_summary_helper_family_flags(const char *policy_str,
   short_policy_t *short_policy = NULL;
   int success = 0;
 
-  line.key = (char*)"foo";
-  line.value = (char *)policy_str;
+  line.key = (char *) "foo";
+  line.value = (char *) policy_str;
   line.next = NULL;
 
   r = policies_parse_exit_policy(&line, &policy,
@@ -2024,6 +2029,101 @@ test_policies_fascist_firewall_allows_address(void *arg)
                            expect_ap); \
   STMT_END
 
+/* Check that fascist_firewall_choose_address_ls() returns the expected
+ * results. */
+#define CHECK_CHOSEN_ADDR_NULL_LS() \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(NULL, 1, &chosen_ls_ap); \
+    expect_single_log_msg("Unknown or missing link specifiers"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_CHOSEN_ADDR_LS(fake_ls, pref_only, expect_rv, expect_ap) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, pref_only, &chosen_ls_ap); \
+    if (smartlist_len(fake_ls) == 0) { \
+      expect_single_log_msg("Link specifiers are empty"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_ls_ap.addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, chosen_ls_ap.port); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_LS_LEGACY_ONLY(fake_ls) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, 0, &chosen_ls_ap); \
+    expect_single_log_msg("None of our link specifiers have IPv4 or IPv6"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS(fake_ls, direct_conn, expect_ap) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    if (fake_ls == NULL) { \
+      tt_ptr_op(ei, OP_EQ, NULL); \
+      expect_single_log_msg("Specified link specifiers is null"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &ei->addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, ei->port); \
+      extend_info_free(ei); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(fake_ls) \
+  STMT_BEGIN \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, NULL, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg("Specified onion key is null"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(fake_ls, direct_conn) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(fake_ls, msg_level, msg) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(msg_level); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg(msg); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
 /** Run unit tests for fascist_firewall_choose_address */
 static void
 test_policies_fascist_firewall_choose_address(void *arg)
@@ -2422,6 +2522,141 @@ test_policies_fascist_firewall_choose_address(void *arg)
   CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1,
                        ipv4_dir_ap);
 
+  /* Test firewall_choose_address_ls(). To do this, we make a fake link
+   * specifier. */
+  smartlist_t *lspecs = smartlist_new(),
+              *lspecs_blank = smartlist_new(),
+              *lspecs_v4 = smartlist_new(),
+              *lspecs_v6 = smartlist_new(),
+              *lspecs_no_legacy = smartlist_new(),
+              *lspecs_legacy_only = smartlist_new();
+  link_specifier_t *fake_ls;
+
+  /* IPv4 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(fake_ls,
+                                  tor_addr_to_ipv4h(&ipv4_or_ap.addr));
+  link_specifier_set_un_ipv4_port(fake_ls, ipv4_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, sizeof(ipv4_or_ap.addr.addr.in_addr) +
+                            sizeof(ipv4_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_no_legacy, fake_ls);
+
+  /* IPv6 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV6);
+  size_t addr_len = link_specifier_getlen_un_ipv6_addr(fake_ls);
+  const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ipv6_or_ap.addr);
+  uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(fake_ls);
+  memcpy(ipv6_array, in6_addr, addr_len);
+  link_specifier_set_un_ipv6_port(fake_ls, ipv6_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, addr_len + sizeof(ipv6_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Legacy ID link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_LEGACY_ID);
+  uint8_t *legacy_id = link_specifier_getarray_un_legacy_id(fake_ls);
+  memset(legacy_id, 'A', sizeof(*legacy_id));
+  link_specifier_set_ls_len(fake_ls,
+                            link_specifier_getlen_un_legacy_id(fake_ls));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_legacy_only, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Check with bogus requests. */
+  tor_addr_port_t null_ap; \
+  tor_addr_make_null(&null_ap.addr, AF_UNSPEC); \
+  null_ap.port = 0; \
+
+  /* Check for a null link state. */
+  CHECK_CHOSEN_ADDR_NULL_LS();
+  CHECK_HS_EXTEND_INFO_ADDR_LS(NULL, 1, null_ap);
+
+  /* Check for a blank link state. */
+  CHECK_CHOSEN_ADDR_LS(lspecs_blank, 0, 0, null_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_blank, 0);
+
+  /* Check for a link state with only a Legacy ID. */
+  CHECK_LS_LEGACY_ONLY(lspecs_legacy_only);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_legacy_only, 0);
+  smartlist_free(lspecs_legacy_only);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(lspecs_blank);
+  smartlist_free(lspecs_blank);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(lspecs_no_legacy, LOG_WARN,
+                                          "Missing Legacy ID in link state");
+  smartlist_free(lspecs_no_legacy);
+
+  /* Enable both IPv4 and IPv6. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Prefer IPv4, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* Prefer IPv6, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* IPv4-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v6, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 1);
+
+  /* IPv6-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 0;
+  mock_options.ClientUseIPv6 = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v4, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v4, 1);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+
+  smartlist_free(lspecs_v4);
+  smartlist_free(lspecs_v6);
+
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, lspec, \
+                    link_specifier_free(lspec)); \
+  smartlist_free(lspecs);
+
  done:
   UNMOCK(get_options);
 }
diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c
new file mode 100644
index 0000000000..541a81df3a
--- /dev/null
+++ b/src/test/test_prob_distr.c
@@ -0,0 +1,1406 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_prob_distr.c
+ * \brief Test probability distributions.
+ * \detail
+ *
+ * For each probability distribution we do two kinds of tests:
+ *
+ * a) We do numerical deterministic testing of their cdf/icdf/sf/isf functions
+ *    and the various relationships between them for each distribution. We also
+ *    do deterministic tests on their sampling functions. Test vectors for
+ *    these tests were computed from alternative implementations and were
+ *    eyeballed to make sure they make sense
+ *    (e.g. src/test/prob_distr_mpfr_ref.c computes logit(p) using GNU mpfr
+ *    with 200-bit precision and is then tested in test_logit_logistic()).
+ *
+ * b) We do stochastic hypothesis testing (G-test) to ensure that sampling from
+ *    the given distributions is distributed properly. The stochastic tests are
+ *    slow and their false positive rate is not well suited for CI, so they are
+ *    currently disabled-by-default and put into 'tests-slow'.
+ */
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "test/test.h"
+
+#include "core/or/or.h"
+
+#include "lib/math/prob_distr.h"
+#include "lib/math/fp.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "test/rng_test_helpers.h"
+
+#include <float.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/**
+ * Return floor(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)floor(d).
+ */
+static size_t
+floor_to_size_t(double d)
+{
+  double integral_d = floor(d);
+  return (size_t)integral_d;
+}
+
+/**
+ * Return ceil(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)ceil(d).
+ */
+static size_t
+ceil_to_size_t(double d)
+{
+  double integral_d = ceil(d);
+  return (size_t)integral_d;
+}
+
+/*
+ * Geometric(p) distribution, supported on {1, 2, 3, ...}.
+ *
+ * Compute the probability mass function Geom(n; p) of the number of
+ * trials before the first success when success has probability p.
+ */
+static double
+logpmf_geometric(unsigned n, double p)
+{
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1) {
+    if (n == 1)
+      return 0;
+    else
+      return -HUGE_VAL;
+  }
+  return (n - 1)*log1p(-p) + log(p);
+}
+
+/**
+ * Compute the logistic function, translated in output by 1/2:
+ * logistichalf(x) = logistic(x) - 1/2.  Well-conditioned on the entire
+ * real plane, with maximum condition number 1 at 0.
+ *
+ * This implementation gives relative error bounded by 5 eps.
+ */
+static double
+logistichalf(double x)
+{
+  /*
+   * Rewrite this with the identity
+   *
+   *  1/(1 + e^{-x}) - 1/2
+   *  = (1 - 1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1 - e^{-x})/[2 (1 + e^{-x})]
+   *  = -(e^{-x} - 1)/[2 (1 + e^{-x})],
+   *
+   * which we can evaluate by -expm1(-x)/[2 (1 + exp(-x))].
+   *
+   * Suppose exp has error d0, + has error d1, expm1 has error
+   * d2, and / has error d3, so we evaluate
+   *
+   *  -(1 + d2) (1 + d3) (e^{-x} - 1)
+   *    / [2 (1 + d1) (1 + (1 + d0) e^{-x})].
+   *
+   * In the denominator,
+   *
+   *  1 + (1 + d0) e^{-x}
+   *  = 1 + e^{-x} + d0 e^{-x}
+   *  = (1 + e^{-x}) (1 + d0 e^{-x}/(1 + e^{-x})),
+   *
+   * so the relative error of the numerator is
+   *
+   *  d' = d2 + d3 + d2 d3,
+   * and of the denominator,
+   *  d'' = d1 + d0 e^{-x}/(1 + e^{-x}) + d0 d1 e^{-x}/(1 + e^{-x})
+   *      = d1 + d0 L(-x) + d0 d1 L(-x),
+   *
+   * where L(-x) is logistic(-x).  By Lemma 1 the relative error
+   * of the quotient is bounded by
+   *
+   *  2|d2 + d3 + d2 d3 - d1 - d0 L(x) + d0 d1 L(x)|,
+   *
+   * Since 0 < L(x) < 1, this is bounded by
+   *
+   *  2|d2| + 2|d3| + 2|d2 d3| + 2|d1| + 2|d0| + 2|d0 d1|
+   *  <= 4 eps + 2 eps^2.
+   */
+  if (x < log(DBL_EPSILON/8)) {
+    /*
+     * Avoid overflow in e^{-x}.  When x < log(eps/4), we
+     * we further have x < logit(eps/4), so that
+     * logistic(x) < eps/4.  Hence the relative error of
+     * logistic(x) - 1/2 from -1/2 is bounded by eps/2, and
+     * so the relative error of -1/2 from logistic(x) - 1/2
+     * is bounded by eps.
+     */
+    return -0.5;
+  } else {
+    return -expm1(-x)/(2*(1 + exp(-x)));
+  }
+}
+
+/**
+ * Compute the log of the sum of the exps.  Caller should arrange the
+ * array in descending order to minimize error because I don't want to
+ * deal with using temporary space and the one caller in this file
+ * arranges that anyway.
+ *
+ * Warning: This implementation does not handle infinite or NaN inputs
+ * sensibly, because I don't need that here at the moment.  (NaN, or
+ * -inf and +inf together, should yield NaN; +inf and finite should
+ * yield +inf; otherwise all -inf should be ignored because exp(-inf) =
+ * 0.)
+ */
+static double
+logsumexp(double *A, size_t n)
+{
+  double maximum, sum;
+  size_t i;
+
+  if (n == 0)
+    return log(0);
+
+  maximum = A[0];
+  for (i = 1; i < n; i++) {
+    if (A[i] > maximum)
+      maximum = A[i];
+  }
+
+  sum = 0;
+  for (i = n; i --> 0;)
+    sum += exp(A[i] - maximum);
+
+  return log(sum) + maximum;
+}
+
+/**
+ * Compute log(1 - e^x).  Defined only for negative x so that e^x < 1.
+ * This is the complement of a probability in log space.
+ */
+static double
+log1mexp(double x)
+{
+
+  /*
+   * We want to compute log on [0, 1/2) but log1p on [1/2, +inf),
+   * so partition x at -log(2) = log(1/2).
+   */
+  if (-log(2) < x)
+    return log(-expm1(x));
+  else
+    return log1p(-exp(x));
+}
+
+/*
+ * Tests of numerical errors in computing logit, logistic, and the
+ * various cdfs, sfs, icdfs, and isfs.
+ */
+
+#define arraycount(A) (sizeof(A)/sizeof(A[0]))
+
+/** Return relative error between <b>actual</b> and <b>expected</b>.
+ *  Special cases: If <b>expected</b> is zero or infinite, return 1 if
+ *  <b>actual</b> is equal to <b>expected</b> and 0 if not, since the
+ *  usual notion of relative error is undefined but we only use this
+ *  for testing relerr(e, a) <= bound.  If either is NaN, return NaN,
+ *  which has the property that NaN <= bound is false no matter what
+ *  bound is.
+ *
+ *  Beware: if you test !(relerr(e, a) > bound), then then the result
+ *  is true when a is NaN because NaN > bound is false too.  See
+ *  CHECK_RELERR for correct use to decide when to report failure.
+ */
+static double
+relerr(double expected, double actual)
+{
+  /*
+   * To silence -Wfloat-equal, we have to test for equality using
+   * inequalities: we have (fabs(expected) <= 0) iff (expected == 0),
+   * and (actual <= expected && actual >= expected) iff actual ==
+   * expected whether expected is zero or infinite.
+   */
+  if (fabs(expected) <= 0 || tor_isinf(expected)) {
+    if (actual <= expected && actual >= expected)
+      return 0;
+    else
+      return 1;
+  } else {
+    return fabs((expected - actual)/expected);
+  }
+}
+
+/** Check that relative error of <b>expected</b> and <b>actual</b> is within
+ *  <b>relerr_bound</b>.  Caller must arrange to have i and relerr_bound in
+ *  scope.  */
+#define CHECK_RELERR(expected, actual) do {                                   \
+  double check_expected = (expected);                                         \
+  double check_actual = (actual);                                             \
+  const char *str_expected = #expected;                                       \
+  const char *str_actual = #actual;                                           \
+  double check_relerr = relerr(expected, actual);                             \
+  if (!(relerr(check_expected, check_actual) <= relerr_bound)) {              \
+    log_warn(LD_GENERAL, "%s:%d: case %u: relerr(%s=%.17e, %s=%.17e)"        \
+             " = %.17e > %.17e\n",                                            \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_expected, check_expected,                                    \
+             str_actual, check_actual,                                        \
+             check_relerr, relerr_bound);                                     \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/* Check that a <= b.
+ * Caller must arrange to have i in scope.  */
+#define CHECK_LE(a, b) do {                                                   \
+  double check_a = (a);                                                       \
+  double check_b = (b);                                                       \
+  const char *str_a = #a;                                                     \
+  const char *str_b = #b;                                                     \
+  if (!(check_a <= check_b)) {                                                \
+    log_warn(LD_GENERAL, "%s:%d: case %u: %s=%.17e > %s=%.17e\n",             \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_a, check_a, str_b, check_b);                                 \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/**
+ * Test the logit and logistic functions.  Confirm that they agree with
+ * the cdf, sf, icdf, and isf of the standard Logistic distribution.
+ * Confirm that the sampler for the standard logistic distribution maps
+ * [0, 1] into the right subinterval for the inverse transform, for
+ * this implementation.
+ */
+static void
+test_logit_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    double x;                   /* x = logit(p) */
+    double p;                   /* p = logistic(x) */
+    double phalf;               /* p - 1/2 = logistic(x) - 1/2 */
+  } cases[] = {
+    { -HUGE_VAL, 0, -0.5 },
+    { -1000, 0, -0.5 },
+    { -710, 4.47628622567513e-309, -0.5 },
+    { -708, 3.307553003638408e-308, -0.5 },
+    { -2, .11920292202211755, -.3807970779778824 },
+    { -1.0000001, .2689414017088022, -.23105859829119776 },
+    { -1, .2689414213699951, -.23105857863000487 },
+    { -0.9999999, .26894144103118883, -.2310585589688111 },
+    /* see src/test/prob_distr_mpfr_ref.c for computation */
+    { -4.000000000537333e-5, .49999, -1.0000000000010001e-5 },
+    { -4.000000000533334e-5, .49999, -.00001 },
+    { -4.000000108916878e-9, .499999999, -1.0000000272292198e-9 },
+    { -4e-9, .499999999, -1e-9 },
+    { -4e-16, .5, -1e-16 },
+    { -4e-300, .5, -1e-300 },
+    { 0, .5, 0 },
+    { 4e-300, .5, 1e-300 },
+    { 4e-16, .5, 1e-16 },
+    { 3.999999886872274e-9, .500000001, 9.999999717180685e-10 },
+    { 4e-9, .500000001, 1e-9 },
+    { 4.0000000005333336e-5, .50001, .00001 },
+    { 8.000042667076272e-3, .502, .002 },
+    { 0.9999999, .7310585589688111, .2310585589688111 },
+    { 1, .7310585786300049, .23105857863000487 },
+    { 1.0000001, .7310585982911977, .23105859829119774 },
+    { 2, .8807970779778823, .3807970779778824 },
+    { 708, 1, .5 },
+    { 710, 1, .5 },
+    { 1000, 1, .5 },
+    { HUGE_VAL, 1, .5 },
+  };
+  double relerr_bound = 3e-15; /* >10eps */
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double phalf = cases[i].phalf;
+
+    /*
+     * cdf is logistic, icdf is logit, and symmetry for
+     * sf/isf.
+     */
+    CHECK_RELERR(logistic(x), cdf_logistic(x, 0, 1));
+    CHECK_RELERR(logistic(-x), sf_logistic(x, 0, 1));
+    CHECK_RELERR(logit(p), icdf_logistic(p, 0, 1));
+    CHECK_RELERR(-logit(p), isf_logistic(p, 0, 1));
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2, 0, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2, 0, 2));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0, 2)/2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, 2)/2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x/2, 0, .5));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x/2, 0, .5));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0,.5)*2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, .5)*2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2 + 1, 1, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2 + 1, 1, 2));
+
+    /*
+     * For p near 0 and p near 1/2, the arithmetic of
+     * translating by 1 loses precision.
+     */
+    if (fabs(p) > DBL_EPSILON && fabs(p) < 0.4) {
+      CHECK_RELERR(icdf_logistic(p, 0, 1),
+          (icdf_logistic(p, 1, 2) - 1)/2);
+      CHECK_RELERR(isf_logistic(p, 0, 1),
+          (isf_logistic(p, 1, 2) - 1)/2);
+    }
+
+    CHECK_RELERR(p, logistic(x));
+    CHECK_RELERR(phalf, logistichalf(x));
+
+    /*
+     * On the interior floating-point numbers, either logit or
+     * logithalf had better give the correct answer.
+     *
+     * For probabilities near 0, we can get much finer resolution with
+     * logit, and for probabilities near 1/2, we can get much finer
+     * resolution with logithalf by representing them using p - 1/2.
+     *
+     * E.g., we can write -.00001 for phalf, and .49999 for p, but the
+     * difference 1/2 - .00001 gives 1.0000000000010001e-5 in binary64
+     * arithmetic.  So test logit(.49999) which should give the same
+     * answer as logithalf(-1.0000000000010001e-5), namely
+     * -4.000000000537333e-5, and also test logithalf(-.00001) which
+     * gives -4.000000000533334e-5 instead -- but don't expect
+     * logit(.49999) to give -4.000000000533334e-5 even though it looks
+     * like 1/2 - .00001.
+     *
+     * A naive implementation of logit will just use log(p/(1 - p)) and
+     * give the answer -4.000000000551673e-05 for .49999, which is
+     * wrong in a lot of digits, which happens because log is
+     * ill-conditioned near 1 and thus amplifies whatever relative
+     * error we made in computing p/(1 - p).
+     */
+    if ((0 < p && p < 1) || tor_isinf(x)) {
+      if (phalf >= p - 0.5 && phalf <= p - 0.5)
+        CHECK_RELERR(x, logit(p));
+      if (p >= 0.5 + phalf && p <= 0.5 + phalf)
+        CHECK_RELERR(x, logithalf(phalf));
+    }
+
+    CHECK_RELERR(-phalf, logistichalf(-x));
+    if (fabs(phalf) < 0.5 || tor_isinf(x))
+      CHECK_RELERR(-x, logithalf(-phalf));
+    if (p < 1 || tor_isinf(x)) {
+      CHECK_RELERR(1 - p, logistic(-x));
+      if (p > .75 || tor_isinf(x))
+        CHECK_RELERR(-x, logit(1 - p));
+    } else {
+      CHECK_LE(logistic(-x), 1e-300);
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(logit(p0/(1 + M_E)), sample_logistic(0, 0, p0));
+    CHECK_RELERR(-logit(p0/(1 + M_E)), sample_logistic(1, 0, p0));
+    CHECK_RELERR(logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(0, 1, p0));
+    CHECK_RELERR(-logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(1, 1, p0));
+  }
+
+  if (!ok)
+    printf("fail logit/logistic / logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the LogLogistic distribution.
+ */
+static void
+test_log_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the LogLogistic distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given LogLogistic
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given LogLogistic
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { 1e-15, 1e-15, .999999999999999 },
+    { .1, .09090909090909091, .90909090909090909 },
+    { .25, .2, .8 },
+    { .5, .33333333333333333, .66666666666666667 },
+    { .75, .42857142857142855, .5714285714285714 },
+    { .9999, .49997499874993756, .5000250012500626 },
+    { .99999999, .49999999749999996, .5000000025 },
+    { .999999999999999, .49999999999999994, .5000000000000002 },
+    { 1, .5, .5 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_log_logistic(x, 1, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(p, cdf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, sf_log_logistic(x, 1, 1));
+    CHECK_RELERR(np, sf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(np, sf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(np, sf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, cdf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(np, cdf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(np, cdf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(np, cdf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(np, cdf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(np, cdf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(p, sf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(p, sf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(p, sf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(p, sf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(p, sf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(p, sf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(x/2, icdf_log_logistic(p, .5, 1));
+    CHECK_RELERR(x*2, icdf_log_logistic(p, 2, 1));
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(sqrt(x)/2, icdf_log_logistic(p, .5, 2));
+    CHECK_RELERR(sqrt(x)*2, icdf_log_logistic(p, 2, 2));
+    CHECK_RELERR(sqrt(x), icdf_log_logistic(p, 1, 2));
+    CHECK_RELERR(x*x/2, icdf_log_logistic(p, .5, .5));
+    CHECK_RELERR(x*x*2, icdf_log_logistic(p, 2, .5));
+
+    if (np < .9) {
+      CHECK_RELERR(x, isf_log_logistic(np, 1, 1));
+      CHECK_RELERR(x/2, isf_log_logistic(np, .5, 1));
+      CHECK_RELERR(x*2, isf_log_logistic(np, 2, 1));
+      CHECK_RELERR(sqrt(x), isf_log_logistic(np, 1, 2));
+      CHECK_RELERR(sqrt(x)/2, isf_log_logistic(np, .5, 2));
+      CHECK_RELERR(sqrt(x)*2, isf_log_logistic(np, 2, 2));
+      CHECK_RELERR(x*x, isf_log_logistic(np, 1, .5));
+      CHECK_RELERR(x*x/2, isf_log_logistic(np, .5, .5));
+      CHECK_RELERR(x*x*2, isf_log_logistic(np, 2, .5));
+
+      CHECK_RELERR(1/x, icdf_log_logistic(np, 1, 1));
+      CHECK_RELERR(1/(2*x), icdf_log_logistic(np, .5, 1));
+      CHECK_RELERR(2/x, icdf_log_logistic(np, 2, 1));
+      CHECK_RELERR(1/sqrt(x), icdf_log_logistic(np, 1, 2));
+      CHECK_RELERR(1/(2*sqrt(x)),
+          icdf_log_logistic(np, .5, 2));
+      CHECK_RELERR(2/sqrt(x), icdf_log_logistic(np, 2, 2));
+      CHECK_RELERR(1/(x*x), icdf_log_logistic(np, 1, .5));
+      CHECK_RELERR(1/(2*x*x), icdf_log_logistic(np, .5, .5));
+      CHECK_RELERR(2/(x*x), icdf_log_logistic(np, 2, .5));
+    }
+
+    CHECK_RELERR(1/x, isf_log_logistic(p, 1, 1));
+    CHECK_RELERR(1/(2*x), isf_log_logistic(p, .5, 1));
+    CHECK_RELERR(2/x, isf_log_logistic(p, 2, 1));
+    CHECK_RELERR(1/sqrt(x), isf_log_logistic(p, 1, 2));
+    CHECK_RELERR(1/(2*sqrt(x)), isf_log_logistic(p, .5, 2));
+    CHECK_RELERR(2/sqrt(x), isf_log_logistic(p, 2, 2));
+    CHECK_RELERR(1/(x*x), isf_log_logistic(p, 1, .5));
+    CHECK_RELERR(1/(2*x*x), isf_log_logistic(p, .5, .5));
+    CHECK_RELERR(2/(x*x), isf_log_logistic(p, 2, .5));
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(0.5*p0/(1 - 0.5*p0), sample_log_logistic(0, p0));
+    CHECK_RELERR((1 - 0.5*p0)/(0.5*p0),
+        sample_log_logistic(1, p0));
+  }
+
+  if (!ok)
+    printf("fail log logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, isf of the Weibull distribution.
+ */
+static void
+test_weibull(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the Weibull distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given Weibull
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given Weibull
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { .1, .09516258196404043, .9048374180359595 },
+    { .5, .3934693402873666, .6065306597126334 },
+    { .6931471805599453, .5, .5 },
+    { 1, .6321205588285577, .36787944117144233 },
+    { 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 36, .9999999999999998, 2.319522830243569e-16 },
+    { 37, .9999999999999999, 8.533047625744066e-17 },
+    { 38, 1, 3.1391327920480296e-17 },
+    { 100, 1, 3.720075976020836e-44 },
+    { 708, 1, 3.307553003638408e-308 },
+    { 710, 1, 4.47628622567513e-309 },
+    { 1000, 1, 0 },
+    { HUGE_VAL, 1, 0 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_weibull(x, 1, 1));
+    CHECK_RELERR(p, cdf_weibull(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_weibull(x*2, 2, 1));
+    /* For 0 < x < sqrt(DBL_MIN), x^2 loses lots of bits.  */
+    if (x <= 0 ||
+        sqrt(DBL_MIN) <= x) {
+      CHECK_RELERR(p, cdf_weibull(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x*2, 2, .5));
+    }
+    CHECK_RELERR(p, cdf_weibull(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)*2, 2, 2));
+    CHECK_RELERR(np, sf_weibull(x, 1, 1));
+    CHECK_RELERR(np, sf_weibull(x/2, .5, 1));
+    CHECK_RELERR(np, sf_weibull(x*2, 2, 1));
+    CHECK_RELERR(np, sf_weibull(x*x, 1, .5));
+    CHECK_RELERR(np, sf_weibull(x*x/2, .5, .5));
+    CHECK_RELERR(np, sf_weibull(x*x*2, 2, .5));
+    if (x >= 10) {
+      /*
+       * exp amplifies the error of sqrt(x)^2
+       * proportionally to exp(x); for large inputs
+       * this is significant.
+       */
+      double t = -expm1(-x*(2*DBL_EPSILON + DBL_EPSILON));
+      relerr_bound = t + DBL_EPSILON + t*DBL_EPSILON;
+      if (relerr_bound < 3e-15)
+        /*
+         * The tests are written only to 16
+         * decimal places anyway even if your
+         * `double' is, say, i387 binary80, for
+         * whatever reason.
+         */
+        relerr_bound = 3e-15;
+      CHECK_RELERR(np, sf_weibull(sqrt(x), 1, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)/2, .5, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)*2, 2, 2));
+    }
+
+    if (p <= 0.75) {
+      /*
+       * For p near 1, not enough precision near 1 to
+       * recover x.
+       */
+      CHECK_RELERR(x, icdf_weibull(p, 1, 1));
+      CHECK_RELERR(x/2, icdf_weibull(p, .5, 1));
+      CHECK_RELERR(x*2, icdf_weibull(p, 2, 1));
+    }
+    if (p >= 0.25 && !tor_isinf(x) && np > 0) {
+      /*
+       * For p near 0, not enough precision in np
+       * near 1 to recover x.  For 0, isf gives inf,
+       * even if p is precise enough for the icdf to
+       * work.
+       */
+      CHECK_RELERR(x, isf_weibull(np, 1, 1));
+      CHECK_RELERR(x/2, isf_weibull(np, .5, 1));
+      CHECK_RELERR(x*2, isf_weibull(np, 2, 1));
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(3*sqrt(-log(p0/2)), sample_weibull(0, p0, 3, 2));
+    CHECK_RELERR(3*sqrt(-log1p(-p0/2)),
+        sample_weibull(1, p0, 3, 2));
+  }
+
+  if (!ok)
+    printf("fail Weibull cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the generalized Pareto
+ * distribution.
+ */
+static void
+test_genpareto(void *arg)
+{
+  (void) arg;
+
+  struct {
+    /* xi is the 'xi' parameter of the generalized Pareto distribution, and the
+     * rest are the same as in the above tests */
+    double xi, x, p, np;
+  } cases[] = {
+    { 0, 0, 0, 1 },
+    { 1e-300, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-300, .1, .09516258196404043, .9048374180359595 },
+    { 1e-300, 1, .6321205588285577, .36787944117144233 },
+    { 1e-300, 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 1e-200, 1e-16, 9.999999999999999e-17, .9999999999999999 },
+    { 1e-16, 1e-200, 9.999999999999998e-201, 1 },
+    { 1e-16, 1e-16, 1e-16, 1 },
+    { 1e-16, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-16, .1, .09516258196404043, .9048374180359595 },
+    { 1e-16, 1, .6321205588285577, .36787944117144233 },
+    { 1e-16, 10, .9999546000702375, 4.539992976248509e-5 },
+    { 1e-10, 1e-6, 9.999995000001667e-7, .9999990000005 },
+    { 1e-8, 1e-8, 9.999999950000001e-9, .9999999900000001 },
+    { 1, 1e-300, 1e-300, 1 },
+    { 1, 1e-16, 1e-16, .9999999999999999 },
+    { 1, .1, .09090909090909091, .9090909090909091 },
+    { 1, 1, .5, .5 },
+    { 1, 10, .9090909090909091, .0909090909090909 },
+    { 1, 100, .9900990099009901, .0099009900990099 },
+    { 1, 1000, .999000999000999, 9.990009990009992e-4 },
+    { 10, 1e-300, 1e-300, 1 },
+    { 10, 1e-16, 9.999999999999995e-17, .9999999999999999 },
+    { 10, .1, .06696700846319258, .9330329915368074 },
+    { 10, 1, .21320655780322778, .7867934421967723 },
+    { 10, 10, .3696701667040189, .6303298332959811 },
+    { 10, 100, .49886285755007337, .5011371424499267 },
+    { 10, 1000, .6018968102992647, .3981031897007353 },
+  };
+  double xi_array[] = { -1.5, -1, -1e-30, 0, 1e-30, 1, 1.5 };
+  size_t i, j;
+  double relerr_bound = 3e-15;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double xi = cases[i].xi;
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(p, cdf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(p, cdf_genpareto(x/2, 0, .5, xi));
+    CHECK_RELERR(np, sf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(np, sf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(np, sf_genpareto(x/2, 0, .5, xi));
+
+    if (p < .5) {
+      CHECK_RELERR(x, icdf_genpareto(p, 0, 1, xi));
+      CHECK_RELERR(x*2, icdf_genpareto(p, 0, 2, xi));
+      CHECK_RELERR(x/2, icdf_genpareto(p, 0, .5, xi));
+    }
+    if (np < .5) {
+      CHECK_RELERR(x, isf_genpareto(np, 0, 1, xi));
+      CHECK_RELERR(x*2, isf_genpareto(np, 0, 2, xi));
+      CHECK_RELERR(x/2, isf_genpareto(np, 0, .5, xi));
+    }
+  }
+
+  for (i = 0; i < arraycount(xi_array); i++) {
+    for (j = 0; j <= 100; j++) {
+      double p0 = (j == 0 ? 2*DBL_MIN : (double)j/100);
+
+      /* This is actually a check against 0, but we do <= so that the compiler
+         does not raise a -Wfloat-equal */
+      if (fabs(xi_array[i]) <= 0) {
+        /*
+         * When xi == 0, the generalized Pareto
+         * distribution reduces to an
+         * exponential distribution.
+         */
+        CHECK_RELERR(-log(p0/2),
+            sample_genpareto(0, p0, 0));
+        CHECK_RELERR(-log1p(-p0/2),
+            sample_genpareto(1, p0, 0));
+      } else {
+        CHECK_RELERR(expm1(-xi_array[i]*log(p0/2))/xi_array[i],
+            sample_genpareto(0, p0, xi_array[i]));
+        CHECK_RELERR((j == 0 ? DBL_MIN :
+                expm1(-xi_array[i]*log1p(-p0/2))/xi_array[i]),
+            sample_genpareto(1, p0, xi_array[i]));
+      }
+
+      CHECK_RELERR(isf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(0, p0, xi_array[i]));
+      CHECK_RELERR(icdf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(1, p0, xi_array[i]));
+    }
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the deterministic sampler for uniform distribution on [a, b].
+ *
+ * This currently only tests whether the outcome lies within [a, b].
+ */
+static void
+test_uniform_interval(void *arg)
+{
+  (void) arg;
+  struct {
+    /* Sample from a uniform distribution with parameters 'a' and 'b', using
+     * 't' as the sampling index. */
+    double t, a, b;
+  } cases[] = {
+    { 0, 0, 0 },
+    { 0, 0, 1 },
+    { 0, 1.0000000000000007, 3.999999999999995 },
+    { 0, 4000, 4000 },
+    { 0.42475836677491291, 4000, 4000 },
+    { 0, -DBL_MAX, DBL_MAX },
+    { 0.25, -DBL_MAX, DBL_MAX },
+    { 0.5, -DBL_MAX, DBL_MAX },
+  };
+  size_t i = 0;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double t = cases[i].t;
+    double a = cases[i].a;
+    double b = cases[i].b;
+
+    CHECK_LE(a, sample_uniform_interval(t, a, b));
+    CHECK_LE(sample_uniform_interval(t, a, b), b);
+
+    CHECK_LE(a, sample_uniform_interval(1 - t, a, b));
+    CHECK_LE(sample_uniform_interval(1 - t, a, b), b);
+
+    CHECK_LE(sample_uniform_interval(t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(t, -b, -a));
+
+    CHECK_LE(sample_uniform_interval(1 - t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(1 - t, -b, -a));
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/********************** Stochastic tests ****************************/
+
+/*
+ * Psi test, sometimes also called G-test.  The psi test statistic,
+ * suitably scaled, has chi^2 distribution, but the psi test tends to
+ * have better statistical power in practice to detect deviations than
+ * the chi^2 test does.  (The chi^2 test statistic is the first term of
+ * the Taylor expansion of the psi test statistic.)  The psi test is
+ * generic, for any CDF; particular distributions might have higher-
+ * power tests to distinguish them from predictable deviations or bugs.
+ *
+ * We choose the psi critical value so that a single psi test has
+ * probability below alpha = 1% of spuriously failing even if all the
+ * code is correct.  But the false positive rate for a suite of n tests
+ * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n.  For n = 10,
+ * this is about 10%, and for n = 100 it is well over 50%.
+ *
+ * Given that these tests will run with every CI job, we want to drive down the
+ * false positive rate. We can drive it down by running each test four times,
+ * and accepting it if it passes at least once; in that case, it is as if we
+ * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test,
+ * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14
+ * jobs, then the chance of a CI build failing is 1.39999903326e-06, which
+ * means that a CI build will break with probability 50% after about 495106
+ * builds.
+ *
+ * The critical value for a chi^2 distribution with 100 degrees of
+ * freedom and false positive rate alpha = 1% was taken from:
+ *
+ *  NIST/SEMATECH e-Handbook of Statistical Methods, Section
+ *  1.3.6.7.4 `Critical Values of the Chi-Square Distribution',
+ *  <https://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm>,
+ *  retrieved 2018-10-28.
+ */
+
+static const size_t NSAMPLES = 100000;
+/* Number of chances we give to the test to succeed. */
+static const unsigned NTRIALS = 4;
+/* Number of times we want the test to pass per NTRIALS. */
+static const unsigned NPASSES_MIN = 1;
+
+#define PSI_DF 100                          /* degrees of freedom */
+static const double PSI_CRITICAL = 135.807; /* critical value, alpha = .01 */
+
+/**
+ * Perform a psi test on an array of sample counts, C, adding up to N
+ * samples, and an array of log expected probabilities, logP,
+ * representing the null hypothesis for the distribution of samples
+ * counted.  Return false if the psi test rejects the null hypothesis,
+ * true if otherwise.
+ */
+static bool
+psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N)
+{
+  double psi = 0;
+  double c = 0;                 /* Kahan compensation */
+  double t, u;
+  size_t i;
+
+  for (i = 0; i < PSI_DF; i++) {
+    /*
+     * c*log(c/(n*p)) = (1/n) * f*log(f/p) where f = c/n is
+     * the frequency, and f*log(f/p) ---> 0 as f ---> 0, so
+     * this is a reasonable choice.  Further, any mass that
+     * _fails_ to turn up in this bin will inflate another
+     * bin instead, so we don't really lose anything by
+     * ignoring empty bins even if they have high
+     * probability.
+     */
+    if (C[i] == 0)
+      continue;
+    t = C[i]*(log((double)C[i]/N) - logP[i]) - c;
+    u = psi + t;
+    c = (u - psi) - t;
+    psi = u;
+  }
+  psi *= 2;
+
+  return psi <= PSI_CRITICAL;
+}
+
+static bool
+test_stochastic_geometric_impl(double p)
+{
+  const struct geometric_t geometric = {
+    .base = GEOMETRIC(geometric),
+    .p = p,
+  };
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  unsigned i;
+  size_t j;
+
+  /* Compute logP[i] = Geom(i + 1; p).  */
+  for (i = 0; i < PSI_DF - 1; i++)
+    logP[i] = logpmf_geometric(i + 1, p);
+
+  /* Compute logP[n-1] = log (1 - (P[0] + P[1] + ... + P[n-2])).  */
+  logP[PSI_DF - 1] = log1mexp(logsumexp(logP, PSI_DF - 1));
+
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+
+    for (j = 0; j < NSAMPLES; j++) {
+      double n_tmp = dist_sample(&geometric.base);
+
+      /* Must be an integer.  (XXX -Wfloat-equal)  */
+      tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp);
+
+      /* Must be a positive integer.  */
+      tor_assert(n_tmp >= 1);
+
+      /* Probability of getting a value in the billions is negligible.  */
+      tor_assert(n_tmp <= (double)UINT_MAX);
+
+      unsigned n = (unsigned) n_tmp;
+
+      if (n > PSI_DF)
+        n = PSI_DF;
+      C[n - 1]++;
+    }
+
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", "geometric"); */
+    return true;
+  } else {
+    printf("fail %s sampler\n", "geometric");
+    return false;
+  }
+}
+
+/**
+ * Divide the support of <b>dist</b> into histogram bins in <b>logP</b>. Start
+ * at the 1st percentile and ending at the 99th percentile. Pick the bin
+ * boundaries using linear interpolation so that they are uniformly spaced.
+ *
+ * In each bin logP[i] we insert the expected log-probability that a sampled
+ * value will fall into that bin. We will use this as the null hypothesis of
+ * the psi test.
+ *
+ * Set logP[i] = log(CDF(x_i) - CDF(x_{i-1})), where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_cdfs(const struct dist_t *dist, double lo, double hi, double *logP,
+         size_t n)
+{
+#define CDF(x)  dist_cdf(dist, x)
+#define SF(x)   dist_sf(dist, x)
+  const double w = (hi - lo)/(n - 2);
+  double halfway = dist_icdf(dist, 0.5);
+  double x_0, x_1;
+  size_t i;
+  size_t n2 = ceil_to_size_t((halfway - lo)/w);
+
+  tor_assert(lo <= halfway);
+  tor_assert(halfway <= hi);
+  tor_assert(n2 <= n);
+
+  x_1 = lo;
+  logP[0] = log(CDF(x_1) - 0); /* 0 = CDF(-inf) */
+  for (i = 1; i < n2; i++) {
+    x_0 = x_1;
+    /* do the linear interpolation */
+    x_1 = (i <= n/2 ? lo + i*w : hi - (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[i] = log(CDF(x_1) - CDF(x_0));
+  }
+  x_0 = hi;
+  logP[n - 1] = log(SF(x_0) - 0); /* 0 = SF(+inf) = 1 - CDF(+inf) */
+
+  /* In this loop we are filling out the high part of the array. We are using
+   * SF because in these cases the CDF is near 1 where precision is lower. So
+   * instead we are using SF near 0 where the precision is higher. We have
+   * SF(t) = 1 - CDF(t).  */
+  for (i = 1; i < n - n2; i++) {
+    x_1 = x_0;
+    /* do the linear interpolation */
+    x_0 = (i <= n/2 ? hi - i*w : lo + (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[n - i - 1] = log(SF(x_0) - SF(x_1));
+  }
+#undef SF
+#undef CDF
+}
+
+/**
+ * Draw NSAMPLES samples from dist, counting the number of samples x in
+ * the ith bin C[i] if x_{i-1} <= x < x_i, where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_samples(const struct dist_t *dist, double lo, double hi, size_t *C,
+            size_t n)
+{
+  const double w = (hi - lo)/(n - 2);
+  size_t i;
+
+  for (i = 0; i < NSAMPLES; i++) {
+    double x = dist_sample(dist);
+    size_t bin;
+
+    if (x < lo)
+      bin = 0;
+    else if (x < hi)
+      bin = 1 + floor_to_size_t((x - lo)/w);
+    else
+      bin = n - 1;
+    tor_assert(bin < n);
+    C[bin]++;
+  }
+}
+
+/**
+ * Carry out a Psi test on <b>dist</b>.
+ *
+ * Sample NSAMPLES from dist, putting them in bins from -inf to lo to
+ * hi to +inf, and apply up to two psi tests.  True if at least one psi
+ * test passes; false if not.  False positive rate should be bounded by
+ * 0.01^2 = 0.0001.
+ */
+static bool
+test_psi_dist_sample(const struct dist_t *dist)
+{
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  double lo = dist_icdf(dist, 1/(double)(PSI_DF + 2));
+  double hi = dist_isf(dist, 1/(double)(PSI_DF + 2));
+
+  /* Create the null hypothesis in logP */
+  bin_cdfs(dist, lo, hi, logP, PSI_DF);
+
+  /* Now run the test */
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+    bin_samples(dist, lo, hi, C, PSI_DF);
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  /* Did we fail or succeed? */
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", dist_name(dist));*/
+    return true;
+  } else {
+    printf("fail %s sampler\n", dist_name(dist));
+    return false;
+  }
+}
+
+static void
+write_stochastic_warning(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    printf("\n"
+         "NOTE: This is a stochastic test, and we expect it to fail from\n"
+         "time to time, with some low probability. If you see it fail more\n"
+         "than one trial in 100, though, please tell us.\n\n");
+  }
+}
+
+static void
+test_stochastic_uniform(void *arg)
+{
+  (void) arg;
+
+  const struct uniform_t uniform01 = {
+    .base = UNIFORM(uniform01),
+    .a = 0,
+    .b = 1,
+  };
+  const struct uniform_t uniform_pos = {
+    .base = UNIFORM(uniform_pos),
+    .a = 1.23,
+    .b = 4.56,
+  };
+  const struct uniform_t uniform_neg = {
+    .base = UNIFORM(uniform_neg),
+    .a = -10,
+    .b = -1,
+  };
+  const struct uniform_t uniform_cross = {
+    .base = UNIFORM(uniform_cross),
+    .a = -1.23,
+    .b = 4.56,
+  };
+  const struct uniform_t uniform_subnormal = {
+    .base = UNIFORM(uniform_subnormal),
+    .a = 4e-324,
+    .b = 4e-310,
+  };
+  const struct uniform_t uniform_subnormal_cross = {
+    .base = UNIFORM(uniform_subnormal_cross),
+    .a = -4e-324,
+    .b = 4e-310,
+  };
+  bool ok = true, tests_failed = true;
+
+  testing_enable_reproducible_rng();
+
+  ok &= test_psi_dist_sample(&uniform01.base);
+  ok &= test_psi_dist_sample(&uniform_pos.base);
+  ok &= test_psi_dist_sample(&uniform_neg.base);
+  ok &= test_psi_dist_sample(&uniform_cross.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal_cross.base);
+
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static bool
+test_stochastic_logistic_impl(double mu, double sigma)
+{
+  const struct logistic_t dist = {
+    .base = LOGISTIC(dist),
+    .mu = mu,
+    .sigma = sigma,
+  };
+
+  /* XXX Consider some fancier logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_log_logistic_impl(double alpha, double beta)
+{
+  const struct log_logistic_t dist = {
+    .base = LOG_LOGISTIC(dist),
+    .alpha = alpha,
+    .beta = beta,
+  };
+
+  /* XXX Consider some fancier log logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_weibull_impl(double lambda, double k)
+{
+  const struct weibull_t dist = {
+    .base = WEIBULL(dist),
+    .lambda = lambda,
+    .k = k,
+  };
+
+// clang-format off
+/*
+ * XXX Consider applying a Tiku-Singh test:
+ *
+ *    M.L. Tiku and M. Singh, `Testing the two-parameter
+ *    Weibull distribution', Communications in Statistics --
+ *    Theory and Methods A10(9), 1981, 907--918.
+https://www.tandfonline.com/doi/pdf/10.1080/03610928108828082?needAccess=true
+ */
+// clang-format on
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_genpareto_impl(double mu, double sigma, double xi)
+{
+  const struct genpareto_t dist = {
+    .base = GENPARETO(dist),
+    .mu = mu,
+    .sigma = sigma,
+    .xi = xi,
+  };
+
+  /* XXX Consider some fancier GPD test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static void
+test_stochastic_genpareto(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_genpareto_impl(0, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, -1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(-1, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(1, 2, 0.25);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_geometric(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_geometric_impl(0.1);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.5);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.9);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(1);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_logistic(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_logistic_impl(0, 1);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(0, 1e-16);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(-10, 100);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_log_logistic(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_log_logistic_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(M_E, 1e-1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_weibull(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_weibull_impl(1, 0.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 2);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(10, 1);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+  UNMOCK(crypto_rand);
+}
+
+struct testcase_t prob_distr_tests[] = {
+  { "logit_logistics", test_logit_logistic, TT_FORK, NULL, NULL },
+  { "log_logistic", test_log_logistic, TT_FORK, NULL, NULL },
+  { "weibull", test_weibull, TT_FORK, NULL, NULL },
+  { "genpareto", test_genpareto, TT_FORK, NULL, NULL },
+  { "uniform_interval", test_uniform_interval, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
+
+struct testcase_t slow_stochastic_prob_distr_tests[] = {
+  { "stochastic_genpareto", test_stochastic_genpareto, TT_FORK, NULL, NULL },
+  { "stochastic_geometric", test_stochastic_geometric, TT_FORK, NULL, NULL },
+  { "stochastic_uniform", test_stochastic_uniform, TT_FORK, NULL, NULL },
+  { "stochastic_logistic", test_stochastic_logistic, TT_FORK, NULL, NULL },
+  { "stochastic_log_logistic", test_stochastic_log_logistic, TT_FORK, NULL,
+    NULL },
+  { "stochastic_weibull", test_stochastic_weibull, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process.c b/src/test/test_process.c
new file mode 100644
index 0000000000..c1da6db278
--- /dev/null
+++ b/src/test/test_process.c
@@ -0,0 +1,669 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process.c
+ * \brief Test cases for the Process API.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/process/env.h"
+
+#define PROCESS_PRIVATE
+#include "lib/process/process.h"
+#define PROCESS_UNIX_PRIVATE
+#include "lib/process/process_unix.h"
+#define PROCESS_WIN32_PRIVATE
+#include "lib/process/process_win32.h"
+
+static const char *stdout_read_buffer;
+static const char *stderr_read_buffer;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static int
+process_mocked_read_stdout(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stdout_read_buffer != NULL) {
+    buf_add_string(buffer, stdout_read_buffer);
+    stdout_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static int
+process_mocked_read_stderr(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stderr_read_buffer != NULL) {
+    buf_add_string(buffer, stderr_read_buffer);
+    stderr_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static void
+process_mocked_write_stdin(process_t *process, buf_t *buffer)
+{
+  const size_t size = buf_datalen(buffer);
+
+  if (size == 0)
+    return;
+
+  char *data = tor_malloc_zero(size + 1);
+  process_data_t *process_data = process_get_data(process);
+
+  buf_get_bytes(buffer, data, size);
+  smartlist_add(process_data->stdin_data, data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+static void
+test_default_values(void *arg)
+{
+  (void)arg;
+  process_t *process = process_new("/path/to/nothing");
+
+  /* We are not running by default. */
+  tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process));
+
+  /* We use the line protocol by default. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  /* We don't set any custom data by default. */
+  tt_ptr_op(NULL, OP_EQ, process_get_data(process));
+
+  /* Our command was given to the process_t's constructor in process_new(). */
+  tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process));
+
+  /* Make sure we are listed in the list of proccesses. */
+  tt_assert(smartlist_contains(process_get_all_processes(),
+                               process));
+
+  /* Default PID is 0. */
+  tt_u64_op(0, OP_EQ, process_get_pid(process));
+
+  /* Our arguments should be empty. */
+  tt_int_op(0, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+ done:
+  process_free(process);
+}
+
+static void
+test_environment(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("");
+  process_environment_t *env = NULL;
+
+  process_set_environment(process, "E", "F");
+  process_set_environment(process, "C", "D");
+  process_set_environment(process, "A", "B");
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "A=B\0C=D\0E=F\0", 12);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "A=B");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "C=D");
+  tt_str_op(env->unixoid_environment_block[2], OP_EQ,
+            "E=F");
+  tt_ptr_op(env->unixoid_environment_block[3], OP_EQ,
+            NULL);
+  process_environment_free(env);
+
+  /* Reset our environment. */
+  smartlist_t *new_env = smartlist_new();
+  smartlist_add(new_env, (char *)"FOO=bar");
+  smartlist_add(new_env, (char *)"HELLO=world");
+
+  process_reset_environment(process, new_env);
+  smartlist_free(new_env);
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "FOO=bar\0HELLO=world\0", 20);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "FOO=bar");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "HELLO=world");
+  tt_ptr_op(env->unixoid_environment_block[2], OP_EQ,
+            NULL);
+
+ done:
+  process_environment_free(env);
+  process_free(process);
+}
+
+static void
+test_stringified_types(void *arg)
+{
+  (void)arg;
+
+  /* process_protocol_t values. */
+  tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW));
+  tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE));
+
+  /* process_status_t values. */
+  tt_str_op("not running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_NOT_RUNNING));
+  tt_str_op("running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_RUNNING));
+  tt_str_op("error", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_ERROR));
+
+ done:
+  return;
+}
+
+static void
+test_line_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\r\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_multi(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Onion Onion Onion");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "A B C D");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Foo bar baz");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Onion Onion Onion");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_partial(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout this is a partial line ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr this is a partial line ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should NOT be ready. */
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+                          "Hello stdout this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+                          "Hello stderr this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_raw_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_protocol(process, PROCESS_PROTOCOL_RAW);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the raw protocol. */
+  tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout\n");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Hello, again, stdout\nThis contains multiple lines");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr\n");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Hello, again, stderr\nThis contains multiple lines");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_write_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  MOCK(process_write_stdin, process_mocked_write_stdin);
+
+  process_write(process, (uint8_t *)"Hello world\n", 12);
+  process_notify_event_stdin(process);
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data));
+
+  process_printf(process, "Hello %s !\n", "moon");
+  process_notify_event_stdin(process);
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data));
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_write_stdin);
+}
+
+static void
+test_exit_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Our default is 0. */
+  tt_u64_op(0, OP_EQ, process_data->exit_code);
+
+  /* Fake that we are a running process. */
+  process_set_status(process, PROCESS_STATUS_RUNNING);
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Fake an exit. */
+  process_notify_event_exit(process, 1337);
+
+  /* Check if our state changed and if our callback fired. */
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+  tt_u64_op(1337, OP_EQ, process_data->exit_code);
+
+ done:
+  process_set_data(process, process_data);
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_argv_simple(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("/bin/cat");
+  char **argv = NULL;
+
+  /* Setup some arguments. */
+  process_append_argument(process, "foo");
+  process_append_argument(process, "bar");
+  process_append_argument(process, "baz");
+
+  /* Check the number of elements. */
+  tt_int_op(3, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+  /* Let's try to convert it into a Unix style char **argv. */
+  argv = process_get_argv(process);
+
+  /* Check our values. */
+  tt_str_op(argv[0], OP_EQ, "/bin/cat");
+  tt_str_op(argv[1], OP_EQ, "foo");
+  tt_str_op(argv[2], OP_EQ, "bar");
+  tt_str_op(argv[3], OP_EQ, "baz");
+  tt_ptr_op(argv[4], OP_EQ, NULL);
+
+ done:
+  tor_free(argv);
+  process_free(process);
+}
+
+static void
+test_unix(void *arg)
+{
+  (void)arg;
+#ifndef _WIN32
+  process_t *process = process_new("");
+
+  /* On Unix all processes should have a Unix process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_unix_process(process));
+
+ done:
+  process_free(process);
+#endif /* !defined(_WIN32) */
+}
+
+static void
+test_win32(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  process_t *process = process_new("");
+  char *joined_argv = NULL;
+
+  /* On Win32 all processes should have a Win32 process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_win32_process(process));
+
+  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
+   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
+   * will try to only generate simple cases for the child process to parse;
+   * i.e. we never embed quoted strings in arguments. */
+
+  const char *argvs[][4] = {
+    {"a", "bb", "CCC", NULL}, // Normal
+    {NULL, NULL, NULL, NULL}, // Empty argument list
+    {"", NULL, NULL, NULL}, // Empty argument
+    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
+    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
+    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
+    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
+    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
+    { NULL } // Terminator
+  };
+
+  const char *cmdlines[] = {
+    "a bb CCC",
+    "",
+    "\"\"",
+    "\\\"a b\\\"b CCC\\\"",
+    "\"a\tbc\" \"dd  dd\" E",
+    "a\\\\\\b \"de fg\" H",
+    "a\\\\\\\"b \\c D\\",
+    "\"a\\\\b c\" d E",
+    NULL // Terminator
+  };
+
+  int i;
+
+  for (i=0; cmdlines[i]!=NULL; i++) {
+    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
+    joined_argv = tor_join_win_cmdline(argvs[i]);
+    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
+    tor_free(joined_argv);
+  }
+
+ done:
+  tor_free(joined_argv);
+  process_free(process);
+#endif /* defined(_WIN32) */
+}
+
+struct testcase_t process_tests[] = {
+  { "default_values", test_default_values, TT_FORK, NULL, NULL },
+  { "environment", test_environment, TT_FORK, NULL, NULL },
+  { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL },
+  { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL },
+  { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL },
+  { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL },
+  { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL },
+  { "write_simple", test_write_simple, TT_FORK, NULL, NULL },
+  { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL },
+  { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL },
+  { "unix", test_unix, TT_FORK, NULL, NULL },
+  { "win32", test_win32, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c
new file mode 100644
index 0000000000..14865cff13
--- /dev/null
+++ b/src/test/test_process_descs.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "test/test.h"
+
+static void
+test_process_descs_versions(void *arg)
+{
+  (void)arg;
+  struct {
+    const char *version;
+    bool should_reject;
+  } cases[] = {
+    // a very old version: reject.
+    { "Tor 0.1.2.3-alpha", true },
+    // a non-tor program: don't reject.
+    { "Wombat 0.1.2.3-alpha", false },
+    // some unsupported versions: reject.
+    { "Tor 0.2.9.4-alpha", true },
+    { "Tor 0.2.9.5-alpha", true },
+    { "Tor 0.2.9.100", true },
+    { "Tor 0.3.0.0-alpha-dev", true },
+    { "Tor 0.3.0.2-alpha", true },
+    { "Tor 0.3.0.5", true },
+    { "Tor 0.3.1.4", true },
+    { "Tor 0.3.2.4", true },
+    { "Tor 0.3.3.4", true },
+    { "Tor 0.3.4.1-alpha", true },
+    { "Tor 0.3.4.100", true },
+    { "Tor 0.3.5.1-alpha", true },
+    { "Tor 0.3.5.6-rc", true},
+    { "Tor 0.4.0.1-alpha", true },
+    { "Tor 0.4.0.5", true },
+    { "Tor 0.4.1.1-alpha", true },
+    { "Tor 0.4.1.4-rc", true },
+    // new enough to be supported
+    { "Tor 0.3.5.7", false },
+    { "Tor 0.3.5.8", false },
+    { "Tor 0.4.1.5", false },
+    { "Tor 0.4.2.1-alpha", false },
+    { "Tor 0.4.2.4-rc", false },
+    { "Tor 0.4.3.0-alpha-dev", false },
+    // Very far in the future
+    { "Tor 100.100.1.5", false },
+  };
+  size_t n_cases = ARRAY_LENGTH(cases);
+
+  for (unsigned i = 0; i < n_cases; ++i) {
+    const char *msg = NULL;
+    bool rejected = dirserv_rejects_tor_version(cases[i].version, &msg);
+    tt_int_op(rejected, OP_EQ, cases[i].should_reject);
+    tt_int_op(msg == NULL, OP_EQ, rejected == false);
+  }
+
+ done:
+  ;
+}
+
+#define T(name,flags)                                   \
+  { #name, test_process_descs_##name, (flags), NULL, NULL }
+
+struct testcase_t process_descs_tests[] = {
+  T(versions,0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c
new file mode 100644
index 0000000000..f74d4adc9a
--- /dev/null
+++ b/src/test/test_process_slow.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process_slow.c
+ * \brief Slow test cases for the Process API.
+ */
+
+#define MAINLOOP_PRIVATE
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/process/process.h"
+#include "lib/process/waitpid.h"
+#include "test/test.h"
+
+#ifndef BUILDDIR
+#define BUILDDIR "."
+#endif
+
+#ifdef _WIN32
+#define TEST_PROCESS "test-process.exe"
+#else
+#define TEST_PROCESS BUILDDIR "/src/test/test-process"
+#endif /* defined(_WIN32) */
+
+/** Timer that ticks once a second and stop the event loop after 5 ticks. */
+static periodic_timer_t *main_loop_timeout_timer;
+
+/** How many times have our timer ticked? */
+static int timer_tick_count;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+  bool did_exit;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  process_status_t status;
+
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+  process_data->did_exit = true;
+
+  /* Check if our process is still running? */
+  status = process_get_status(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+#ifdef _WIN32
+static const char *
+get_win32_test_binary_path(void)
+{
+  static char buffer[MAX_PATH];
+
+  /* Get the absolute path of our binary: \path\to\test-slow.exe. */
+  GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer));
+
+  /* Find our process name. */
+  char *offset = strstr(buffer, "test-slow.exe");
+  tt_ptr_op(offset, OP_NE, NULL);
+
+  /* Change test-slow.exe to test-process.exe. */
+  memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS));
+
+  return buffer;
+ done:
+  return NULL;
+}
+#endif /* defined(_WIN32) */
+
+static void
+main_loop_timeout_cb(periodic_timer_t *timer, void *data)
+{
+  /* Sanity check. */
+  tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer);
+  tt_ptr_op(data, OP_NE, NULL);
+
+  /* Our process data. */
+  process_data_t *process_data = data;
+
+  /* Our process did exit. */
+  if (process_data->did_exit)
+    tor_shutdown_event_loop_and_exit(0);
+
+  /* Have we been called 10 times we exit the main loop. */
+  timer_tick_count++;
+
+  tt_int_op(timer_tick_count, OP_LT, 10);
+
+#ifndef _WIN32
+  /* Call waitpid callbacks. */
+  notify_pending_waitpid_callbacks();
+#endif
+
+  return;
+ done:
+  /* Exit with an error. */
+  tor_shutdown_event_loop_and_exit(-1);
+}
+
+static void
+run_main_loop(process_data_t *process_data)
+{
+  int ret;
+
+  /* Wake up after 1 seconds. */
+  static const struct timeval interval = {1, 0};
+
+  timer_tick_count = 0;
+  main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(),
+                                               &interval,
+                                               main_loop_timeout_cb,
+                                               process_data);
+
+  /* Run our main loop. */
+  ret = run_main_loop_until_done();
+
+  /* Clean up our main loop timeout timer. */
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  periodic_timer_free(main_loop_timeout_timer);
+}
+
+static void
+test_callbacks(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Set environment variable. */
+  process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!");
+
+  /* Add some arguments. */
+  process_append_argument(process, "This is the first one");
+  process_append_argument(process, "Second one");
+  process_append_argument(process, "Third: Foo bar baz");
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Write some lines to stdin. */
+  process_printf(process, "Hi process!\r\n");
+  process_printf(process, "Can you read more than one line?\n");
+  process_printf(process, "Can you read partial ...");
+  process_printf(process, " lines?\r\n");
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* We returned. Let's see what our event loop said. */
+  tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12);
+  tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3);
+  tt_assert(process_data->did_exit);
+  tt_u64_op(process_data->exit_code, OP_EQ, 0);
+
+  /* Check stdout output. */
+  char argv0_expected[256];
+  tor_snprintf(argv0_expected, sizeof(argv0_expected),
+               "argv[0] = '%s'", filename);
+
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            argv0_expected);
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "argv[1] = 'This is the first one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "argv[2] = 'Second one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "argv[3] = 'Third: Foo bar baz'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ,
+            "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ,
+            "Output on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ,
+            "Partial line on stdout ...end of partial line on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ,
+            "Read line from stdin: 'Hi process!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ,
+            "Read line from stdin: 'Can you read more than one line?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ,
+            "Read line from stdin: 'Can you read partial ... lines?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ,
+            "We are done for here, thank you!");
+
+  /* Check stderr output. */
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Output on stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Partial line on stderr ...end of partial line on stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_callbacks_terminate(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Zap our process. */
+  bool success;
+
+  success = process_terminate(process);
+  tt_assert(success);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Check if we did exit. */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_nonexistent_executable(void *arg)
+{
+  (void)arg;
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new("binary-does-not-exist");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_exec(process);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Ensure that the exit callback was actually called even though the binary
+   * did not exist.
+   */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+struct testcase_t slow_process_tests[] = {
+  { "callbacks", test_callbacks, 0, NULL, NULL },
+  { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL },
+  { "nonexistent_executable", test_nonexistent_executable, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_procmon.c b/src/test/test_procmon.c
index e23578f4fd..1752008f63 100644
--- a/src/test/test_procmon.c
+++ b/src/test/test_procmon.c
@@ -1,7 +1,6 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define PROCMON_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
 #include "test/test.h"
@@ -10,8 +9,6 @@
 
 #include "test/log_test_helpers.h"
 
-#define NS_MODULE procmon
-
 struct event_base;
 
 static void
diff --git a/src/test/test_proto_haproxy.c b/src/test/test_proto_haproxy.c
new file mode 100644
index 0000000000..040354ec1f
--- /dev/null
+++ b/src/test/test_proto_haproxy.c
@@ -0,0 +1,66 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_proto_haproxy.c
+ * \brief Tests for our HAProxy protocol parser code
+ */
+
+#define PROTO_HAPROXY_PRIVATE
+
+#include "test/test.h"
+#include "core/proto/proto_haproxy.h"
+#include "test/log_test_helpers.h"
+
+static void
+test_format_proxy_header_line(void *arg)
+{
+  tor_addr_t addr;
+  tor_addr_port_t *addr_port = NULL;
+  char *output = NULL;
+
+  (void) arg;
+
+  /* IPv4 address. */
+  tor_addr_parse(&addr, "192.168.1.2");
+  addr_port = tor_addr_port_new(&addr, 8000);
+  output = haproxy_format_proxy_header_line(addr_port);
+
+  tt_str_op(output, OP_EQ, "PROXY TCP4 0.0.0.0 192.168.1.2 0 8000\r\n");
+
+  tor_free(addr_port);
+  tor_free(output);
+
+  /* IPv6 address. */
+  tor_addr_parse(&addr, "123:45:6789::5005:11");
+  addr_port = tor_addr_port_new(&addr, 8000);
+  output = haproxy_format_proxy_header_line(addr_port);
+
+  tt_str_op(output, OP_EQ, "PROXY TCP6 :: 123:45:6789::5005:11 0 8000\r\n");
+
+  tor_free(addr_port);
+  tor_free(output);
+
+  /* UNIX socket address. */
+  memset(&addr, 0, sizeof(addr));
+  addr.family = AF_UNIX;
+  addr_port = tor_addr_port_new(&addr, 8000);
+  output = haproxy_format_proxy_header_line(addr_port);
+
+  /* If it's not an IPv4 or IPv6 address, haproxy_format_proxy_header_line
+   * must return NULL. */
+  tt_ptr_op(output, OP_EQ, NULL);
+
+  tor_free(addr_port);
+  tor_free(output);
+
+ done:
+  tor_free(addr_port);
+  tor_free(output);
+}
+
+struct testcase_t proto_haproxy_tests[] = {
+  { "format_proxy_header_line", test_format_proxy_header_line, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c
index 08990c0b6a..481d78b2c1 100644
--- a/src/test/test_proto_http.c
+++ b/src/test/test_proto_http.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 #include "test/log_test_helpers.h"
 
diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c
index af9cf7eee2..64bf5c4993 100644
--- a/src/test/test_proto_misc.c
+++ b/src/test/test_proto_misc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_cell.h"
diff --git a/src/test/test_protover.c b/src/test/test_protover.c
index b4689045cf..71f984a0ac 100644
--- a/src/test/test_protover.c
+++ b/src/test/test_protover.c
@@ -1,7 +1,8 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROTOVER_PRIVATE
+#define DIRVOTE_PRIVATE
 
 #include "orconfig.h"
 #include "test/test.h"
@@ -12,6 +13,8 @@
 #include "core/or/connection_or.h"
 #include "lib/tls/tortls.h"
 
+#include "feature/dirauth/dirvote.h"
+
 static void
 test_protover_parse(void *arg)
 {
@@ -22,7 +25,7 @@ test_protover_parse(void *arg)
   tt_skip();
  done:
   ;
-#else
+#else /* !defined(HAVE_RUST) */
   char *re_encoded = NULL;
 
   const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16";
@@ -85,7 +88,7 @@ test_protover_parse(void *arg)
     SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(elts);
   tor_free(re_encoded);
-#endif
+#endif /* defined(HAVE_RUST) */
 }
 
 static void
@@ -129,7 +132,7 @@ test_protover_parse_fail(void *arg)
                            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
   tt_ptr_op(elts, OP_EQ, NULL);
 
-#endif
+#endif /* defined(HAVE_RUST) */
  done:
   ;
 }
@@ -271,6 +274,7 @@ test_protover_all_supported(void *arg)
   tt_assert(protover_all_supported("Fribble=", &msg));
   tt_ptr_op(msg, OP_EQ, NULL);
 
+#ifndef ALL_BUGS_ARE_FATAL
   /* If we get a completely unparseable list, protover_all_supported should
    * hit a fatal assertion for BUG(entries == NULL). */
   tor_capture_bugs_(1);
@@ -282,9 +286,10 @@ test_protover_all_supported(void *arg)
   tor_capture_bugs_(1);
   tt_assert(protover_all_supported("Sleen=1-4294967295", &msg));
   tor_end_capture_bugs_();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
 
   /* Protocol name too long */
-#ifndef HAVE_RUST // XXXXXX ?????
+#if !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL)
   tor_capture_bugs_(1);
   tt_assert(protover_all_supported(
                  "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
@@ -292,7 +297,7 @@ test_protover_all_supported(void *arg)
                  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
                  "aaaaaaaaaaaa=1-65536", &msg));
   tor_end_capture_bugs_();
-#endif
+#endif /* !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL) */
 
  done:
   tor_end_capture_bugs_();
@@ -416,7 +421,7 @@ test_protover_supported_protocols(void *arg)
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
                                             PROTOVER_LINKAUTH_V1));
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   /* Latest LinkAuth is not exposed in the headers. */
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
@@ -587,6 +592,43 @@ test_protover_vote_roundtrip(void *args)
   tor_free(result);
 }
 
+static void
+test_protover_vote_roundtrip_ours(void *args)
+{
+  (void) args;
+  const char *examples[] = {
+    protover_get_supported_protocols(),
+    DIRVOTE_RECOMMEND_RELAY_PROTO,
+    DIRVOTE_RECOMMEND_CLIENT_PROTO,
+    DIRVOTE_REQUIRE_RELAY_PROTO,
+    DIRVOTE_REQUIRE_CLIENT_PROTO,
+  };
+  unsigned u;
+  smartlist_t *votes = smartlist_new();
+  char *result = NULL;
+
+  for (u = 0; u < ARRAY_LENGTH(examples); ++u) {
+    tt_assert(examples[u]);
+    const char *input = examples[u];
+    const char *expected_output = examples[u];
+
+    smartlist_add(votes, (void*)input);
+    result = protover_compute_vote(votes, 1);
+    if (expected_output != NULL) {
+      tt_str_op(result, OP_EQ, expected_output);
+    } else {
+      tt_str_op(result, OP_EQ, "");
+    }
+
+    smartlist_clear(votes);
+    tor_free(result);
+  }
+
+ done:
+  smartlist_free(votes);
+  tor_free(result);
+}
+
 #define PV_TEST(name, flags)                       \
   { #name, test_protover_ ##name, (flags), NULL, NULL }
 
@@ -600,5 +642,6 @@ struct testcase_t protover_tests[] = {
   PV_TEST(supports_version, 0),
   PV_TEST(supported_protocols, 0),
   PV_TEST(vote_roundtrip, 0),
+  PV_TEST(vote_roundtrip_ours, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_pt.c b/src/test/test_pt.c
index 1f9786648a..893fec3674 100644
--- a/src/test/test_pt.c
+++ b/src/test/test_pt.c
@@ -1,28 +1,30 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define PT_PRIVATE
-#define UTIL_PRIVATE
 #define STATEFILE_PRIVATE
-#define CONTROL_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define PROCESS_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/transports.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/statefile.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/process/process.h"
 
 #include "app/config/or_state_st.h"
 
+#include "test/log_test_helpers.h"
+
 static void
 reset_mp(managed_proxy_t *mp)
 {
@@ -288,41 +290,35 @@ test_pt_get_extrainfo_string(void *arg)
   tor_free(s);
 }
 
-#ifdef _WIN32
-#define STDIN_HANDLE HANDLE*
-#else
-#define STDIN_HANDLE int
-#endif
-
-static smartlist_t *
-tor_get_lines_from_handle_replacement(STDIN_HANDLE handle,
-                                      enum stream_status *stream_status_out)
+static int
+process_read_stdout_replacement(process_t *process, buf_t *buffer)
 {
+  (void)process;
   static int times_called = 0;
-  smartlist_t *retval_sl = smartlist_new();
-
-  (void) handle;
-  (void) stream_status_out;
 
   /* Generate some dummy CMETHOD lines the first 5 times. The 6th
      time, send 'CMETHODS DONE' to finish configuring the proxy. */
-  if (times_called++ != 5) {
-    smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d",
+  times_called++;
+
+  if (times_called <= 5) {
+    buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n",
                            times_called, times_called);
-  } else {
-    smartlist_add_strdup(retval_sl, "SMETHODS DONE");
+  } else if (times_called <= 6) {
+    buf_add_string(buffer, "SMETHODS DONE\n");
+  } else if (times_called <= 7) {
+    buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something "
+                           "bad happened. What do we do!?\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n");
+  } else if (times_called <= 8) {
+    buf_add_string(buffer, "STATUS TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\n");
   }
 
-  return retval_sl;
-}
-
-/* NOP mock */
-static void
-tor_process_handle_destroy_replacement(process_handle_t *process_handle,
-                                       int also_terminate_process)
-{
-  (void) process_handle;
-  (void) also_terminate_process;
+  return (int)buf_datalen(buffer);
 }
 
 static or_state_t *dummy_state = NULL;
@@ -355,12 +351,9 @@ test_pt_configure_proxy(void *arg)
   managed_proxy_t *mp = NULL;
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
-  MOCK(tor_get_lines_from_handle,
-       tor_get_lines_from_handle_replacement);
-  MOCK(tor_process_handle_destroy,
-       tor_process_handle_destroy_replacement);
+  MOCK(process_read_stdout, process_read_stdout_replacement);
   MOCK(get_or_state,
        get_or_state_replacement);
   MOCK(queue_control_event_string,
@@ -372,24 +365,34 @@ test_pt_configure_proxy(void *arg)
   mp->conf_state = PT_PROTO_ACCEPTING_METHODS;
   mp->transports = smartlist_new();
   mp->transports_to_launch = smartlist_new();
-  mp->process_handle = tor_malloc_zero(sizeof(process_handle_t));
   mp->argv = tor_malloc_zero(sizeof(char*)*2);
   mp->argv[0] = tor_strdup("<testcase>");
   mp->is_server = 1;
 
+  /* Configure the process. */
+  mp->process = process_new("");
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_data(mp->process, mp);
+
   /* Test the return value of configure_proxy() by calling it some
      times while it is uninitialized and then finally finalizing its
      configuration. */
   for (i = 0 ; i < 5 ; i++) {
+    /* force a read from our mocked stdout reader. */
+    process_notify_event_stdout(mp->process);
+    /* try to configure our proxy. */
     retval = configure_proxy(mp);
     /* retval should be zero because proxy hasn't finished configuring yet */
     tt_int_op(retval, OP_EQ, 0);
     /* check the number of registered transports */
-    tt_assert(smartlist_len(mp->transports) == i+1);
+    tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1);
     /* check that the mp is still waiting for transports */
     tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS);
   }
 
+  /* Get the SMETHOD DONE written to the process. */
+  process_notify_event_stdout(mp->process);
+
   /* this last configure_proxy() should finalize the proxy configuration. */
   retval = configure_proxy(mp);
   /* retval should be 1 since the proxy finished configuring */
@@ -412,6 +415,49 @@ test_pt_configure_proxy(void *arg)
   tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ,
             "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n");
 
+  /* Get the log message out. */
+  setup_full_capture_of_logs(LOG_ERR);
+  process_notify_event_stdout(mp->process);
+  expect_single_log_msg_containing("Oh noes, something bad happened");
+  teardown_capture_of_logs();
+
+  tt_int_op(controlevent_n, OP_EQ, 10);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10);
+  tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=error "
+            "MESSAGE=\"Oh noes, "
+            "something bad happened. What do we do!?\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=warning "
+            "MESSAGE=\"warning msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=notice "
+            "MESSAGE=\"notice msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=info "
+            "MESSAGE=\"info msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=debug "
+            "MESSAGE=\"debug msg\"\r\n");
+
+  /* Get the STATUS messages out. */
+  process_notify_event_stdout(mp->process);
+
+  tt_int_op(controlevent_n, OP_EQ, 13);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13);
+
+  tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\r\n");
+
   { /* check that the transport info were saved properly in the tor state */
     config_line_t *transport_in_state = NULL;
     smartlist_t *transport_info_sl = smartlist_new();
@@ -434,9 +480,9 @@ test_pt_configure_proxy(void *arg)
   }
 
  done:
+  teardown_capture_of_logs();
   or_state_free(dummy_state);
-  UNMOCK(tor_get_lines_from_handle);
-  UNMOCK(tor_process_handle_destroy);
+  UNMOCK(process_read_stdout);
   UNMOCK(get_or_state);
   UNMOCK(queue_control_event_string);
   if (controlevent_msgs) {
@@ -449,7 +495,7 @@ test_pt_configure_proxy(void *arg)
     smartlist_free(mp->transports);
   }
   smartlist_free(mp->transports_to_launch);
-  tor_free(mp->process_handle);
+  process_free(mp->process);
   tor_free(mp->argv[0]);
   tor_free(mp->argv);
   tor_free(mp);
@@ -533,8 +579,10 @@ test_get_pt_proxy_uri(void *arg)
     tor_free(uri);
 }
 
+#ifndef COCCI
 #define PT_LEGACY(name)                                               \
-  { #name, test_pt_ ## name , 0, NULL, NULL }
+  { (#name), test_pt_ ## name , 0, NULL, NULL }
+#endif
 
 struct testcase_t pt_tests[] = {
   PT_LEGACY(parsing),
diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c
new file mode 100644
index 0000000000..25b893c4c0
--- /dev/null
+++ b/src/test/test_ptr_slow.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/ptr_helpers.h"
+
+#include <stdint.h>
+#include <limits.h>
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_int_voidptr_roundtrip(int a)
+{
+  intptr_t ap = (intptr_t)a;
+  void *b = cast_intptr_to_voidstar(ap);
+  intptr_t c = cast_voidstar_to_intptr(b);
+  void *d = cast_intptr_to_voidstar(c);
+
+  tt_assert(ap == c);
+  tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_int_voidstar_interop(void *arg)
+{
+  int a;
+  (void)arg;
+
+  for (a = -1024; a <= 1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MIN; a <= INT_MIN+1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MAX-1024; a < INT_MAX; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_int_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_uint_voidptr_roundtrip(unsigned int a)
+{
+ uintptr_t ap = (uintptr_t)a;
+ void *b = cast_uintptr_to_voidstar(ap);
+ uintptr_t c = cast_voidstar_to_uintptr(b);
+ void *d = cast_uintptr_to_voidstar(c);
+
+ tt_assert(ap == c);
+ tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_uint_voidstar_interop(void *arg)
+{
+  unsigned int a;
+  (void)arg;
+
+  for (a = 0; a <= 1024; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  for (a = UINT_MAX-1024; a < UINT_MAX; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_uint_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+struct testcase_t slow_ptr_tests[] = {
+  { .name = "int_voidstar_interop",
+    .fn = test_int_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  { .name = "uint_voidstar_interop",
+    .fn = test_uint_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c
new file mode 100644
index 0000000000..5f9005926c
--- /dev/null
+++ b/src/test/test_pubsub_build.c
@@ -0,0 +1,578 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+#define PUBSUB_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "test/log_test_helpers.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_int_fmt(msg_aux_data_t aux)
+{
+  int val = (int) aux.u64;
+  char *r=NULL;
+  tor_asprintf(&r, "%d", val);
+  return r;
+}
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+
+static dispatch_typefns_t intfns = {
+  .fmt_fn = ex_int_fmt
+};
+
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+DECLARE_MESSAGE_INT(bunch_of_coconuts, int, int);
+DECLARE_PUBLISH(bunch_of_coconuts);
+DECLARE_SUBSCRIBE(bunch_of_coconuts, coconut_recipient_cb);
+
+DECLARE_MESSAGE(yes_we_have_no, string, char *);
+DECLARE_PUBLISH(yes_we_have_no);
+DECLARE_SUBSCRIBE(yes_we_have_no, absent_item_cb);
+
+static void
+coconut_recipient_cb(const msg_t *m, int n_coconuts)
+{
+  (void)m;
+  (void)n_coconuts;
+}
+
+static void
+absent_item_cb(const msg_t *m, const char *fruitname)
+{
+  (void)m;
+  (void)fruitname;
+}
+
+#define FLAG_SKIP 99999
+
+static void
+seed_dispatch_builder(pubsub_builder_t *b,
+                      unsigned fl1, unsigned fl2, unsigned fl3, unsigned fl4)
+{
+  pubsub_connector_t *c = NULL;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    DISPATCH_REGISTER_TYPE(c, int, &intfns);
+    if (fl1 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, fl1);
+    if (fl2 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, yes_we_have_no, fl2);
+    pubsub_connector_free(c);
+  }
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2"));
+    DISPATCH_REGISTER_TYPE(c, string, &stringfns);
+    if (fl3 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, yes_we_have_no, fl3);
+    if (fl4 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, bunch_of_coconuts, fl4);
+    pubsub_connector_free(c);
+  }
+}
+
+static void
+seed_pubsub_builder_basic(pubsub_builder_t *b)
+{
+  seed_dispatch_builder(b, 0, 0, 0, 0);
+}
+
+/* Regular builder with valid types and messages.
+ */
+static void
+test_pubsub_build_types_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  pubsub_items_t *items = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  dispatcher = pubsub_builder_finalize(b, &items);
+  b = NULL;
+  tt_assert(dispatcher);
+  tt_assert(items);
+  tt_int_op(smartlist_len(items->items), OP_EQ, 4);
+
+  // Make sure that the bindings got build correctly.
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, dispatcher);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+  tt_int_op(dispatcher->n_types, OP_GE, 2);
+  tt_assert(dispatcher->typefns);
+
+  tt_assert(dispatcher->typefns[get_msg_type_id("int")].fmt_fn == ex_int_fmt);
+  tt_assert(dispatcher->typefns[get_msg_type_id("string")].fmt_fn ==
+            ex_str_fmt);
+
+  // Now clear the bindings, like we would do before freeing the
+  // the dispatcher.
+  pubsub_items_clear_bindings(items);
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, NULL);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  pubsub_items_free(items);
+}
+
+/* We fail if the same type is defined in two places with different functions.
+ */
+static void
+test_pubsub_build_types_decls_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    // Extra declaration of int: we don't allow this.
+    DISPATCH_REGISTER_TYPE(c, int, &stringfns);
+    pubsub_connector_free(c);
+  }
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+  // expect_log_msg_containing("(int) declared twice"); // XXXX
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* If a message ID exists but nobody is publishing or subscribing to it,
+ * that's okay. */
+static void
+test_pubsub_build_unused_message(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  // This message isn't actually generated by anyone, but that will be fine:
+  // we just log it at info.
+  get_message_id("unused");
+  setup_capture_of_logs(LOG_INFO);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+  expect_log_msg_containing(
+     "Nobody is publishing or subscribing to message");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Publishing or subscribing to a message with no subscribers / publishers
+ * should fail and warn. */
+static void
+test_pubsub_build_missing_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, FLAG_SKIP, FLAG_SKIP);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing(
+       "Message \"bunch_of_coconuts\" has publishers, but no subscribers.");
+  expect_log_msg_containing(
+       "Message \"yes_we_have_no\" has subscribers, but no publishers.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Make sure that a stub publisher or subscriber prevents an error from
+ * happening even if there are no other publishers/subscribers for a message
+ */
+static void
+test_pubsub_build_stub_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, DISP_FLAG_STUB, DISP_FLAG_STUB);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // no subscribers
+  tt_ptr_op(NULL, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* Only one channel per msg id. */
+static void
+test_pubsub_build_channels_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("int"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent channels.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Only one type per msg id. */
+static void
+test_pubsub_build_types_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent message types.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* The same module can't publish and subscribe the same message */
+static void
+test_pubsub_build_pubsub_same(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    // already publishing this.
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is published "
+                            "and subscribed by the same subsystem \"sys1\".");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* More than one subsystem may publish or subscribe, and that's okay. */
+static void
+test_pubsub_build_pubsub_multi(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_add_pub_(c, &btmp, get_channel_id("main"),
+                    get_message_id("yes_we_have_no"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 2 subscribers.
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(2, OP_EQ, ent->n_enabled);
+  tt_int_op(2, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+static void
+some_other_coconut_hook(const msg_t *m)
+{
+  (void)m;
+}
+
+/* Subscribe hooks should be build correctly when there are a bunch of
+ * them. */
+static void
+test_pubsub_build_sub_many(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  char *sysname = NULL;
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  int i;
+  for (i = 1; i < 100; ++i) {
+    tor_asprintf(&sysname, "system%d",i);
+    c = pubsub_connector_for_subsystem(b, get_subsys_id(sysname));
+    if (i % 7) {
+      DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    } else {
+      pubsub_add_sub_(c, some_other_coconut_hook,
+                      get_channel_id("main"),
+                      get_message_id("bunch_of_coconuts"),
+                      get_msg_type_id("int"),
+                      0 /* flags */,
+                      "somewhere.c", 22);
+    }
+    pubsub_connector_free(c);
+    tor_free(sysname);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(100, OP_EQ, ent->n_enabled);
+  tt_int_op(100, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[76].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[77].fn, OP_EQ, some_other_coconut_hook);
+  tt_ptr_op(ent->rcv[78].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  tor_free(sysname);
+}
+
+/* It's fine to declare the excl flag. */
+static void
+test_pubsub_build_excl_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  // Try one excl/excl pair and one excl/non pair.
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, 0,
+                        DISP_FLAG_EXCL, DISP_FLAG_EXCL);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]->n_enabled);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* but if you declare the excl flag, you need to mean it. */
+static void
+test_pubsub_build_excl_bad(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, DISP_FLAG_EXCL,
+                        0, 0);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, 0);
+    DISPATCH_ADD_SUB_(c, main, yes_we_have_no, 0);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("has multiple publishers, but at least one is "
+                            "marked as exclusive.");
+  expect_log_msg_containing("has multiple subscribers, but at least one is "
+                            "marked as exclusive.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+#define T(name, flags)                                          \
+  { #name, test_pubsub_build_ ## name , (flags), NULL, NULL }
+
+struct testcase_t pubsub_build_tests[] = {
+  T(types_ok, TT_FORK),
+  T(types_decls_conflict, TT_FORK),
+  T(unused_message, TT_FORK),
+  T(missing_pubsub, TT_FORK),
+  T(stub_pubsub, TT_FORK),
+  T(channels_conflict, TT_FORK),
+  T(types_conflict, TT_FORK),
+  T(pubsub_same, TT_FORK),
+  T(pubsub_multi, TT_FORK),
+  T(sub_many, TT_FORK),
+  T(excl_ok, TT_FORK),
+  T(excl_bad, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_msg.c b/src/test/test_pubsub_msg.c
new file mode 100644
index 0000000000..3054db885d
--- /dev/null
+++ b/src/test/test_pubsub_msg.c
@@ -0,0 +1,305 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+// We're using the lowest-level publish/subscribe logic here, to avoid the
+// pubsub_macros.h macros and just test the dispatch core.  We'll use a string
+// type for everything.
+
+#define DECLARE_MESSAGE(suffix)                         \
+  static pub_binding_t pub_binding_##suffix;            \
+  static int msg_received_##suffix = 0;                 \
+  static void recv_msg_##suffix(const msg_t *m) {       \
+    (void)m;                                            \
+    ++msg_received_##suffix;                            \
+  }                                                     \
+  EAT_SEMICOLON
+
+#define ADD_PUBLISH(binding_suffix, subsys, channel, msg, flags)        \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_pub_(con, &pub_binding_##binding_suffix,                 \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define ADD_SUBSCRIBE(hook_suffix, subsys, channel, msg, flags)         \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_sub_(con, recv_msg_##hook_suffix,                        \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define SEND(binding_suffix, val)                          \
+  STMT_BEGIN {                                             \
+    msg_aux_data_t data_;                                  \
+    data_.ptr = tor_strdup(val);                           \
+    pubsub_pub_(&pub_binding_##binding_suffix, data_);     \
+  } STMT_END
+
+DECLARE_MESSAGE(msg1);
+DECLARE_MESSAGE(msg2);
+DECLARE_MESSAGE(msg3);
+DECLARE_MESSAGE(msg4);
+DECLARE_MESSAGE(msg5);
+
+static smartlist_t *strings_received = NULL;
+static void
+recv_msg_copy_string(const msg_t *m)
+{
+  const char *s = m->aux_data__.ptr;
+  smartlist_add(strings_received, tor_strdup(s));
+}
+
+static void *
+setup_dispatcher(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  pubsub_connector_t *con;
+
+  {
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id("types"));
+    pubsub_connector_register_type_(con,
+                                    get_msg_type_id("string"),
+                                    &stringfns,
+                                    "nowhere.c", 99);
+    pubsub_connector_free(con);
+  }
+  // message1 has one publisher and one subscriber.
+  ADD_PUBLISH(msg1, sys1, main, message1, 0);
+  ADD_SUBSCRIBE(msg1, sys2, main, message1, 0);
+
+  // message2 has a publisher and a stub subscriber.
+  ADD_PUBLISH(msg2, sys1, main, message2, 0);
+  ADD_SUBSCRIBE(msg2, sys2, main, message2, DISP_FLAG_STUB);
+
+  // message3 has a publisher and three subscribers.
+  ADD_PUBLISH(msg3, sys1, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys2, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys3, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys4, main, message3, 0);
+
+  // message4 has one publisher and two subscribers, but it's on another
+  // channel.
+  ADD_PUBLISH(msg4, sys2, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys1, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys3, other, message4, 0);
+
+  // message5 has a huge number of recipients.
+  ADD_PUBLISH(msg5, sys3, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys4, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys5, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys6, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys7, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys8, main, message5, 0);
+  for (int i = 0; i < 1000-5; ++i) {
+    char *sys;
+    tor_asprintf(&sys, "xsys-%d", i);
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id(sys));
+    pubsub_add_sub_(con, recv_msg_copy_string,
+                    get_channel_id("main"),
+                    get_message_id("message5"),
+                    get_msg_type_id("string"), 0, "here", 100);
+    pubsub_connector_free(con);
+    tor_free(sys);
+  }
+
+  return pubsub_builder_finalize(builder, NULL);
+}
+
+static int
+cleanup_dispatcher(const struct testcase_t *testcase, void *dispatcher_)
+{
+  (void)testcase;
+  dispatch_t *dispatcher = dispatcher_;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+static const struct testcase_setup_t dispatcher_setup = {
+  setup_dispatcher, cleanup_dispatcher
+};
+
+static void
+test_pubsub_msg_minimal(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  SEND(msg1, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(1, OP_EQ, msg_received_msg1); // we got the message!
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_send_to_stub(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg2);
+  SEND(msg2, "hello silence");
+  tt_int_op(0, OP_EQ, msg_received_msg2); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(0, OP_EQ, msg_received_msg2); // doesn't arrive -- stub hook.
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_cancel_msgs(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  for (int i = 0; i < 100; ++i) {
+    SEND(msg1, "hello world");
+  }
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 10));
+  tt_int_op(10, OP_EQ, msg_received_msg1); // we got the message 10 times.
+
+  // At this point, the dispatcher will be freed with queued, undelivered
+  // messages.
+ done:
+  ;
+}
+
+struct alertfn_target {
+  dispatch_t *d;
+  channel_id_t ch;
+  int count;
+};
+static void
+alertfn_generic(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  struct alertfn_target *t = arg;
+  tt_ptr_op(d, OP_EQ, t->d);
+  tt_int_op(ch, OP_EQ, t->ch);
+  ++t->count;
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_alertfns(void *arg)
+{
+  dispatch_t *d = arg;
+  struct alertfn_target ch1_a = { d, get_channel_id("main"), 0 };
+  struct alertfn_target ch2_a = { d, get_channel_id("other"), 0 };
+
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("main"),
+                                  alertfn_generic, &ch1_a));
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("other"),
+                                  alertfn_generic, &ch2_a));
+
+  SEND(msg3, "hello");
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg3, "world");
+  tt_int_op(ch1_a.count, OP_EQ, 1); // only the first message sends an alert
+  tt_int_op(ch2_a.count, OP_EQ, 0); // no alert for 'other'
+
+  SEND(msg4, "worse things happen in C");
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+  // flush the first (main) channel...
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(6, OP_EQ, msg_received_msg3); // 3 subscribers, 2 instances.
+
+  // now that the main channel is flushed, sending another message on it
+  // starts another alert.
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg1, "plover");
+  tt_int_op(ch1_a.count, OP_EQ, 2);
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+/* try more than N_FAST_FNS hooks on msg5 */
+static void
+test_pubsub_msg_many_hooks(void *arg)
+{
+  dispatch_t *d = arg;
+  strings_received = smartlist_new();
+
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  SEND(msg5, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  tt_int_op(0, OP_EQ, smartlist_len(strings_received));
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 100000));
+  tt_int_op(5, OP_EQ, msg_received_msg5);
+  tt_int_op(995, OP_EQ, smartlist_len(strings_received));
+
+ done:
+  SMARTLIST_FOREACH(strings_received, char *, s, tor_free(s));
+  smartlist_free(strings_received);
+}
+
+#define T(name)                                                 \
+  { #name, test_pubsub_msg_ ## name , TT_FORK,                \
+      &dispatcher_setup, NULL }
+
+struct testcase_t pubsub_msg_tests[] = {
+  T(minimal),
+  T(send_to_stub),
+  T(cancel_msgs),
+  T(alertfns),
+  T(many_hooks),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py
index 30a587858f..3fc3deb68e 100644
--- a/src/test/test_rebind.py
+++ b/src/test/test_rebind.py
@@ -1,4 +1,7 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import errno
 import logging
@@ -22,9 +25,10 @@ def skip(msg):
 
 def try_connecting_to_socksport():
     socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    if socks_socket.connect_ex(('127.0.0.1', socks_port)):
+    e = socks_socket.connect_ex(('127.0.0.1', socks_port))
+    if e:
         tor_process.terminate()
-        fail('Cannot connect to SOCKSPort')
+        fail('Cannot connect to SOCKSPort: error ' + os.strerror(e))
     socks_socket.close()
 
 def wait_for_log(s):
diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh
index ea2012957e..879008c1c1 100755
--- a/src/test/test_rebind.sh
+++ b/src/test/test_rebind.sh
@@ -1,24 +1,60 @@
 #!/bin/sh
 
+umask 077
+set -e
 set -x
 
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
 UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
   if test "$APPVEYOR" = 'True'; then
-    echo "This test is disabled on Windows CI, as it requires firewall examptions. Skipping." >&2
+    echo "This test is disabled on Windows CI, as it requires firewall exemptions. Skipping." >&2
     exit 77
   fi
 fi
 
-exitcode=0
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "${TOR_BINARY}" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
+fi
 
 tmpdir=
-clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; }
+clean () {
+  if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then
+    rm -rf "$tmpdir"
+  fi
+}
+
 trap clean EXIT HUP INT TERM
 
-tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`"
+tmpdir="$(mktemp -d -t tor_rebind_test.XXXXXX)"
 if [ -z "$tmpdir" ]; then
   echo >&2 mktemp failed
   exit 2
@@ -27,6 +63,6 @@ elif [ ! -d "$tmpdir" ]; then
   exit 3
 fi
 
-"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TESTING_TOR_BINARY}" "$tmpdir"
+"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TOR_BINARY}" "$tmpdir"
 
 exit $?
diff --git a/src/test/test_relay.c b/src/test/test_relay.c
index 0b7a7be332..066aeaa7b3 100644
--- a/src/test/test_relay.c
+++ b/src/test/test_relay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITBUILD_PRIVATE
@@ -10,7 +10,6 @@
 #include "core/or/channeltls.h"
 #include "feature/stats/rephist.h"
 #include "core/or/relay.h"
-#include "feature/stats/rephist.h"
 #include "lib/container/order.h"
 /* For init/free stuff */
 #include "core/or/scheduler.h"
@@ -21,42 +20,10 @@
 /* Test suite stuff */
 #include "test/test.h"
 #include "test/fakechans.h"
-
-static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan);
+#include "test/fakecircs.h"
 
 static void test_relay_append_cell_to_circuit_queue(void *arg);
 
-static or_circuit_t *
-new_fake_orcirc(channel_t *nchan, channel_t *pchan)
-{
-  or_circuit_t *orcirc = NULL;
-  circuit_t *circ = NULL;
-
-  orcirc = tor_malloc_zero(sizeof(*orcirc));
-  circ = &(orcirc->base_);
-  circ->magic = OR_CIRCUIT_MAGIC;
-
-  circuit_set_n_circid_chan(circ, get_unique_circ_id_by_chan(nchan), nchan);
-  cell_queue_init(&(circ->n_chan_cells));
-
-  circ->n_hop = NULL;
-  circ->streams_blocked_on_n_chan = 0;
-  circ->streams_blocked_on_p_chan = 0;
-  circ->n_delete_pending = 0;
-  circ->p_delete_pending = 0;
-  circ->received_destroy = 0;
-  circ->state = CIRCUIT_STATE_OPEN;
-  circ->purpose = CIRCUIT_PURPOSE_OR;
-  circ->package_window = CIRCWINDOW_START_MAX;
-  circ->deliver_window = CIRCWINDOW_START_MAX;
-  circ->n_chan_create_cell = NULL;
-
-  circuit_set_p_circid_chan(orcirc, get_unique_circ_id_by_chan(pchan), pchan);
-  cell_queue_init(&(orcirc->p_chan_cells));
-
-  return orcirc;
-}
-
 static void
 assert_circuit_ok_mock(const circuit_t *c)
 {
@@ -145,7 +112,7 @@ test_relay_close_circuit(void *arg)
     cell_queue_clear(&orcirc->base_.n_chan_cells);
     cell_queue_clear(&orcirc->p_chan_cells);
   }
-  tor_free(orcirc);
+  free_fake_orcirc(orcirc);
   free_fake_channel(nchan);
   free_fake_channel(pchan);
   UNMOCK(assert_circuit_ok);
@@ -218,7 +185,7 @@ test_relay_append_cell_to_circuit_queue(void *arg)
     cell_queue_clear(&orcirc->base_.n_chan_cells);
     cell_queue_clear(&orcirc->p_chan_cells);
   }
-  tor_free(orcirc);
+  free_fake_orcirc(orcirc);
   free_fake_channel(nchan);
   free_fake_channel(pchan);
 
diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c
index 0623583511..da9e791fb6 100644
--- a/src/test/test_relaycell.c
+++ b/src/test/test_relaycell.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for handling different kinds of relay cell */
@@ -17,6 +17,7 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/connection_edge.h"
+#include "core/or/sendme.h"
 #include "core/or/relay.h"
 #include "test/test.h"
 #include "test/log_test_helpers.h"
@@ -29,7 +30,6 @@
 #include "core/or/half_edge_st.h"
 
 #include "feature/client/circpathbias.h"
-#include "core/or/connection_edge.h"
 
 static int srm_ncalls;
 static entry_connection_t *srm_conn;
@@ -812,7 +812,11 @@ test_circbw_relay(void *arg)
   ASSERT_UNCOUNTED_BW();
 
   /* Sendme on circuit with non-full window: counted */
-  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "");
+  /* Recording a cell, the window is updated after decryption so off by one in
+   * order to record and then we process it with the proper window. */
+  circ->cpath->package_window = 901;
+  sendme_record_cell_digest_on_circ(TO_CIRCUIT(circ), circ->cpath);
   circ->cpath->package_window = 900;
   connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
                                      circ->cpath);
diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c
index fe6889e521..737c243e2d 100644
--- a/src/test/test_relaycrypt.c
+++ b/src/test/test_relaycrypt.c
@@ -1,8 +1,10 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define CRYPT_PATH_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/circuitbuild.h"
 #define CIRCUITLIST_PRIVATE
@@ -10,7 +12,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
-
+#include "core/or/crypt_path.h"
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -49,10 +51,10 @@ testing_circuitset_setup(const struct testcase_t *testcase)
   cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
   for (i=0; i<3; ++i) {
     crypt_path_t *hop = tor_malloc_zero(sizeof(*hop));
-    relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
-                      0, 0);
+    relay_crypto_init(&hop->pvt_crypto, KEY_MATERIAL[i],
+                      sizeof(KEY_MATERIAL[i]), 0, 0);
     hop->state = CPATH_STATE_OPEN;
-    onion_append_to_cpath(&cs->origin_circ->cpath, hop);
+    cpath_extend_linked_list(&cs->origin_circ->cpath, hop);
     tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev);
   }
 
diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c
index 8b0e2df485..06167635c1 100644
--- a/src/test/test_rendcache.c
+++ b/src/test/test_rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -21,8 +21,6 @@
 #include "test/rend_test_helpers.h"
 #include "test/log_test_helpers.h"
 
-#define NS_MODULE rend_cache
-
 static const int RECENT_TIME = -10;
 static const int TIME_IN_THE_PAST = -(REND_CACHE_MAX_AGE + \
                                       REND_CACHE_MAX_SKEW + 60);
@@ -369,13 +367,12 @@ test_rend_cache_store_v2_desc_as_client_with_different_time(void *data)
   rend_data_free(mock_rend_query);
 }
 
-#define NS_SUBMODULE lookup_v2_desc_as_dir
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t *rcache_lookup_v2_as_dir_get_my_routerinfo(void);
 
 static routerinfo_t *mock_routerinfo;
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+rcache_lookup_v2_as_dir_get_my_routerinfo(void)
 {
   if (!mock_routerinfo) {
     mock_routerinfo = tor_malloc(sizeof(routerinfo_t));
@@ -395,7 +392,8 @@ test_rend_cache_lookup_v2_desc_as_dir(void *data)
 
   (void)data;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_lookup_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -418,20 +416,17 @@ test_rend_cache_lookup_v2_desc_as_dir(void *data)
   tt_assert(ret_desc);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   tor_free(mock_routerinfo);
   rend_cache_free_all();
   rend_encoded_v2_service_descriptor_free(desc_holder);
   tor_free(service_id);
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE store_v2_desc_as_dir
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t *rcache_store_v2_as_dir_get_my_routerinfo(void);
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+rcache_store_v2_as_dir_get_my_routerinfo(void)
 {
   return mock_routerinfo;
 }
@@ -444,7 +439,8 @@ test_rend_cache_store_v2_desc_as_dir(void *data)
   rend_encoded_v2_service_descriptor_t *desc_holder = NULL;
   char *service_id = NULL;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_store_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -485,7 +481,7 @@ test_rend_cache_store_v2_desc_as_dir(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   rend_encoded_v2_service_descriptor_free(desc_holder);
   tor_free(service_id);
   rend_cache_free_all();
@@ -505,7 +501,8 @@ test_rend_cache_store_v2_desc_as_dir_with_different_time(void *data)
   rend_encoded_v2_service_descriptor_t *desc_holder_newer;
   rend_encoded_v2_service_descriptor_t *desc_holder_older;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_store_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -543,7 +540,7 @@ test_rend_cache_store_v2_desc_as_dir_with_different_time(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   rend_cache_free_all();
   rend_service_descriptor_free(generated);
   tor_free(service_id);
@@ -568,7 +565,8 @@ test_rend_cache_store_v2_desc_as_dir_with_different_content(void *data)
   rend_encoded_v2_service_descriptor_t *desc_holder_one = NULL;
   rend_encoded_v2_service_descriptor_t *desc_holder_two = NULL;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_store_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -602,7 +600,7 @@ test_rend_cache_store_v2_desc_as_dir_with_different_content(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   rend_cache_free_all();
   rend_service_descriptor_free(generated);
   tor_free(service_id);
@@ -613,8 +611,6 @@ test_rend_cache_store_v2_desc_as_dir_with_different_content(void *data)
   rend_encoded_v2_service_descriptor_free(desc_holder_two);
 }
 
-#undef NS_SUBMODULE
-
 static void
 test_rend_cache_init(void *data)
 {
@@ -1077,8 +1073,6 @@ test_rend_cache_intro_failure_note(void *data)
   rend_cache_free_all();
 }
 
-#define NS_SUBMODULE clean_v2_descs_as_dir
-
 static void
 test_rend_cache_clean_v2_descs_as_dir(void *data)
 {
@@ -1120,8 +1114,6 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   rend_cache_free_all();
 }
 
-#undef NS_SUBMODULE
-
 static void
 test_rend_cache_entry_allocation(void *data)
 {
diff --git a/src/test/test_replay.c b/src/test/test_replay.c
index 28a508bf4d..1487b0a29d 100644
--- a/src/test/test_replay.c
+++ b/src/test/test_replay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define REPLAYCACHE_PRIVATE
diff --git a/src/test/test_rng.c b/src/test/test_rng.c
new file mode 100644
index 0000000000..ebaffb74f5
--- /dev/null
+++ b/src/test/test_rng.c
@@ -0,0 +1,59 @@
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Example usage:
+ *
+ * ./src/test/test-rng --emit | dieharder -g 200 -a
+ *
+ * Remember, dieharder can tell you that your RNG is completely broken, but if
+ * your RNG is not _completely_ broken, dieharder cannot tell you whether your
+ * RNG is actually secure.
+ */
+
+#include "orconfig.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+int
+main(int argc, char **argv)
+{
+  uint8_t buf[0x123];
+
+  if (argc != 2 || strcmp(argv[1], "--emit")) {
+    fprintf(stderr, "If you want me to fill stdout with a bunch of random "
+            "bytes, you need to say --emit.\n");
+    return 1;
+  }
+
+  if (crypto_seed_rng() < 0) {
+    fprintf(stderr, "Can't seed RNG.\n");
+    return 1;
+  }
+
+#if 0
+  while (1) {
+    crypto_rand(buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+#endif /* 0 */
+
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  while (1) {
+    crypto_fast_rng_getbytes(rng, buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+}
diff --git a/src/test/test_router.c b/src/test/test_router.c
index e0d3adfdbd..cf0c2b3dd1 100644
--- a/src/test/test_router.c
+++ b/src/test/test_router.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* Copyright (c) 2017, isis agora lovecruft  */
 /* See LICENSE for licensing information     */
 
@@ -9,13 +9,20 @@
 
 #define CONFIG_PRIVATE
 #define CONNECTION_PRIVATE
+#define ROUTER_PRIVATE
+
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/connection.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerstatus_st.h"
 #include "feature/relay/router.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
@@ -28,12 +35,13 @@
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t * rtr_tests_router_get_my_routerinfo(void);
+ATTR_UNUSED static int rtr_tests_router_get_my_routerinfo_called = 0;
 
 static routerinfo_t* mock_routerinfo;
 
 static const routerinfo_t*
-NS(router_get_my_routerinfo)(void)
+rtr_tests_router_get_my_routerinfo(void)
 {
   crypto_pk_t* ident_key;
   crypto_pk_t* tap_key;
@@ -83,20 +91,24 @@ test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
   char* found = NULL;
   (void)arg;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rtr_tests_router_get_my_routerinfo);
 
   options->ORPort_set = 1;
   options->BridgeRelay = 1;
 
   /* Generate keys which router_dump_router_to_string() expects to exist. */
-  tt_int_op(0, ==, curve25519_keypair_generate(&ntor_keypair, 0));
-  tt_int_op(0, ==, ed25519_keypair_generate(&signing_keypair, 0));
+  tt_int_op(0, OP_EQ, curve25519_keypair_generate(&ntor_keypair, 0));
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&signing_keypair, 0));
 
   /* Set up part of our routerinfo_t so that we don't trigger any other
    * assertions in router_dump_router_to_string(). */
   router = (routerinfo_t*)router_get_my_routerinfo();
-  tt_ptr_op(router, !=, NULL);
+  tt_ptr_op(router, OP_NE, NULL);
 
+  /* The real router_get_my_routerinfo() looks up onion_curve25519_pkey using
+   * get_current_curve25519_keypair(), but we don't initialise static data in
+   * this test. */
   router->onion_curve25519_pkey = &ntor_keypair.pubkey;
 
   /* Generate our server descriptor and ensure that the substring
@@ -109,12 +121,12 @@ test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
                                       &ntor_keypair,
                                       &signing_keypair);
   crypto_pk_free(onion_pkey);
-  tt_ptr_op(desc, !=, NULL);
+  tt_ptr_op(desc, OP_NE, NULL);
   found = strstr(desc, needle);
-  tt_ptr_op(found, !=, NULL);
+  tt_ptr_op(found, OP_NE, NULL);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
 
   tor_free(desc);
 }
@@ -237,6 +249,247 @@ test_router_check_descriptor_bandwidth_changed(void *arg)
   UNMOCK(we_are_hibernating);
 }
 
+static networkstatus_t *mock_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void)now;
+  return mock_ns;
+}
+
+static routerstatus_t *mock_rs = NULL;
+static const routerstatus_t *
+mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+{
+  (void)ns;
+  (void)digest;
+  return mock_rs;
+}
+
+static void
+test_router_mark_if_too_old(void *arg)
+{
+  (void)arg;
+  time_t now = approx_time();
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+  MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry);
+
+  routerstatus_t rs;
+  networkstatus_t ns;
+  memset(&rs, 0, sizeof(rs));
+  memset(&ns, 0, sizeof(ns));
+  mock_ns = &ns;
+  mock_ns->valid_after = now-3600;
+  mock_rs = &rs;
+  mock_rs->published_on = now - 10;
+
+  // no reason to mark this time.
+  desc_clean_since = now-10;
+  desc_dirty_reason = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+
+  // Doesn't appear in consensus?  Still don't mark it.
+  mock_ns = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+  mock_ns = &ns;
+
+  // No new descriptor in a long time?  Mark it.
+  desc_clean_since = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor");
+
+  // Version in consensus published a long time ago?  We won't mark it
+  // if it's been clean for only a short time.
+  desc_clean_since = now - 10;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now - 10);
+
+  // ... but if it's been clean a while, we mark.
+  desc_clean_since = now - 2 * 3600;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "version listed in consensus is quite old");
+
+  // same deal if we're marked stale.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 10;
+  mock_rs->is_staledesc = 1;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "listed as stale in consensus");
+
+  // same deal if we're absent from the consensus.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "not listed in consensus");
+
+ done:
+  UNMOCK(networkstatus_get_live_consensus);
+  UNMOCK(networkstatus_vote_find_entry);
+}
+
+static node_t fake_node;
+static const node_t *
+mock_node_get_by_nickname(const char *name, unsigned flags)
+{
+  (void)flags;
+  if (!strcasecmp(name, "crumpet"))
+    return &fake_node;
+  else
+    return NULL;
+}
+
+static void
+test_router_get_my_family(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  smartlist_t *sl = NULL;
+  char *join = NULL;
+  // Overwrite the result of router_get_my_identity_digest().  This
+  // happens to be okay, but only for testing.
+  set_server_identity_key_digest_testing(
+                                   (const uint8_t*)"holeinthebottomofthe");
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // No family listed -- so there's no list.
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+#define CLEAR() do {                                    \
+    if (sl) {                                           \
+      SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));  \
+      smartlist_free(sl);                               \
+    }                                                   \
+    tor_free(join);                                     \
+    mock_clean_saved_logs();                            \
+  } while (0)
+
+  // Add a single nice friendly hex member.  This should be enough
+  // to have our own ID added.
+  tt_ptr_op(options->MyFamily, OP_EQ, NULL);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 2);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Add a hex member with a ~.  The ~ part should get removed.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$0123456789abcdef0123456789abcdef01234567~Muffin");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Nickname lookup will fail, so a nickname will appear verbatim.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "BAGEL");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  expect_single_log_msg_containing(
+           "There is a router named \"BAGEL\" in my declared family, but "
+           "I have no descriptor for it.");
+  CLEAR();
+
+  // A bogus digest should fail entirely.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$painauchocolat");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.
+  expect_single_log_msg_containing(
+           "There is a router named \"$painauchocolat\" in my declared "
+           "family, but that isn't a legal digest or nickname. Skipping it.");
+  CLEAR();
+
+  // Let's introduce a node we can look up by nickname
+  memset(&fake_node, 0, sizeof(fake_node));
+  memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN);
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "CRUmpeT");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 5);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.  Some with "$painauchocolat".
+  expect_single_log_msg_containing(
+           "There is a router named \"CRUmpeT\" in my declared "
+           "family, but it wasn't listed by digest. Please consider saying "
+           "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's "
+           "what you meant.");
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+  // Try a singleton list containing only us: It should give us NULL.
+  config_free_lines(options->MyFamily);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$686F6C65696E746865626F74746F6D6F66746865");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+ done:
+  or_options_free(options);
+  teardown_capture_of_logs();
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+#undef CLEAR
+}
+
 static smartlist_t *fake_connection_array = NULL;
 static smartlist_t *
 mock_get_connection_array(void)
@@ -354,6 +607,8 @@ test_router_get_advertised_or_port_localhost(void *arg)
 struct testcase_t router_tests[] = {
   ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK),
   ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK),
+  ROUTER_TEST(mark_if_too_old, TT_FORK),
+  ROUTER_TEST(get_my_family, TT_FORK),
   ROUTER_TEST(get_advertised_or_port, TT_FORK),
   ROUTER_TEST(get_advertised_or_port_localhost, TT_FORK),
   END_OF_TESTCASES
diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c
index 727fa5660f..fc437dccc0 100644
--- a/src/test/test_routerkeys.c
+++ b/src/test/test_routerkeys.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -399,7 +399,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -409,7 +409,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -455,11 +455,11 @@ test_routerkeys_ed_keys_init_all(void *arg)
   options->TestingLinkKeySlop = 2*3600;
 
 #ifdef _WIN32
-  mkdir(dir);
-  mkdir(keydir);
+  tt_int_op(0, OP_EQ, mkdir(dir));
+  tt_int_op(0, OP_EQ, mkdir(keydir));
 #else
-  mkdir(dir, 0700);
-  mkdir(keydir, 0700);
+  tt_int_op(0, OP_EQ, mkdir(dir, 0700));
+  tt_int_op(0, OP_EQ, mkdir(keydir, 0700));
 #endif /* defined(_WIN32) */
 
   options->DataDirectory = dir;
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 95c9176faa..f2a83c18a3 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -46,7 +46,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "lib/encoding/confline.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "test/test.h"
 #include "test/test_dir_common.h"
@@ -265,7 +265,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   /* Init SR subsystem. */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -275,7 +277,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   construct_consensus(&consensus_text_md, now);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string(consensus_text_md,
+                                                strlen(consensus_text_md),
+                                                NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -301,7 +305,6 @@ test_router_pick_directory_server_impl(void *arg)
   tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until
                                                       + 24*60*60));
   /* These times are outside the test validity period */
-  tt_assert(networkstatus_consensus_is_bootstrapping(now));
   tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60));
   tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60));
 
@@ -475,7 +478,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -626,7 +631,7 @@ mock_clock_skew_warning(const connection_t *conn, long apparent_skew,
   (void)conn;
   mock_apparent_skew = apparent_skew;
   tt_int_op(trusted, OP_EQ, 1);
-  tt_int_op(domain, OP_EQ, LD_GENERAL);
+  tt_i64_op(domain, OP_EQ, LD_GENERAL);
   tt_str_op(received, OP_EQ, "microdesc flavor consensus");
   tt_str_op(source, OP_EQ, "CONSENSUS");
  done:
@@ -648,7 +653,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -662,7 +669,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
   MOCK(clock_skew_warning, mock_clock_skew_warning);
   /* Caller will call teardown_capture_of_logs() */
   setup_capture_of_logs(LOG_WARN);
-  retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+  retval = networkstatus_set_current_consensus(consensus, strlen(consensus),
+                                               "microdesc", 0,
                                                NULL);
 
  done:
diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c
index c45f0e1595..892ac6e210 100644
--- a/src/test/test_routerset.c
+++ b/src/test/test_routerset.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ROUTERSET_PRIVATE
@@ -18,17 +18,13 @@
 
 #include "test/test.h"
 
-#define NS_MODULE routerset
-
-#define NS_SUBMODULE routerset_new
-
 /*
  * Functional (blackbox) test to determine that each member of the routerset
  * is non-NULL
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_new(void *arg)
 {
   routerset_t *rs;
   (void)arg;
@@ -46,15 +42,12 @@ NS(test_main)(void *arg)
     routerset_free(rs);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_get_countryname
-
 /*
  * Functional test to strip the braces from a "{xx}" country code string.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_countryname(void *arg)
 {
   const char *input;
   char *name;
@@ -91,257 +84,272 @@ NS(test_main)(void *arg)
     tor_free(name);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, geoip_not_loaded)
-
 /*
  * Structural (whitebox) test for routerset_refresh_counties, when the GeoIP DB
  * is not loaded.
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
+static int rset_refresh_geoip_not_loaded_geoip_is_loaded(sa_family_t family);
+static int rset_refresh_geoip_not_loaded_geoip_is_loaded_called = 0;
+static int rset_refresh_geoip_not_loaded_geoip_get_n_countries(void);
+static int rset_refresh_geoip_not_loaded_geoip_get_n_countries_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_geoip_not_loaded(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
+  MOCK(geoip_is_loaded,
+       rset_refresh_geoip_not_loaded_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_geoip_not_loaded_geoip_get_n_countries);
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_EQ, NULL);
   tt_int_op(set->n_countries, OP_EQ, 0);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 0);
+  tt_int_op(rset_refresh_geoip_not_loaded_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_geoip_not_loaded_geoip_get_n_countries_called,
+            OP_EQ, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_geoip_not_loaded_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_geoip_not_loaded_geoip_is_loaded_called++;
 
   return 0;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_geoip_not_loaded_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_geoip_not_loaded_geoip_get_n_countries_called++;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, no_countries)
-
 /*
  * Structural test for routerset_refresh_counties, when there are no countries.
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static int rset_refresh_no_countries_geoip_is_loaded(sa_family_t family);
+static int rset_refresh_no_countries_geoip_is_loaded_called = 0;
+static int rset_refresh_no_countries_geoip_get_n_countries(void);
+static int rset_refresh_no_countries_geoip_get_n_countries_called = 0;
+static country_t rset_refresh_no_countries_geoip_get_country(
+                                                    const char *country);
+static int rset_refresh_no_countries_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_no_countries(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_refresh_no_countries_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_no_countries_geoip_get_n_countries);
+  MOCK(geoip_get_country,
+       rset_refresh_no_countries_geoip_get_country);
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_NE, NULL);
   tt_int_op(set->n_countries, OP_EQ, 1);
   tt_int_op((unsigned int)(*set->countries), OP_EQ, 0);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country), OP_EQ, 0);
+  tt_int_op(rset_refresh_no_countries_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_no_countries_geoip_get_n_countries_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_no_countries_geoip_get_country_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
-    NS_UNMOCK(geoip_get_country);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_get_country);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_no_countries_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_no_countries_geoip_is_loaded_called++;
 
   return 1;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_no_countries_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_no_countries_geoip_get_n_countries_called++;
 
   return 1;
 }
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+rset_refresh_no_countries_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  rset_refresh_no_countries_geoip_get_country_called++;
 
   return 1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, one_valid_country)
-
 /*
  * Structural test for routerset_refresh_counties, with one valid country.
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static int rset_refresh_one_valid_country_geoip_is_loaded(sa_family_t family);
+static int rset_refresh_one_valid_country_geoip_is_loaded_called = 0;
+static int rset_refresh_one_valid_country_geoip_get_n_countries(void);
+static int rset_refresh_one_valid_country_geoip_get_n_countries_called = 0;
+static country_t rset_refresh_one_valid_country_geoip_get_country(
+                                                    const char *country);
+static int rset_refresh_one_valid_country_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_one_valid_country(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_refresh_one_valid_country_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_one_valid_country_geoip_get_n_countries);
+  MOCK(geoip_get_country,
+       rset_refresh_one_valid_country_geoip_get_country);
   smartlist_add(set->country_names, tor_strndup("foo", 3));
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_NE, NULL);
   tt_int_op(set->n_countries, OP_EQ, 2);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country), OP_EQ, 1);
+  tt_int_op(rset_refresh_one_valid_country_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_one_valid_country_geoip_get_n_countries_called,
+            OP_EQ, 1);
+  tt_int_op(rset_refresh_one_valid_country_geoip_get_country_called, OP_EQ, 1);
   tt_int_op((unsigned int)(*set->countries), OP_NE, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
-    NS_UNMOCK(geoip_get_country);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_get_country);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_one_valid_country_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_one_valid_country_geoip_is_loaded_called++;
 
   return 1;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_one_valid_country_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_one_valid_country_geoip_get_n_countries_called++;
 
   return 2;
 }
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+rset_refresh_one_valid_country_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  rset_refresh_one_valid_country_geoip_get_country_called++;
 
   return 1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, one_invalid_country)
-
 /*
  * Structural test for routerset_refresh_counties, with one invalid
  * country code..
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static int rset_refresh_one_invalid_country_geoip_is_loaded(
+                                               sa_family_t family);
+static int rset_refresh_one_invalid_country_geoip_is_loaded_called = 0;
+static int rset_refresh_one_invalid_country_geoip_get_n_countries(void);
+static int rset_refresh_one_invalid_country_geoip_get_n_countries_called = 0;
+static country_t rset_refresh_one_invalid_country_geoip_get_country(
+                                               const char *country);
+static int rset_refresh_one_invalid_country_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_one_invalid_country(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_refresh_one_invalid_country_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_one_invalid_country_geoip_get_n_countries);
+  MOCK(geoip_get_country,
+       rset_refresh_one_invalid_country_geoip_get_country);
   smartlist_add(set->country_names, tor_strndup("foo", 3));
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_NE, NULL);
   tt_int_op(set->n_countries, OP_EQ, 2);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country), OP_EQ, 1);
+  tt_int_op(rset_refresh_one_invalid_country_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_one_invalid_country_geoip_get_n_countries_called,
+            OP_EQ, 1);
+  tt_int_op(rset_refresh_one_invalid_country_geoip_get_country_called,
+            OP_EQ, 1);
   tt_int_op((unsigned int)(*set->countries), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
-    NS_UNMOCK(geoip_get_country);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_get_country);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_one_invalid_country_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_one_invalid_country_geoip_is_loaded_called++;
 
   return 1;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_one_invalid_country_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_one_invalid_country_geoip_get_n_countries_called++;
 
   return 2;
 }
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+rset_refresh_one_invalid_country_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  rset_refresh_one_invalid_country_geoip_get_country_called++;
 
   return -1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, malformed)
-
 /*
  * Functional test, with a malformed string to parse.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_malformed(void *arg)
 {
   routerset_t *set = routerset_new();
   const char *s = "_";
@@ -356,16 +364,13 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, valid_hexdigest)
-
 /*
  * Functional test for routerset_parse, that routerset_parse returns 0
  * on a valid hexdigest entry.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_valid_hexdigest(void *arg)
 {
   routerset_t *set;
   const char *s;
@@ -382,15 +387,12 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, valid_nickname)
-
 /*
  * Functional test for routerset_parse, when given a valid nickname as input.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_valid_nickname(void *arg)
 {
   routerset_t *set;
   const char *s;
@@ -407,15 +409,12 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, get_countryname)
-
 /*
  * Functional test for routerset_parse, when given a valid countryname.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_get_countryname(void *arg)
 {
   routerset_t *set;
   const char *s;
@@ -432,158 +431,158 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, policy_wildcard)
-
 /*
  * Structural test for routerset_parse, when given a valid wildcard policy.
  */
 
-NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
-    (const char *s, int assume_action, int *malformed_list));
+static addr_policy_t * rset_parse_policy_wildcard_parse_item_from_string(
+               const char *s, int assume_action, int *malformed_list);
+static int rset_parse_policy_wildcard_parse_item_from_string_called = 0;
 
-static addr_policy_t *NS(mock_addr_policy);
+static addr_policy_t *rset_parse_policy_wildcard_mock_addr_policy;
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_policy_wildcard(void *arg)
 {
   routerset_t *set;
   const char *s;
   int r;
   (void)arg;
 
-  NS_MOCK(router_parse_addr_policy_item_from_string);
-  NS(mock_addr_policy) = tor_malloc_zero(sizeof(addr_policy_t));
+  MOCK(router_parse_addr_policy_item_from_string,
+       rset_parse_policy_wildcard_parse_item_from_string);
+  rset_parse_policy_wildcard_mock_addr_policy =
+    tor_malloc_zero(sizeof(addr_policy_t));
 
   set = routerset_new();
   s = "*";
   r = routerset_parse(set, s, "");
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(smartlist_len(set->policies), OP_NE, 0);
-  tt_int_op(CALLED(router_parse_addr_policy_item_from_string), OP_EQ, 1);
+  tt_int_op(rset_parse_policy_wildcard_parse_item_from_string_called,
+            OP_EQ, 1);
 
   done:
     routerset_free(set);
 }
 
 addr_policy_t *
-NS(router_parse_addr_policy_item_from_string)(const char *s,
+rset_parse_policy_wildcard_parse_item_from_string(const char *s,
                                               int assume_action,
                                               int *malformed_list)
 {
   (void)s;
   (void)assume_action;
   (void)malformed_list;
-  CALLED(router_parse_addr_policy_item_from_string)++;
+  rset_parse_policy_wildcard_parse_item_from_string_called++;
 
-  return NS(mock_addr_policy);
+  return rset_parse_policy_wildcard_mock_addr_policy;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, policy_ipv4)
-
 /*
  * Structural test for routerset_parse, when given a valid IPv4 address
  * literal policy.
  */
 
-NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
-        (const char *s, int assume_action, int *bogus));
+static addr_policy_t * rset_parse_policy_ipv4_parse_item_from_string(
+              const char *s, int assume_action, int *bogus);
+static int rset_parse_policy_ipv4_parse_item_from_string_called = 0;
 
-static addr_policy_t *NS(mock_addr_policy);
+static addr_policy_t *rset_parse_policy_ipv4_mock_addr_policy;
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_policy_ipv4(void *arg)
 {
   routerset_t *set;
   const char *s;
   int r;
   (void)arg;
 
-  NS_MOCK(router_parse_addr_policy_item_from_string);
-  NS(mock_addr_policy) = tor_malloc_zero(sizeof(addr_policy_t));
+  MOCK(router_parse_addr_policy_item_from_string,
+       rset_parse_policy_ipv4_parse_item_from_string);
+  rset_parse_policy_ipv4_mock_addr_policy =
+    tor_malloc_zero(sizeof(addr_policy_t));
 
   set = routerset_new();
   s = "127.0.0.1";
   r = routerset_parse(set, s, "");
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(smartlist_len(set->policies), OP_NE, 0);
-  tt_int_op(CALLED(router_parse_addr_policy_item_from_string), OP_EQ, 1);
+  tt_int_op(rset_parse_policy_ipv4_parse_item_from_string_called, OP_EQ, 1);
 
  done:
   routerset_free(set);
 }
 
 addr_policy_t *
-NS(router_parse_addr_policy_item_from_string)(const char *s, int assume_action,
-                                              int *bogus)
+rset_parse_policy_ipv4_parse_item_from_string(
+                                      const char *s, int assume_action,
+                                      int *bogus)
 {
   (void)s;
   (void)assume_action;
-  CALLED(router_parse_addr_policy_item_from_string)++;
+  rset_parse_policy_ipv4_parse_item_from_string_called++;
   *bogus = 0;
 
-  return NS(mock_addr_policy);
+  return rset_parse_policy_ipv4_mock_addr_policy;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, policy_ipv6)
-
 /*
  * Structural test for routerset_parse, when given a valid IPv6 address
  * literal policy.
  */
 
-NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
-        (const char *s, int assume_action, int *bad));
+static addr_policy_t * rset_parse_policy_ipv6_parse_item_from_string(
+               const char *s, int assume_action, int *bad);
+static int rset_parse_policy_ipv6_parse_item_from_string_called = 0;
 
-static addr_policy_t *NS(mock_addr_policy);
+static addr_policy_t *rset_parse_policy_ipv6_mock_addr_policy;
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_policy_ipv6(void *arg)
 {
   routerset_t *set;
   const char *s;
   int r;
   (void)arg;
 
-  NS_MOCK(router_parse_addr_policy_item_from_string);
-  NS(mock_addr_policy) = tor_malloc_zero(sizeof(addr_policy_t));
+  MOCK(router_parse_addr_policy_item_from_string,
+       rset_parse_policy_ipv6_parse_item_from_string);
+  rset_parse_policy_ipv6_mock_addr_policy =
+    tor_malloc_zero(sizeof(addr_policy_t));
 
   set = routerset_new();
   s = "::1";
   r = routerset_parse(set, s, "");
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(smartlist_len(set->policies), OP_NE, 0);
-  tt_int_op(CALLED(router_parse_addr_policy_item_from_string), OP_EQ, 1);
+  tt_int_op(rset_parse_policy_ipv6_parse_item_from_string_called, OP_EQ, 1);
 
  done:
   routerset_free(set);
 }
 
 addr_policy_t *
-NS(router_parse_addr_policy_item_from_string)(const char *s,
+rset_parse_policy_ipv6_parse_item_from_string(const char *s,
                                               int assume_action, int *bad)
 {
   (void)s;
   (void)assume_action;
-  CALLED(router_parse_addr_policy_item_from_string)++;
+  rset_parse_policy_ipv6_parse_item_from_string_called++;
   *bad = 0;
 
-  return NS(mock_addr_policy);
+  return rset_parse_policy_ipv6_mock_addr_policy;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_union, source_bad)
-
 /*
  * Structural test for routerset_union, when given a bad source argument.
  */
 
-NS_DECL(smartlist_t *, smartlist_new, (void));
+static smartlist_t * rset_union_source_bad_smartlist_new(void);
+static int rset_union_source_bad_smartlist_new_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_union_source_bad(void *arg)
 {
   routerset_t *set, *bad_set;
   (void)arg;
@@ -593,16 +592,17 @@ NS(test_main)(void *arg)
   smartlist_free(bad_set->list);
   bad_set->list = NULL;
 
-  NS_MOCK(smartlist_new);
+  MOCK(smartlist_new,
+       rset_union_source_bad_smartlist_new);
 
   routerset_union(set, NULL);
-  tt_int_op(CALLED(smartlist_new), OP_EQ, 0);
+  tt_int_op(rset_union_source_bad_smartlist_new_called, OP_EQ, 0);
 
   routerset_union(set, bad_set);
-  tt_int_op(CALLED(smartlist_new), OP_EQ, 0);
+  tt_int_op(rset_union_source_bad_smartlist_new_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(smartlist_new);
+    UNMOCK(smartlist_new);
     routerset_free(set);
 
     /* Just recreate list, so we can simply use routerset_free. */
@@ -611,22 +611,19 @@ NS(test_main)(void *arg)
 }
 
 static smartlist_t *
-NS(smartlist_new)(void)
+rset_union_source_bad_smartlist_new(void)
 {
-  CALLED(smartlist_new)++;
+  rset_union_source_bad_smartlist_new_called++;
 
   return NULL;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_union, one)
-
 /*
  * Functional test for routerset_union.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_union_one(void *arg)
 {
   routerset_t *src = routerset_new();
   routerset_t *tgt;
@@ -643,15 +640,12 @@ NS(test_main)(void *arg)
     routerset_free(tgt);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_is_list
-
 /*
  * Functional tests for routerset_is_list.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_is_list(void *arg)
 {
   routerset_t *set;
   addr_policy_t *policy;
@@ -696,15 +690,12 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_needs_geoip
-
 /*
  * Functional tests for routerset_needs_geoip.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_needs_geoip(void *arg)
 {
   routerset_t *set;
   int needs_geoip;
@@ -731,15 +722,12 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_is_empty
-
 /*
  * Functional tests for routerset_is_empty.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_is_empty(void *arg)
 {
   routerset_t *set = NULL;
   int is_empty;
@@ -765,16 +753,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, null_set_or_null_set_list)
-
 /*
  * Functional test for routerset_contains, when given a NULL set or the
  * set has a NULL list.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_set_or_list(void *arg)
 {
   routerset_t *set = NULL;
   int contains;
@@ -794,16 +779,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_null_nickname)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset but a
  * NULL nickname.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_nickname(void *arg)
 {
   routerset_t *set = routerset_new();
   char *nickname = NULL;
@@ -819,16 +801,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_nickname)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the nickname is in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_nickname(void *arg)
 {
   routerset_t *set = routerset_new();
   const char *nickname;
@@ -845,16 +824,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_no_nickname)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the nickname is not in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_no_nickname(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -869,16 +845,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_digest)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the digest is contained in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_digest(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -894,16 +867,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_no_digest)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the digest is not contained in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_no_digest(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -920,16 +890,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_null_digest)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the digest is NULL.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_digest(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -945,34 +912,34 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_addr)
-
 /*
  * Structural test for routerset_contains, when given a valid routerset
  * and the address is rejected by policy.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
+static addr_policy_result_t rset_contains_addr_cmp_addr_to_policy(
+              const tor_addr_t *addr, uint16_t port,
+              const smartlist_t *policy);
+static int rset_contains_addr_cmp_addr_to_policy_called = 0;
 
 static tor_addr_t MOCK_TOR_ADDR;
 #define MOCK_TOR_ADDR_PTR (&MOCK_TOR_ADDR)
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_addr(void *arg)
 {
   routerset_t *set = routerset_new();
   tor_addr_t *addr = MOCK_TOR_ADDR_PTR;
   int contains;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_contains_addr_cmp_addr_to_policy);
 
   contains = routerset_contains(set, addr, 0, NULL, NULL, 0);
   routerset_free(set);
 
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
+  tt_int_op(rset_contains_addr_cmp_addr_to_policy_called, OP_EQ, 1);
   tt_int_op(contains, OP_EQ, 3);
 
   done:
@@ -980,12 +947,12 @@ NS(test_main)(void *arg)
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_contains_addr_cmp_addr_to_policy(const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_contains_addr_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
   return ADDR_POLICY_REJECTED;
 
@@ -993,31 +960,31 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_no_addr)
-
 /*
  * Structural test for routerset_contains, when given a valid routerset
  * and the address is not rejected by policy.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
+static addr_policy_result_t rset_contains_no_addr_cmp_addr_to_policy(
+              const tor_addr_t *addr, uint16_t port,
+              const smartlist_t *policy);
+static int rset_contains_no_addr_cmp_addr_to_policy_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_no_addr(void *arg)
 {
   routerset_t *set = routerset_new();
   tor_addr_t *addr = MOCK_TOR_ADDR_PTR;
   int contains;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_contains_no_addr_cmp_addr_to_policy);
 
   contains = routerset_contains(set, addr, 0, NULL, NULL, 0);
   routerset_free(set);
 
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
+  tt_int_op(rset_contains_no_addr_cmp_addr_to_policy_called, OP_EQ, 1);
   tt_int_op(contains, OP_EQ, 0);
 
   done:
@@ -1025,12 +992,12 @@ NS(test_main)(void *arg)
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_contains_no_addr_cmp_addr_to_policy(const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_contains_no_addr_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   return ADDR_POLICY_ACCEPTED;
@@ -1039,25 +1006,25 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
       return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_null_addr)
-
 /*
  * Structural test for routerset_contains, when given a valid routerset
  * and the address is NULL.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
+static addr_policy_result_t rset_contains_null_addr_cmp_addr_to_policy(
+                 const tor_addr_t *addr, uint16_t port,
+                 const smartlist_t *policy);
+static int rset_contains_null_addr_cmp_addr_to_policy_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_addr(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_contains_null_addr_cmp_addr_to_policy);
 
   contains = routerset_contains(set, NULL, 0, NULL, NULL, 0);
   routerset_free(set);
@@ -1069,12 +1036,13 @@ NS(test_main)(void *arg)
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_contains_null_addr_cmp_addr_to_policy(
+    const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_contains_null_addr_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   return ADDR_POLICY_ACCEPTED;
@@ -1083,27 +1051,30 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, countries_no_geoip)
-
 /*
  * Structural test for routerset_contains, when there is no matching country
  * for the address.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
-NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
+static addr_policy_result_t rset_countries_no_geoip_cmp_addr_to_policy(
+                  const tor_addr_t *addr, uint16_t port,
+                  const smartlist_t *policy);
+static int rset_countries_no_geoip_cmp_addr_to_policy_called = 0;
+static int rset_countries_no_geoip_geoip_get_country_by_addr(
+                  const tor_addr_t *addr);
+static int rset_countries_no_geoip_geoip_get_country_by_addr_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_countries_no_geoip(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains = 1;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
-  NS_MOCK(geoip_get_country_by_addr);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_countries_no_geoip_cmp_addr_to_policy);
+  MOCK(geoip_get_country_by_addr,
+       rset_countries_no_geoip_geoip_get_country_by_addr);
 
   set->countries = bitarray_init_zero(1);
   bitarray_set(set->countries, 1);
@@ -1111,20 +1082,23 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(contains, OP_EQ, 0);
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country_by_addr), OP_EQ, 1);
+  tt_int_op(rset_countries_no_geoip_cmp_addr_to_policy_called,
+            OP_EQ, 1);
+  tt_int_op(rset_countries_no_geoip_geoip_get_country_by_addr_called,
+            OP_EQ, 1);
 
   done:
     ;
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_countries_no_geoip_cmp_addr_to_policy(
+    const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_countries_no_geoip_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
@@ -1132,36 +1106,39 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
 }
 
 int
-NS(geoip_get_country_by_addr)(const tor_addr_t *addr)
+rset_countries_no_geoip_geoip_get_country_by_addr(const tor_addr_t *addr)
 {
-  CALLED(geoip_get_country_by_addr)++;
+  rset_countries_no_geoip_geoip_get_country_by_addr_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
     return -1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, countries_geoip)
-
 /*
  * Structural test for routerset_contains, when there a matching country
  * for the address.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
-NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
+static addr_policy_result_t rset_countries_geoip_cmp_addr_to_policy(
+              const tor_addr_t *addr, uint16_t port,
+              const smartlist_t *policy);
+static int rset_countries_geoip_cmp_addr_to_policy_called = 0;
+static int rset_countries_geoip_geoip_get_country_by_addr(
+                                          const tor_addr_t *addr);
+static int rset_countries_geoip_geoip_get_country_by_addr_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_countries_geoip(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains = 1;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
-  NS_MOCK(geoip_get_country_by_addr);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_countries_geoip_cmp_addr_to_policy);
+  MOCK(geoip_get_country_by_addr,
+       rset_countries_geoip_geoip_get_country_by_addr);
 
   set->n_countries = 2;
   set->countries = bitarray_init_zero(1);
@@ -1170,20 +1147,24 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(contains, OP_EQ, 2);
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country_by_addr), OP_EQ, 1);
+  tt_int_op(
+     rset_countries_geoip_cmp_addr_to_policy_called,
+     OP_EQ, 1);
+  tt_int_op(rset_countries_geoip_geoip_get_country_by_addr_called,
+            OP_EQ, 1);
 
   done:
     ;
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_countries_geoip_cmp_addr_to_policy(
+    const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_countries_geoip_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
@@ -1191,25 +1172,22 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
 }
 
 int
-NS(geoip_get_country_by_addr)(const tor_addr_t *addr)
+rset_countries_geoip_geoip_get_country_by_addr(const tor_addr_t *addr)
 {
-  CALLED(geoip_get_country_by_addr)++;
+  rset_countries_geoip_geoip_get_country_by_addr_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
     return 1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, only_flag_and_no_ccs)
-
 /*
  * Functional test for routerset_add_unknown_ccs, where only_if_some_cc_set
  * is set and there are no country names.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_only_flag(void *arg)
 {
   routerset_t *set = routerset_new();
   routerset_t **setp = &set;
@@ -1224,26 +1202,26 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, creates_set)
-
 /*
  * Functional test for routerset_add_unknown_ccs, where the set argument
  * is created if passed in as NULL.
  */
 
 /* The mock is only used to stop the test from asserting erroneously. */
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static country_t rset_add_unknown_ccs_creates_set_geoip_get_country(
+                                                     const char *country);
+static int rset_add_unknown_ccs_creates_set_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_creates_set(void *arg)
 {
   routerset_t *set = NULL;
   routerset_t **setp = &set;
   int r;
   (void)arg;
 
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       rset_add_unknown_ccs_creates_set_geoip_get_country);
 
   r = routerset_add_unknown_ccs(setp, 0);
 
@@ -1256,35 +1234,38 @@ NS(test_main)(void *arg)
 }
 
 country_t
-NS(geoip_get_country)(const char *country)
+rset_add_unknown_ccs_creates_set_geoip_get_country(const char *country)
 {
   (void)country;
-  CALLED(geoip_get_country)++;
+  rset_add_unknown_ccs_creates_set_geoip_get_country_called++;
 
   return -1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, add_unknown)
-
 /*
  * Structural test for routerset_add_unknown_ccs, that the "{??}"
  * country code is added to the list.
  */
 
-NS_DECL(country_t, geoip_get_country, (const char *country));
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
+static country_t rset_add_unknown_ccs_add_unknown_geoip_get_country(
+                                                 const char *country);
+static int rset_add_unknown_ccs_add_unknown_geoip_get_country_called = 0;
+static int rset_add_unknown_ccs_add_unknown_geoip_is_loaded(
+                                                 sa_family_t family);
+static int rset_add_unknown_ccs_add_unknown_geoip_is_loaded_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_add_unknown(void *arg)
 {
   routerset_t *set = routerset_new();
   routerset_t **setp = &set;
   int r;
   (void)arg;
 
-  NS_MOCK(geoip_get_country);
-  NS_MOCK(geoip_is_loaded);
+  MOCK(geoip_get_country,
+       rset_add_unknown_ccs_add_unknown_geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_add_unknown_ccs_add_unknown_geoip_is_loaded);
 
   r = routerset_add_unknown_ccs(setp, 0);
 
@@ -1298,11 +1279,11 @@ NS(test_main)(void *arg)
 }
 
 country_t
-NS(geoip_get_country)(const char *country)
+rset_add_unknown_ccs_add_unknown_geoip_get_country(const char *country)
 {
   int arg_is_qq, arg_is_a1;
 
-  CALLED(geoip_get_country)++;
+  rset_add_unknown_ccs_add_unknown_geoip_get_country_called++;
 
   arg_is_qq = !strcmp(country, "??");
   arg_is_a1 = !strcmp(country, "A1");
@@ -1317,9 +1298,9 @@ NS(geoip_get_country)(const char *country)
 }
 
 int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_add_unknown_ccs_add_unknown_geoip_is_loaded(sa_family_t family)
 {
-  CALLED(geoip_is_loaded)++;
+  rset_add_unknown_ccs_add_unknown_geoip_is_loaded_called++;
 
   tt_int_op(family, OP_EQ, AF_INET);
 
@@ -1327,27 +1308,29 @@ NS(geoip_is_loaded)(sa_family_t family)
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, add_a1)
-
 /*
  * Structural test for routerset_add_unknown_ccs, that the "{a1}"
  * country code is added to the list.
  */
 
-NS_DECL(country_t, geoip_get_country, (const char *country));
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
+static country_t rset_add_unknown_ccs_add_a1_geoip_get_country(
+                                           const char *country);
+static int rset_add_unknown_ccs_add_a1_geoip_get_country_called = 0;
+static int rset_add_unknown_ccs_add_a1_geoip_is_loaded(sa_family_t family);
+static int rset_add_unknown_ccs_add_a1_geoip_is_loaded_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_add_a1(void *arg)
 {
   routerset_t *set = routerset_new();
   routerset_t **setp = &set;
   int r;
   (void)arg;
 
-  NS_MOCK(geoip_get_country);
-  NS_MOCK(geoip_is_loaded);
+  MOCK(geoip_get_country,
+       rset_add_unknown_ccs_add_a1_geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_add_unknown_ccs_add_a1_geoip_is_loaded);
 
   r = routerset_add_unknown_ccs(setp, 0);
 
@@ -1361,11 +1344,11 @@ NS(test_main)(void *arg)
 }
 
 country_t
-NS(geoip_get_country)(const char *country)
+rset_add_unknown_ccs_add_a1_geoip_get_country(const char *country)
 {
   int arg_is_qq, arg_is_a1;
 
-  CALLED(geoip_get_country)++;
+  rset_add_unknown_ccs_add_a1_geoip_get_country_called++;
 
   arg_is_qq = !strcmp(country, "??");
   arg_is_a1 = !strcmp(country, "A1");
@@ -1380,9 +1363,9 @@ NS(geoip_get_country)(const char *country)
 }
 
 int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_add_unknown_ccs_add_a1_geoip_is_loaded(sa_family_t family)
 {
-  CALLED(geoip_is_loaded)++;
+  rset_add_unknown_ccs_add_a1_geoip_is_loaded_called++;
 
   tt_int_op(family, OP_EQ, AF_INET);
 
@@ -1390,15 +1373,12 @@ NS(geoip_is_loaded)(sa_family_t family)
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_contains_extendinfo
-
 /*
  * Functional test for routerset_contains_extendinfo.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_extendinfo(void *arg)
 {
   routerset_t *set = routerset_new();
   extend_info_t ei;
@@ -1418,15 +1398,12 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_contains_router
-
 /*
  * Functional test for routerset_contains_router.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_router(void *arg)
 {
   routerset_t *set = routerset_new();
   routerinfo_t ri;
@@ -1446,9 +1423,6 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_contains_routerstatus
-
 /*
  * Functional test for routerset_contains_routerstatus.
  */
@@ -1458,7 +1432,7 @@ NS(test_main)(void *arg)
 // a bit more or test a bit more.
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_routerstatus(void *arg)
 {
   routerset_t *set = routerset_new();
   routerstatus_t rs;
@@ -1479,46 +1453,41 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains_node, none)
-
 /*
  * Functional test for routerset_contains_node, when the node has no
  * routerset or routerinfo.
  */
 
-static node_t NS(mock_node);
+static node_t rset_contains_none_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_none(void *arg)
 {
   routerset_t *set = routerset_new();
   int r;
   (void)arg;
 
-  memset(&NS(mock_node), 0, sizeof(NS(mock_node)));
-  NS(mock_node).ri = NULL;
-  NS(mock_node).rs = NULL;
+  memset(&rset_contains_none_mock_node, 0,
+         sizeof(rset_contains_none_mock_node));
+  rset_contains_none_mock_node.ri = NULL;
+  rset_contains_none_mock_node.rs = NULL;
 
-  r = routerset_contains_node(set, &NS(mock_node));
+  r = routerset_contains_node(set, &rset_contains_none_mock_node);
   tt_int_op(r, OP_EQ, 0);
 
   done:
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains_node, routerstatus)
-
 /*
  * Functional test for routerset_contains_node, when the node has a
  * routerset and no routerinfo.
  */
 
-static node_t NS(mock_node);
+static node_t rset_contains_rs_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_rs(void *arg)
 {
   routerset_t *set = routerset_new();
   int r;
@@ -1530,27 +1499,24 @@ NS(test_main)(void *arg)
 
   strncpy(rs.nickname, nickname, sizeof(rs.nickname) - 1);
   rs.nickname[sizeof(rs.nickname) - 1] = '\0';
-  memset(&NS(mock_node), 0, sizeof(NS(mock_node)));
-  NS(mock_node).ri = NULL;
-  NS(mock_node).rs = &rs;
+  memset(&rset_contains_rs_mock_node, 0, sizeof(rset_contains_rs_mock_node));
+  rset_contains_rs_mock_node.ri = NULL;
+  rset_contains_rs_mock_node.rs = &rs;
 
-  r = routerset_contains_node(set, &NS(mock_node));
+  r = routerset_contains_node(set, &rset_contains_rs_mock_node);
 
   tt_int_op(r, OP_EQ, 4);
   done:
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains_node, routerinfo)
-
 /*
  * Functional test for routerset_contains_node, when the node has no
  * routerset and a routerinfo.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_routerinfo(void *arg)
 {
   routerset_t *set = routerset_new();
   int r;
@@ -1573,16 +1539,13 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, no_routerset)
-
 /*
  * Functional test for routerset_get_all_nodes, when routerset is NULL or
  * the routerset list is NULL.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_no_routerset(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = NULL;
@@ -1606,30 +1569,29 @@ NS(test_main)(void *arg)
   smartlist_free(out);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, list_with_no_nodes)
-
 /*
  * Structural test for routerset_get_all_nodes, when the routerset list
  * is empty.
  */
 
-NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, unsigned flags));
-static const char *NS(mock_nickname);
+static const node_t * rset_get_all_l_no_nodes_node_get_by_nickname(
+                                       const char *nickname, unsigned flags);
+static int rset_get_all_l_no_nodes_node_get_by_nickname_called = 0;
+static const char *rset_get_all_l_no_nodes_mock_nickname;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_l_no_nodes(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = routerset_new();
   int out_len;
   (void)arg;
 
-  NS_MOCK(node_get_by_nickname);
+  MOCK(node_get_by_nickname,
+       rset_get_all_l_no_nodes_node_get_by_nickname);
 
-  NS(mock_nickname) = "foo";
-  smartlist_add_strdup(set->list, NS(mock_nickname));
+  rset_get_all_l_no_nodes_mock_nickname = "foo";
+  smartlist_add_strdup(set->list, rset_get_all_l_no_nodes_mock_nickname);
 
   routerset_get_all_nodes(out, set, NULL, 0);
   out_len = smartlist_len(out);
@@ -1638,49 +1600,49 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(out_len, OP_EQ, 0);
-  tt_int_op(CALLED(node_get_by_nickname), OP_EQ, 1);
+  tt_int_op(rset_get_all_l_no_nodes_node_get_by_nickname_called, OP_EQ, 1);
 
   done:
     ;
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, unsigned flags)
+rset_get_all_l_no_nodes_node_get_by_nickname(const char *nickname,
+                                             unsigned flags)
 {
-  CALLED(node_get_by_nickname)++;
-  tt_str_op(nickname, OP_EQ, NS(mock_nickname));
+  rset_get_all_l_no_nodes_node_get_by_nickname_called++;
+  tt_str_op(nickname, OP_EQ, rset_get_all_l_no_nodes_mock_nickname);
   tt_uint_op(flags, OP_EQ, 0);
 
   done:
     return NULL;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, list_flag_not_running)
-
 /*
  * Structural test for routerset_get_all_nodes, with the running_only flag
  * is set but the nodes are not running.
  */
 
-NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, unsigned flags));
-static const char *NS(mock_nickname);
-static node_t NS(mock_node);
+static const node_t * rset_get_all_l_not_running_node_get_by_nickname(
+                                       const char *nickname, unsigned flags);
+static int rset_get_all_l_not_running_node_get_by_nickname_called = 0;
+static const char *rset_get_all_l_not_running_mock_nickname;
+static node_t rset_get_all_l_not_running_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_l_not_running(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = routerset_new();
   int out_len;
   (void)arg;
 
-  NS_MOCK(node_get_by_nickname);
+  MOCK(node_get_by_nickname,
+       rset_get_all_l_not_running_node_get_by_nickname);
 
-  NS(mock_node).is_running = 0;
-  NS(mock_nickname) = "foo";
-  smartlist_add_strdup(set->list, NS(mock_nickname));
+  rset_get_all_l_not_running_mock_node.is_running = 0;
+  rset_get_all_l_not_running_mock_nickname = "foo";
+  smartlist_add_strdup(set->list, rset_get_all_l_not_running_mock_nickname);
 
   routerset_get_all_nodes(out, set, NULL, 1);
   out_len = smartlist_len(out);
@@ -1689,37 +1651,36 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(out_len, OP_EQ, 0);
-  tt_int_op(CALLED(node_get_by_nickname), OP_EQ, 1);
+  tt_int_op(rset_get_all_l_not_running_node_get_by_nickname_called, OP_EQ, 1);
 
   done:
     ;
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, unsigned flags)
+rset_get_all_l_not_running_node_get_by_nickname(const char *nickname,
+                                                unsigned flags)
 {
-  CALLED(node_get_by_nickname)++;
-  tt_str_op(nickname, OP_EQ, NS(mock_nickname));
+  rset_get_all_l_not_running_node_get_by_nickname_called++;
+  tt_str_op(nickname, OP_EQ, rset_get_all_l_not_running_mock_nickname);
   tt_int_op(flags, OP_EQ, 0);
 
   done:
-    return &NS(mock_node);
+    return &rset_get_all_l_not_running_mock_node;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, list)
-
 /*
  * Structural test for routerset_get_all_nodes.
  */
 
-NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, unsigned flags));
-static char *NS(mock_nickname);
-static node_t NS(mock_node);
+static const node_t * rset_get_all_list_node_get_by_nickname(
+                             const char *nickname, unsigned flags);
+static int rset_get_all_list_node_get_by_nickname_called = 0;
+static char *rset_get_all_list_mock_nickname;
+static node_t rset_get_all_list_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_list(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = routerset_new();
@@ -1727,10 +1688,11 @@ NS(test_main)(void *arg)
   node_t *ent;
   (void)arg;
 
-  NS_MOCK(node_get_by_nickname);
+  MOCK(node_get_by_nickname,
+       rset_get_all_list_node_get_by_nickname);
 
-  NS(mock_nickname) = tor_strdup("foo");
-  smartlist_add(set->list, NS(mock_nickname));
+  rset_get_all_list_mock_nickname = tor_strdup("foo");
+  smartlist_add(set->list, rset_get_all_list_mock_nickname);
 
   routerset_get_all_nodes(out, set, NULL, 0);
   out_len = smartlist_len(out);
@@ -1740,127 +1702,122 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(out_len, OP_EQ, 1);
-  tt_ptr_op(ent, OP_EQ, &NS(mock_node));
-  tt_int_op(CALLED(node_get_by_nickname), OP_EQ, 1);
+  tt_ptr_op(ent, OP_EQ, &rset_get_all_list_mock_node);
+  tt_int_op(rset_get_all_list_node_get_by_nickname_called, OP_EQ, 1);
 
   done:
     ;
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, unsigned flags)
+rset_get_all_list_node_get_by_nickname(const char *nickname, unsigned flags)
 {
-  CALLED(node_get_by_nickname)++;
-  tt_str_op(nickname, OP_EQ, NS(mock_nickname));
+  rset_get_all_list_node_get_by_nickname_called++;
+  tt_str_op(nickname, OP_EQ, rset_get_all_list_mock_nickname);
   tt_int_op(flags, OP_EQ, 0);
 
   done:
-    return &NS(mock_node);
+    return &rset_get_all_list_mock_node;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, nodelist_with_no_nodes)
-
 /*
  * Structural test for routerset_get_all_nodes, when the nodelist has no nodes.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
-
-static smartlist_t *NS(mock_smartlist);
+static const smartlist_t * rset_get_all_n_no_nodes_nodelist_get_list(void);
+static int rset_get_all_n_no_nodes_nodelist_get_list_called = 0;
 
+static smartlist_t *rset_get_all_n_no_nodes_mock_smartlist;
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_n_no_nodes(void *arg)
 {
   routerset_t *set = routerset_new();
   smartlist_t *out = smartlist_new();
   int r;
   (void)arg;
 
-  NS_MOCK(nodelist_get_list);
+  MOCK(nodelist_get_list,
+       rset_get_all_n_no_nodes_nodelist_get_list);
 
   smartlist_add_strdup(set->country_names, "{xx}");
-  NS(mock_smartlist) = smartlist_new();
+  rset_get_all_n_no_nodes_mock_smartlist = smartlist_new();
 
   routerset_get_all_nodes(out, set, NULL, 1);
   r = smartlist_len(out);
   routerset_free(set);
   smartlist_free(out);
-  smartlist_free(NS(mock_smartlist));
+  smartlist_free(rset_get_all_n_no_nodes_mock_smartlist);
 
   tt_int_op(r, OP_EQ, 0);
-  tt_int_op(CALLED(nodelist_get_list), OP_EQ, 1);
+  tt_int_op(rset_get_all_n_no_nodes_nodelist_get_list_called, OP_EQ, 1);
 
   done:
     ;
 }
 
-smartlist_t *
-NS(nodelist_get_list)(void)
+const smartlist_t *
+rset_get_all_n_no_nodes_nodelist_get_list(void)
 {
-  CALLED(nodelist_get_list)++;
+  rset_get_all_n_no_nodes_nodelist_get_list_called++;
 
-  return NS(mock_smartlist);
+  return rset_get_all_n_no_nodes_mock_smartlist;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, nodelist_flag_not_running)
-
 /*
  * Structural test for routerset_get_all_nodes, with a non-list routerset
  * the running_only flag is set, but the nodes are not running.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
+static const smartlist_t * rset_get_all_n_not_running_nodelist_get_list(void);
+static int rset_get_all_n_not_running_nodelist_get_list_called = 0;
 
-static smartlist_t *NS(mock_smartlist);
-static node_t NS(mock_node);
+static smartlist_t *rset_get_all_n_not_running_mock_smartlist;
+static node_t rset_get_all_n_not_running_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_n_not_running(void *arg)
 {
   routerset_t *set = routerset_new();
   smartlist_t *out = smartlist_new();
   int r;
   (void)arg;
 
-  NS_MOCK(nodelist_get_list);
+  MOCK(nodelist_get_list,
+       rset_get_all_n_not_running_nodelist_get_list);
 
   smartlist_add_strdup(set->country_names, "{xx}");
-  NS(mock_smartlist) = smartlist_new();
-  NS(mock_node).is_running = 0;
-  smartlist_add(NS(mock_smartlist), (void *)&NS(mock_node));
+  rset_get_all_n_not_running_mock_smartlist = smartlist_new();
+  rset_get_all_n_not_running_mock_node.is_running = 0;
+  smartlist_add(rset_get_all_n_not_running_mock_smartlist,
+                (void *)&rset_get_all_n_not_running_mock_node);
 
   routerset_get_all_nodes(out, set, NULL, 1);
   r = smartlist_len(out);
   routerset_free(set);
   smartlist_free(out);
-  smartlist_free(NS(mock_smartlist));
+  smartlist_free(rset_get_all_n_not_running_mock_smartlist);
 
   tt_int_op(r, OP_EQ, 0);
-  tt_int_op(CALLED(nodelist_get_list), OP_EQ, 1);
+  tt_int_op(rset_get_all_n_not_running_nodelist_get_list_called, OP_EQ, 1);
 
   done:
     ;
 }
 
-smartlist_t *
-NS(nodelist_get_list)(void)
+const smartlist_t *
+rset_get_all_n_not_running_nodelist_get_list(void)
 {
-  CALLED(nodelist_get_list)++;
+  rset_get_all_n_not_running_nodelist_get_list_called++;
 
-  return NS(mock_smartlist);
+  return rset_get_all_n_not_running_mock_smartlist;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_subtract_nodes
-
 /*
  * Functional test for routerset_subtract_nodes.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_subtract_nodes(void *arg)
 {
   routerset_t *set = routerset_new();
   smartlist_t *list = smartlist_new();
@@ -1885,15 +1842,12 @@ NS(test_main)(void *arg)
     smartlist_free(list);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_subtract_nodes, null_routerset)
-
 /*
  * Functional test for routerset_subtract_nodes, with a NULL routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_subtract_nodes_null_routerset(void *arg)
 {
   routerset_t *set = NULL;
   smartlist_t *list = smartlist_new();
@@ -1915,15 +1869,12 @@ NS(test_main)(void *arg)
     smartlist_free(list);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_to_string
-
 /*
  * Functional test for routerset_to_string.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_to_string(void *arg)
 {
   routerset_t *set = NULL;
   char *s = NULL;
@@ -1960,15 +1911,12 @@ NS(test_main)(void *arg)
   routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, empty_empty)
-
 /*
  * Functional test for routerset_equal, with both routersets empty.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_empty_empty(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -1984,15 +1932,12 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, empty_not_empty)
-
 /*
  * Functional test for routerset_equal, with one routersets empty.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_empty_not_empty(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2008,16 +1953,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, differing_lengths)
-
 /*
  * Functional test for routerset_equal, with the routersets having
  * differing lengths.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_differing_lengths(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2035,16 +1977,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, unequal)
-
 /*
  * Functional test for routerset_equal, with the routersets being
  * different.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_unequal(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2061,16 +2000,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, equal)
-
 /*
  * Functional test for routerset_equal, with the routersets being
  * equal.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_equal(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2087,147 +2023,176 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_free, null_routerset)
-
 /*
  * Structural test for routerset_free, where the routerset is NULL.
  */
 
-NS_DECL(void, smartlist_free_, (smartlist_t *sl));
+static void rset_free_null_routerset_smartlist_free_(smartlist_t *sl);
+static int rset_free_null_routerset_smartlist_free__called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_free_null_routerset(void *arg)
 {
   (void)arg;
 
-  NS_MOCK(smartlist_free_);
+  MOCK(smartlist_free_,
+       rset_free_null_routerset_smartlist_free_);
 
   routerset_free_(NULL);
 
-  tt_int_op(CALLED(smartlist_free_), OP_EQ, 0);
+  tt_int_op(rset_free_null_routerset_smartlist_free__called, OP_EQ, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free_)(smartlist_t *s)
+rset_free_null_routerset_smartlist_free_(smartlist_t *s)
 {
   (void)s;
-  CALLED(smartlist_free_)++;
+  rset_free_null_routerset_smartlist_free__called++;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_free
-
 /*
  * Structural test for routerset_free.
  */
 
-NS_DECL(void, smartlist_free_, (smartlist_t *sl));
-NS_DECL(void, strmap_free_,(strmap_t *map, void (*free_val)(void*)));
-NS_DECL(void, digestmap_free_, (digestmap_t *map, void (*free_val)(void*)));
+static void rset_free_smartlist_free_(smartlist_t *sl);
+static int rset_free_smartlist_free__called = 0;
+static void rset_free_strmap_free_(strmap_t *map, void (*free_val)(void*));
+static int rset_free_strmap_free__called = 0;
+static void rset_free_digestmap_free_(digestmap_t *map,
+                                      void (*free_val)(void*));
+static int rset_free_digestmap_free__called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_free(void *arg)
 {
   routerset_t *routerset = routerset_new();
   (void)arg;
 
-  NS_MOCK(smartlist_free_);
-  NS_MOCK(strmap_free_);
-  NS_MOCK(digestmap_free_);
+  MOCK(smartlist_free_,
+       rset_free_smartlist_free_);
+  MOCK(strmap_free_,
+       rset_free_strmap_free_);
+  MOCK(digestmap_free_,
+       rset_free_digestmap_free_);
 
   routerset_free(routerset);
 
-  tt_int_op(CALLED(smartlist_free_), OP_NE, 0);
-  tt_int_op(CALLED(strmap_free_), OP_NE, 0);
-  tt_int_op(CALLED(digestmap_free_), OP_NE, 0);
+  tt_int_op(rset_free_smartlist_free__called, OP_NE, 0);
+  tt_int_op(rset_free_strmap_free__called, OP_NE, 0);
+  tt_int_op(rset_free_digestmap_free__called, OP_NE, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free_)(smartlist_t *s)
+rset_free_smartlist_free_(smartlist_t *s)
 {
-  CALLED(smartlist_free_)++;
+  rset_free_smartlist_free__called++;
   smartlist_free___real(s);
 }
 
 void
-NS(strmap_free_)(strmap_t *map, void (*free_val)(void*))
+rset_free_strmap_free_(strmap_t *map, void (*free_val)(void*))
 {
-  CALLED(strmap_free_)++;
+  rset_free_strmap_free__called++;
   strmap_free___real(map, free_val);
 }
 
 void
-NS(digestmap_free_)(digestmap_t *map, void (*free_val)(void*))
+rset_free_digestmap_free_(digestmap_t *map, void (*free_val)(void*))
 {
-  CALLED(digestmap_free_)++;
+  rset_free_digestmap_free__called++;
   digestmap_free___real(map, free_val);
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t routerset_tests[] = {
-  TEST_CASE(routerset_new),
-  TEST_CASE(routerset_get_countryname),
-  TEST_CASE(routerset_is_list),
-  TEST_CASE(routerset_needs_geoip),
-  TEST_CASE(routerset_is_empty),
-  TEST_CASE_ASPECT(routerset_contains, null_set_or_null_set_list),
-  TEST_CASE_ASPECT(routerset_contains, set_and_nickname),
-  TEST_CASE_ASPECT(routerset_contains, set_and_null_nickname),
-  TEST_CASE_ASPECT(routerset_contains, set_and_no_nickname),
-  TEST_CASE_ASPECT(routerset_contains, set_and_digest),
-  TEST_CASE_ASPECT(routerset_contains, set_and_no_digest),
-  TEST_CASE_ASPECT(routerset_contains, set_and_null_digest),
-  TEST_CASE_ASPECT(routerset_contains, set_and_addr),
-  TEST_CASE_ASPECT(routerset_contains, set_and_no_addr),
-  TEST_CASE_ASPECT(routerset_contains, set_and_null_addr),
-  TEST_CASE_ASPECT(routerset_contains, countries_no_geoip),
-  TEST_CASE_ASPECT(routerset_contains, countries_geoip),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, only_flag_and_no_ccs),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, creates_set),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, add_unknown),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, add_a1),
-  TEST_CASE(routerset_contains_extendinfo),
-  TEST_CASE(routerset_contains_router),
-  TEST_CASE(routerset_contains_routerstatus),
-  TEST_CASE_ASPECT(routerset_contains_node, none),
-  TEST_CASE_ASPECT(routerset_contains_node, routerinfo),
-  TEST_CASE_ASPECT(routerset_contains_node, routerstatus),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, no_routerset),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, list_with_no_nodes),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, list_flag_not_running),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, list),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, nodelist_with_no_nodes),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, nodelist_flag_not_running),
-  TEST_CASE_ASPECT(routerset_refresh_counties, geoip_not_loaded),
-  TEST_CASE_ASPECT(routerset_refresh_counties, no_countries),
-  TEST_CASE_ASPECT(routerset_refresh_counties, one_valid_country),
-  TEST_CASE_ASPECT(routerset_refresh_counties, one_invalid_country),
-  TEST_CASE_ASPECT(routerset_union, source_bad),
-  TEST_CASE_ASPECT(routerset_union, one),
-  TEST_CASE_ASPECT(routerset_parse, malformed),
-  TEST_CASE_ASPECT(routerset_parse, valid_hexdigest),
-  TEST_CASE_ASPECT(routerset_parse, valid_nickname),
-  TEST_CASE_ASPECT(routerset_parse, get_countryname),
-  TEST_CASE_ASPECT(routerset_parse, policy_wildcard),
-  TEST_CASE_ASPECT(routerset_parse, policy_ipv4),
-  TEST_CASE_ASPECT(routerset_parse, policy_ipv6),
-  TEST_CASE(routerset_subtract_nodes),
-  TEST_CASE_ASPECT(routerset_subtract_nodes, null_routerset),
-  TEST_CASE(routerset_to_string),
-  TEST_CASE_ASPECT(routerset_equal, empty_empty),
-  TEST_CASE_ASPECT(routerset_equal, empty_not_empty),
-  TEST_CASE_ASPECT(routerset_equal, differing_lengths),
-  TEST_CASE_ASPECT(routerset_equal, unequal),
-  TEST_CASE_ASPECT(routerset_equal, equal),
-  TEST_CASE_ASPECT(routerset_free, null_routerset),
-  TEST_CASE(routerset_free),
+  { "new", test_rset_new, TT_FORK, NULL, NULL },
+  { "get_countryname", test_rset_get_countryname, TT_FORK, NULL, NULL },
+  { "is_list", test_rset_is_list, TT_FORK, NULL, NULL },
+  { "needs_geoip", test_rset_needs_geoip, TT_FORK, NULL, NULL },
+  { "is_empty", test_rset_is_empty, TT_FORK, NULL, NULL },
+  { "contains_null_set_or_list", test_rset_contains_null_set_or_list,
+    TT_FORK, NULL, NULL },
+  { "contains_nickname", test_rset_contains_nickname, TT_FORK, NULL, NULL },
+  { "contains_null_nickname", test_rset_contains_null_nickname,
+    TT_FORK, NULL, NULL },
+  { "contains_no_nickname", test_rset_contains_no_nickname,
+    TT_FORK, NULL, NULL },
+  { "contains_digest", test_rset_contains_digest, TT_FORK, NULL, NULL },
+  { "contains_no_digest", test_rset_contains_no_digest, TT_FORK, NULL, NULL },
+  { "contains_null_digest", test_rset_contains_null_digest,
+    TT_FORK, NULL, NULL },
+  { "contains_addr", test_rset_contains_addr, TT_FORK, NULL, NULL },
+  { "contains_no_addr", test_rset_contains_no_addr, TT_FORK, NULL, NULL },
+  { "contains_null_addr", test_rset_contains_null_addr, TT_FORK, NULL, NULL },
+  { "contains_countries_no_geoip", test_rset_countries_no_geoip,
+    TT_FORK, NULL, NULL },
+  { "contains_countries_geoip", test_rset_countries_geoip,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_only_flag", test_rset_add_unknown_ccs_only_flag,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_creates_set", test_rset_add_unknown_ccs_creates_set,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_add_unknown", test_rset_add_unknown_ccs_add_unknown,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_add_a1", test_rset_add_unknown_ccs_add_a1,
+    TT_FORK, NULL, NULL },
+  { "contains_extendinfo", test_rset_contains_extendinfo,
+    TT_FORK, NULL, NULL },
+  { "contains_router", test_rset_contains_router, TT_FORK, NULL, NULL },
+  { "contains_routerstatus", test_rset_contains_routerstatus,
+    TT_FORK, NULL, NULL },
+  { "contains_none", test_rset_contains_none, TT_FORK, NULL, NULL },
+  { "contains_routerinfo", test_rset_contains_routerinfo,
+    TT_FORK, NULL, NULL },
+  { "contains_rs", test_rset_contains_rs, TT_FORK, NULL, NULL },
+  { "get_all_no_routerset", test_rset_get_all_no_routerset,
+    TT_FORK, NULL, NULL },
+  { "get_all_l_no_nodes", test_rset_get_all_l_no_nodes, TT_FORK, NULL, NULL },
+  { "get_all_l_not_running", test_rset_get_all_l_not_running,
+    TT_FORK, NULL, NULL },
+  { "get_all_list", test_rset_get_all_list, TT_FORK, NULL, NULL },
+  { "get_all_n_no_nodes", test_rset_get_all_n_no_nodes, TT_FORK, NULL, NULL },
+  { "get_all_n_not_running", test_rset_get_all_n_not_running,
+    TT_FORK, NULL, NULL },
+  { "refresh_geoip_not_loaded", test_rset_refresh_geoip_not_loaded,
+    TT_FORK, NULL, NULL },
+  { "refresh_no_countries", test_rset_refresh_no_countries,
+    TT_FORK, NULL, NULL },
+  { "refresh_one_valid_country", test_rset_refresh_one_valid_country,
+    TT_FORK, NULL, NULL },
+  { "refresh_one_invalid_country", test_rset_refresh_one_invalid_country,
+    TT_FORK, NULL, NULL },
+  { "union_source_bad", test_rset_union_source_bad, TT_FORK, NULL, NULL },
+  { "union_one", test_rset_union_one, TT_FORK, NULL, NULL },
+  { "parse_malformed", test_rset_parse_malformed, TT_FORK, NULL, NULL },
+  { "parse_valid_hexdigest", test_rset_parse_valid_hexdigest,
+    TT_FORK, NULL, NULL },
+  { "parse_valid_nickname", test_rset_parse_valid_nickname,
+    TT_FORK, NULL, NULL },
+  { "parse_get_countryname", test_rset_parse_get_countryname,
+    TT_FORK, NULL, NULL },
+  { "parse_policy_wildcard", test_rset_parse_policy_wildcard,
+    TT_FORK, NULL, NULL },
+  { "parse_policy_ipv4", test_rset_parse_policy_ipv4, TT_FORK, NULL, NULL },
+  { "parse_policy_ipv6", test_rset_parse_policy_ipv6, TT_FORK, NULL, NULL },
+  { "subtract_nodes", test_rset_subtract_nodes, TT_FORK, NULL, NULL },
+  { "subtract_nodes_null_routerset", test_rset_subtract_nodes_null_routerset,
+    TT_FORK, NULL, NULL },
+  { "to_string", test_rset_to_string, TT_FORK, NULL, NULL },
+  { "equal_empty_empty", test_rset_equal_empty_empty, TT_FORK, NULL, NULL },
+  { "equal_empty_not_empty", test_rset_equal_empty_not_empty,
+    TT_FORK, NULL, NULL },
+  { "equal_differing_lengths", test_rset_equal_differing_lengths,
+    TT_FORK, NULL, NULL },
+  { "equal_unequal", test_rset_equal_unequal, TT_FORK, NULL, NULL },
+  { "equal_equal", test_rset_equal_equal, TT_FORK, NULL, NULL },
+  { "free_null_routerset", test_rset_free_null_routerset,
+    TT_FORK, NULL, NULL },
+  { "free", test_rset_free, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh
index 00b3e88d37..804d2ada36 100755
--- a/src/test/test_rust.sh
+++ b/src/test/test_rust.sh
@@ -14,11 +14,12 @@ rustc_host=$(rustc -vV | grep host | sed 's/host: //')
 
 for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
     if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
+        # shellcheck disable=SC2086
 	cd "${abs_top_builddir:-../../..}/src/rust" && \
 	    CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \
-	    "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} \
+	    "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \
             --features "test_linking_hack" \
-            --target $rustc_host \
+            --target "$rustc_host" \
 	    ${EXTRA_CARGO_OPTIONS} \
 	    --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1
     fi
diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c
index bf9c6a49cd..9ec15948e8 100644
--- a/src/test/test_scheduler.c
+++ b/src/test/test_scheduler.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,8 +6,8 @@
 #include <math.h>
 
 #define SCHEDULER_KIST_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
-#define CHANNEL_PRIVATE_
+#define CHANNEL_OBJECT_PRIVATE
+#define CHANNEL_FILE_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "lib/evloop/compat_libevent.h"
@@ -15,7 +15,7 @@
 #include "core/or/channeltls.h"
 #include "core/mainloop/connection.h"
 #include "feature/nodelist/networkstatus.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #include "core/or/scheduler.h"
 
 /* Test suite stuff */
@@ -848,8 +848,8 @@ test_scheduler_initfree(void *arg)
 {
   (void)arg;
 
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
+  tt_ptr_op(channels_pending, OP_EQ, NULL);
+  tt_ptr_op(run_sched_ev, OP_EQ, NULL);
 
   MOCK(get_options, mock_get_options);
   set_scheduler_options(SCHEDULER_KIST);
@@ -858,17 +858,17 @@ test_scheduler_initfree(void *arg)
 
   scheduler_init();
 
-  tt_ptr_op(channels_pending, !=, NULL);
-  tt_ptr_op(run_sched_ev, !=, NULL);
+  tt_ptr_op(channels_pending, OP_NE, NULL);
+  tt_ptr_op(run_sched_ev, OP_NE, NULL);
   /* We have specified nothing in the torrc and there's no consensus so the
    * KIST scheduler is what should be in use */
-  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
-  tt_int_op(sched_run_interval, ==, 10);
+  tt_ptr_op(the_scheduler, OP_EQ, get_kist_scheduler());
+  tt_int_op(sched_run_interval, OP_EQ, 10);
 
   scheduler_free_all();
 
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
+  tt_ptr_op(channels_pending, OP_EQ, NULL);
+  tt_ptr_op(run_sched_ev, OP_EQ, NULL);
 
  done:
   UNMOCK(get_options);
@@ -890,11 +890,11 @@ test_scheduler_can_use_kist(void *arg)
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
 #ifdef HAVE_KIST_SUPPORT
-  tt_int_op(res_should, ==, 1);
+  tt_int_op(res_should, OP_EQ, 1);
 #else /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
 #endif /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_freq, ==, 1234);
+  tt_int_op(res_freq, OP_EQ, 1234);
 
   /* Test defer to consensus, but no consensus available */
   clear_options();
@@ -902,11 +902,11 @@ test_scheduler_can_use_kist(void *arg)
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
 #ifdef HAVE_KIST_SUPPORT
-  tt_int_op(res_should, ==, 1);
+  tt_int_op(res_should, OP_EQ, 1);
 #else /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
 #endif /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_freq, ==, 10);
+  tt_int_op(res_freq, OP_EQ, 10);
 
   /* Test defer to consensus, and kist consensus available */
   MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
@@ -915,11 +915,11 @@ test_scheduler_can_use_kist(void *arg)
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
 #ifdef HAVE_KIST_SUPPORT
-  tt_int_op(res_should, ==, 1);
+  tt_int_op(res_should, OP_EQ, 1);
 #else /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
 #endif /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_freq, ==, 12);
+  tt_int_op(res_freq, OP_EQ, 12);
   UNMOCK(networkstatus_get_param);
 
   /* Test defer to consensus, and vanilla consensus available */
@@ -928,8 +928,8 @@ test_scheduler_can_use_kist(void *arg)
   mocked_options.KISTSchedRunInterval = 0;
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
-  tt_int_op(res_should, ==, 0);
-  tt_int_op(res_freq, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
+  tt_int_op(res_freq, OP_EQ, 0);
   UNMOCK(networkstatus_get_param);
 
  done:
@@ -956,7 +956,7 @@ test_scheduler_ns_changed(void *arg)
   set_scheduler_options(SCHEDULER_KIST);
   set_scheduler_options(SCHEDULER_VANILLA);
 
-  tt_ptr_op(the_scheduler, ==, NULL);
+  tt_ptr_op(the_scheduler, OP_EQ, NULL);
 
   /* Change from vanilla to kist via consensus */
   the_scheduler = get_vanilla_scheduler();
@@ -964,9 +964,9 @@ test_scheduler_ns_changed(void *arg)
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
 #ifdef HAVE_KIST_SUPPORT
-  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_kist_scheduler());
 #else
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 #endif
 
   /* Change from kist to vanilla via consensus */
@@ -974,7 +974,7 @@ test_scheduler_ns_changed(void *arg)
   MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 
   /* Doesn't change when using KIST */
   the_scheduler = get_kist_scheduler();
@@ -982,9 +982,9 @@ test_scheduler_ns_changed(void *arg)
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
 #ifdef HAVE_KIST_SUPPORT
-  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_kist_scheduler());
 #else
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 #endif
 
   /* Doesn't change when using vanilla */
@@ -992,7 +992,7 @@ test_scheduler_ns_changed(void *arg)
   MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 
  done:
   UNMOCK(get_options);
diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c
new file mode 100644
index 0000000000..2ff4809124
--- /dev/null
+++ b/src/test/test_sendme.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/* Unit tests for handling different kinds of relay cell */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define SENDME_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/circuitlist.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "lib/crypt_ops/crypto_digest.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_md_consensus = current_ns_consensus =
+    tor_malloc_zero(sizeof(networkstatus_t));
+  current_md_consensus->net_params = smartlist_new();
+  current_md_consensus->routerstatus_list = smartlist_new();
+}
+
+static void
+free_mock_consensus(void)
+{
+  SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r,
+                    tor_free(r));
+  smartlist_free(current_md_consensus->routerstatus_list);
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_v1_record_digest(void *arg)
+{
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  /* Create our dummy circuit. */
+  or_circ = or_circuit_new(1, NULL);
+  /* Points it to the OR circuit now. */
+  circ = TO_CIRCUIT(or_circ);
+
+  /* The package window has to be a multiple of CIRCWINDOW_INCREMENT minus 1
+   * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that
+   * shouldn't be noted. */
+  circ->package_window = CIRCWINDOW_INCREMENT;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(!circ->sendme_last_digests);
+
+  /* This should work now. Package window at CIRCWINDOW_INCREMENT + 1. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(circ->sendme_last_digests);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* Next cell in the package window shouldn't do anything. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* The next CIRCWINDOW_INCREMENT should add one more digest. */
+  circ->package_window = (CIRCWINDOW_INCREMENT * 2) + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 2);
+
+ done:
+  circuit_free_(circ);
+}
+
+static void
+test_v1_consensus_params(void *arg)
+{
+  (void) arg;
+
+  setup_mock_consensus();
+  tt_assert(current_md_consensus);
+
+  /* Both zeroes. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=0");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 0);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Both ones. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 1);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Different values from each other. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Validate is the cell version is coherent with our internal default value
+   * and the one in the consensus. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  /* Minimum acceptable value is 1. */
+  tt_int_op(cell_version_can_be_handled(1), OP_EQ, true);
+  /* Minimum acceptable value is 1 so a cell version of 0 is refused. */
+  tt_int_op(cell_version_can_be_handled(0), OP_EQ, false);
+
+ done:
+  free_mock_consensus();
+}
+
+static void
+test_v1_build_cell(void *arg)
+{
+  uint8_t payload[RELAY_PAYLOAD_SIZE], digest[DIGEST_LEN];
+  ssize_t ret;
+  crypto_digest_t *cell_digest = NULL;
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  or_circ = or_circuit_new(1, NULL);
+  circ = TO_CIRCUIT(or_circ);
+  circ->sendme_last_digests = smartlist_new();
+
+  cell_digest = crypto_digest_new();
+  tt_assert(cell_digest);
+  crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20);
+  crypto_digest_get_digest(cell_digest, (char *) digest, sizeof(digest));
+  smartlist_add(circ->sendme_last_digests, tor_memdup(digest, sizeof(digest)));
+
+  /* SENDME v1 payload is 3 bytes + 20 bytes digest. See spec. */
+  ret = build_cell_payload_v1(digest, payload);
+  tt_int_op(ret, OP_EQ, 23);
+
+  /* Validation. */
+
+  /* An empty payload means SENDME version 0 thus valid. */
+  tt_int_op(sendme_is_valid(circ, payload, 0), OP_EQ, true);
+  /* Current phoney digest should have been popped. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+  /* An unparseable cell means invalid. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, (const uint8_t *) "A", 1), OP_EQ, false);
+  expect_log_msg_containing("Unparseable SENDME cell received. "
+                            "Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* No cell digest recorded for this. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  expect_log_msg_containing("We received a SENDME but we have no cell digests "
+                            "to match. Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* Note the wrong digest in the circuit, cell should fail validation. */
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+  expect_log_msg_containing("SENDME v1 cell digest do not match.");
+  teardown_capture_of_logs();
+
+  /* Record the cell digest into the circuit, cell should validate. */
+  memcpy(or_circ->crypto.sendme_digest, digest, sizeof(digest));
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, true);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+ done:
+  crypto_digest_free(cell_digest);
+  circuit_free_(circ);
+}
+
+static void
+test_cell_payload_pad(void *arg)
+{
+  size_t pad_offset, payload_len, expected_offset;
+
+  (void) arg;
+
+  /* Offset should be 0, not enough room for padding. */
+  payload_len = RELAY_PAYLOAD_SIZE;
+  pad_offset = get_pad_cell_offset(payload_len);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Still no room because we keep 4 extra bytes. */
+  pad_offset = get_pad_cell_offset(payload_len - 4);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* We should have 1 byte of padding. Meaning, the offset should be the
+   * CELL_PAYLOAD_SIZE minus 1 byte. */
+  expected_offset = CELL_PAYLOAD_SIZE - 1;
+  pad_offset = get_pad_cell_offset(payload_len - 5);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Now some arbitrary small payload length. The cell size is header + 10 +
+   * extra 4 bytes we keep so the offset should be there. */
+  expected_offset = RELAY_HEADER_SIZE + 10 + 4;
+  pad_offset = get_pad_cell_offset(10);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Data length of 0. */
+  expected_offset = RELAY_HEADER_SIZE + 4;
+  pad_offset = get_pad_cell_offset(0);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+ done:
+  ;
+}
+
+static void
+test_cell_version_validation(void *arg)
+{
+  (void) arg;
+
+  /* We currently only support up to SENDME_MAX_SUPPORTED_VERSION so we are
+   * going to test the boundaries there. */
+
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION));
+
+  /* Version below our supported should pass. */
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION - 1));
+
+  /* Extra version from our supported should fail. */
+  tt_assert(!cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION + 1));
+
+  /* Simple check for version 0. */
+  tt_assert(cell_version_can_be_handled(0));
+
+  /* We MUST handle the default cell version that we emit or accept. */
+  tt_assert(cell_version_can_be_handled(SENDME_EMIT_MIN_VERSION_DEFAULT));
+  tt_assert(cell_version_can_be_handled(SENDME_ACCEPT_MIN_VERSION_DEFAULT));
+
+ done:
+  ;
+}
+
+/* check our decisions about how much stuff to put into relay cells. */
+static void
+test_package_payload_len(void *arg)
+{
+  (void)arg;
+  /* this is not a real circuit: it only has the fields needed for this
+   * test. */
+  circuit_t *c = tor_malloc_zero(sizeof(circuit_t));
+
+  /* check initial conditions. */
+  circuit_reset_sendme_randomness(c);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE, CIRCWINDOW_INCREMENT / 2);
+  tt_int_op(c->send_randomness_after_n_cells, OP_LT, CIRCWINDOW_INCREMENT);
+
+  /* We have a bunch of cells before we need to send randomness, so the first
+   * few can be packaged full. */
+  int initial = c->send_randomness_after_n_cells;
+  size_t n = connection_edge_get_inbuf_bytes_to_package(10000, 0, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  n = connection_edge_get_inbuf_bytes_to_package(95000, 1, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial isn't set, we won't package a partially full cell at
+   * all. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 0, c);
+  tt_int_op(n, OP_EQ, 0);
+  /* no change in our state, since nothing was sent. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial is set and the partial cell is not going to have
+   * _enough_ randomness, we package it, but we don't consider ourselves to
+   * have sent a sufficiently random cell. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-1);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 3);
+
+  /* Make sure we set have_set_sufficiently_random_cell as appropriate. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-64, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-64);
+  tt_assert(c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 4);
+
+  /* Now let's look at what happens when we get down to zero. Since we have
+   * sent a sufficiently random cell, we will not force this one to have a gap.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* What would happen if we hadn't sent a sufficiently random cell? */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  const size_t reduced_payload_size = RELAY_PAYLOAD_SIZE - 4 - 16;
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* Here is a fun case: if it's time to package a small cell, then
+   * package_partial==0 should mean we accept that many bytes.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(reduced_payload_size, 0, c);
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+
+ done:
+  tor_free(c);
+}
+
+struct testcase_t sendme_tests[] = {
+  { "v1_record_digest", test_v1_record_digest, TT_FORK,
+    NULL, NULL },
+  { "v1_consensus_params", test_v1_consensus_params, TT_FORK,
+    NULL, NULL },
+  { "v1_build_cell", test_v1_build_cell, TT_FORK,
+    NULL, NULL },
+  { "cell_payload_pad", test_cell_payload_pad, TT_FORK,
+    NULL, NULL },
+  { "cell_version_validation", test_cell_version_validation, TT_FORK,
+    NULL, NULL },
+  { "package_payload_len", test_package_payload_len, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index 413dfbeb03..9e49e835c9 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define SHARED_RANDOM_PRIVATE
@@ -21,7 +21,7 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/dirparse/authcert_parse.h"
 #include "feature/hs_common/shared_random_client.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dirclient/dir_server_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -58,14 +58,17 @@ trusteddirserver_get_by_v3_auth_digest_m(const char *digest)
 }
 
 /* Setup a minimal dirauth environment by initializing the SR state and
- * making sure the options are set to be an authority directory. */
+ * making sure the options are set to be an authority directory.
+ * You must only call this function once per process. */
 static void
 init_authority_state(void)
 {
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   or_options_t *options = get_options_mutable();
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   tt_assert(mock_cert);
   options->AuthoritativeDir = 1;
   tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
@@ -201,7 +204,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     /* Compare it with the correct result */
@@ -213,7 +216,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 19:22:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -224,7 +227,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -235,7 +238,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -275,7 +278,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -287,7 +290,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:59 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -299,7 +302,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -323,12 +326,13 @@ test_get_start_time_of_current_run(void *arg)
 
     retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC",
                                 &mock_consensus.valid_after);
+    tt_int_op(retval, OP_EQ, 0);
 
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
     update_approx_time(current_time);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
 
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
@@ -336,7 +340,7 @@ test_get_start_time_of_current_run(void *arg)
     format_iso_time(tbuf, run_start_time);
     tt_str_op("2015-04-19 00:00:00", OP_EQ, tbuf);
     /* Check that voting_schedule.interval_starts is at 01:00 (see above) */
-    time_t interval_starts = voting_schedule_get_next_valid_after_time();
+    time_t interval_starts = dirauth_sched_get_next_valid_after_time();
     format_iso_time(tbuf, interval_starts);
     tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf);
   }
@@ -356,7 +360,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:15:32 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -388,13 +392,13 @@ test_get_start_time_functions(void *arg)
   tt_int_op(retval, OP_EQ, 0);
   time_t now = mock_consensus.valid_after;
 
-  voting_schedule_recalculate_timing(get_options(), now);
+  dirauth_sched_recalculate_timing(get_options(), now);
   time_t start_time_of_protocol_run =
     sr_state_get_start_time_of_current_protocol_run();
   tt_assert(start_time_of_protocol_run);
 
   /* Check that the round start time of the beginning of the run, is itself */
-  tt_int_op(get_start_time_of_current_round(), OP_EQ,
+  tt_int_op(dirauth_sched_get_cur_valid_after_time(), OP_EQ,
             start_time_of_protocol_run);
 
  done:
@@ -438,7 +442,9 @@ test_sr_commit(void *arg)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -457,12 +463,12 @@ test_sr_commit(void *arg)
     /* We should have a reveal value. */
     tt_assert(commit_has_reveal_value(our_commit));
     /* We should have a random value. */
-    tt_assert(!tor_mem_is_zero((char *) our_commit->random_number,
+    tt_assert(!fast_mem_is_zero((char *) our_commit->random_number,
                                sizeof(our_commit->random_number)));
     /* Commit and reveal timestamp should be the same. */
     tt_u64_op(our_commit->commit_ts, OP_EQ, our_commit->reveal_ts);
     /* We should have a hashed reveal. */
-    tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal,
+    tt_assert(!fast_mem_is_zero(our_commit->hashed_reveal,
                                sizeof(our_commit->hashed_reveal)));
     /* Do we have a valid encoded commit and reveal. Note the following only
      * tests if the generated values are correct. Their could be a bug in
@@ -849,7 +855,9 @@ test_sr_setup_commits(void)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -1087,73 +1095,99 @@ test_sr_get_majority_srv_from_votes(void *arg)
   smartlist_free(votes);
 }
 
+/* Testing sr_srv_dup(). */
 static void
-test_utils(void *arg)
+test_sr_svr_dup(void *arg)
 {
-  (void) arg;
+  (void)arg;
 
-  /* Testing srv_dup(). */
-  {
-    sr_srv_t *srv = NULL, *dup_srv = NULL;
-    const char *srv_value =
-      "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
-    srv = tor_malloc_zero(sizeof(*srv));
-    srv->num_reveals = 42;
-    memcpy(srv->value, srv_value, sizeof(srv->value));
-    dup_srv = srv_dup(srv);
-    tt_assert(dup_srv);
-    tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
-    tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
-    tor_free(srv);
-    tor_free(dup_srv);
-  }
+  sr_srv_t *srv = NULL, *dup_srv = NULL;
+  const char *srv_value =
+    "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
+  srv = tor_malloc_zero(sizeof(*srv));
+  srv->num_reveals = 42;
+  memcpy(srv->value, srv_value, sizeof(srv->value));
+  dup_srv = sr_srv_dup(srv);
+  tt_assert(dup_srv);
+  tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
+  tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
 
-  /* Testing commitments_are_the_same(). Currently, the check is to test the
-   * value of the encoded commit so let's make sure that actually works. */
-  {
-    /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
-     * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
-     * which comes down to SR_COMMIT_BASE64_LEN + 1. */
-    const char *payload =
-      "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
-      "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
-      "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
-      "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
-    sr_commit_t commit1, commit2;
-    memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
-    memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
-    /* Let's corrupt one of them. */
-    memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
-  }
+ done:
+  tor_free(srv);
+  tor_free(dup_srv);
+}
 
-  /* Testing commit_is_authoritative(). */
-  {
-    crypto_pk_t *k = crypto_pk_new();
-    char digest[DIGEST_LEN];
-    sr_commit_t commit;
+/* Testing commitments_are_the_same(). Currently, the check is to test the
+ * value of the encoded commit so let's make sure that actually works. */
+static void
+test_commitments_are_the_same(void *arg)
+{
+  (void)arg;
+
+  /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
+  * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
+  * which comes down to SR_COMMIT_BASE64_LEN + 1. */
+  const char *payload =
+  "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
+  "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
+  "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
+  "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
+  sr_commit_t commit1, commit2;
+  memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
+  memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
+  /* Let's corrupt one of them. */
+  memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
 
-    tt_assert(!crypto_pk_generate_key(k));
+ done:
+  return;
+}
 
-    tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
-    memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
-    /* Change the pubkey. */
-    memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
-    crypto_pk_free(k);
-  }
+/* Testing commit_is_authoritative(). */
+static void
+test_commit_is_authoritative(void *arg)
+{
+  (void)arg;
 
-  /* Testing get_phase_str(). */
-  {
-    tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
-    tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
-  }
+  crypto_pk_t *k = crypto_pk_new();
+  char digest[DIGEST_LEN];
+  sr_commit_t commit;
+
+  tt_assert(!crypto_pk_generate_key(k));
+
+  tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
+  memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
+  /* Change the pubkey. */
+  memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
+
+ done:
+  crypto_pk_free(k);
+}
+
+static void
+test_get_phase_str(void *arg)
+{
+  (void)arg;
+
+  tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
+  tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
+
+ done:
+  return;
+}
+
+/* Test utils that depend on authority state */
+static void
+test_utils_auth(void *arg)
+{
+  (void)arg;
+  init_authority_state();
 
   /* Testing phase transition */
   {
-    init_authority_state();
     set_sr_phase(SR_PHASE_COMMIT);
     tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1);
     tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0);
@@ -1164,8 +1198,193 @@ test_utils(void *arg)
     tt_int_op(is_phase_transition(42), OP_EQ, 1);
   }
 
+  /* Testing get, set, delete, clean SRVs */
+
+  {
+    /* Just set the previous SRV */
+    test_sr_setup_srv(0);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Delete the SRVs one at a time */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now set the SRVs to NULL instead */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now copy the values across */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the previous: the protocol goes the other way */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_previous_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is the same */
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  {
+    /* Now copy a value onto itself */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Take a copy of the old value */
+    sr_srv_t old_current_srv;
+    memcpy(&old_current_srv, sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the current: the protocol never replaces an SRV with
+     * the same value */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_current_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different between current and previous */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is the same as the old content */
+    tt_mem_op(&old_current_srv, OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  /* I don't think we can say "expect a BUG()" in our tests. */
+#if 0
+  {
+    /* Now copy a value onto itself without sr_srv_dup().
+     * This should fail with a BUG() warning. */
+    test_sr_setup_srv(1);
+    sr_state_set_current_srv(sr_state_get_current_srv());
+    sr_state_set_previous_srv(sr_state_get_previous_srv());
+  }
+#endif /* 0 */
+
  done:
-  return;
+  sr_state_free_all();
 }
 
 static void
@@ -1173,6 +1392,7 @@ test_state_transition(void *arg)
 {
   sr_state_t *state = NULL;
   time_t now = time(NULL);
+  sr_srv_t *cur = NULL;
 
   (void) arg;
 
@@ -1211,44 +1431,47 @@ test_state_transition(void *arg)
 
   /* Test SRV rotation in our state. */
   {
-    const sr_srv_t *cur, *prev;
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     tt_assert(cur);
-    /* After, current srv should be the previous and then set to NULL. */
+    /* After, the previous SRV should be the same as the old current SRV, and
+     * the current SRV should be set to NULL */
     state_rotate_srv();
-    prev = sr_state_get_previous_srv();
-    tt_assert(prev == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
     sr_state_clean_srvs();
+    tor_free(cur);
   }
 
   /* New protocol run. */
   {
-    const sr_srv_t *cur;
     /* Setup some new SRVs so we can confirm that a new protocol run
      * actually makes them rotate and compute new ones. */
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
-    tt_assert(cur);
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     set_sr_phase(SR_PHASE_REVEAL);
     MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
     new_protocol_run(now);
     UNMOCK(get_my_v3_authority_cert);
     /* Rotation happened. */
-    tt_assert(sr_state_get_previous_srv() == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     /* We are going into COMMIT phase so we had to rotate our SRVs. Usually
      * our current SRV would be NULL but a new protocol run should make us
      * compute a new SRV. */
     tt_assert(sr_state_get_current_srv());
     /* Also, make sure we did change the current. */
-    tt_assert(sr_state_get_current_srv() != cur);
+    tt_mem_op(sr_state_get_current_srv(), OP_NE, cur, sizeof(sr_srv_t));
     /* We should have our commitment alone. */
     tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
     tt_int_op(state->n_commit_rounds, OP_EQ, 0);
     /* 46 here since we were at 45 just before. */
     tt_u64_op(state->n_protocol_runs, OP_EQ, 46);
+    tor_free(cur);
   }
 
   /* Cleanup of SRVs. */
@@ -1259,6 +1482,7 @@ test_state_transition(void *arg)
   }
 
  done:
+  tor_free(cur);
   sr_state_free_all();
 }
 
@@ -1454,7 +1678,13 @@ struct testcase_t sr_tests[] = {
   { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL },
   { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes,
     TT_FORK, NULL, NULL },
-  { "utils", test_utils, TT_FORK, NULL, NULL },
+  { "sr_svr_dup", test_sr_svr_dup, TT_FORK, NULL, NULL },
+  { "commitments_are_the_same", test_commitments_are_the_same, TT_FORK, NULL,
+    NULL },
+  { "commit_is_authoritative", test_commit_is_authoritative, TT_FORK, NULL,
+    NULL },
+  { "get_phase_str", test_get_phase_str, TT_FORK, NULL, NULL },
+  { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL },
   { "state_transition", test_state_transition, TT_FORK, NULL, NULL },
   { "state_update", test_state_update, TT_FORK,
     NULL, NULL },
diff --git a/src/test/test_slow.c b/src/test/test_slow.c
index bda67b2d92..49b1066dac 100644
--- a/src/test/test_slow.c
+++ b/src/test/test_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,9 @@
 
 struct testgroup_t testgroups[] = {
   { "slow/crypto/", slow_crypto_tests },
-  { "slow/util/", slow_util_tests },
+  { "slow/process/", slow_process_tests },
+  { "slow/prob_distr/", slow_stochastic_prob_distr_tests },
+  { "slow/ptr/", slow_ptr_tests },
   END_OF_GROUPS
 };
 
diff --git a/src/test/test_socks.c b/src/test/test_socks.c
index 3686e1036b..4a465c7361 100644
--- a/src/test/test_socks.c
+++ b/src/test/test_socks.c
@@ -1,10 +1,10 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_socks.h"
@@ -399,6 +399,43 @@ test_socks_5_supported_commands(void *ptr)
 
   tt_int_op(0,OP_EQ, buf_datalen(buf));
 
+  socks_request_clear(socks);
+
+  /* SOCKS 5 Send RESOLVE_PTR [F1] for an IPv6 address */
+  ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\xF1\x00\x04"
+           "\x20\x01\x0d\xb8\x85\xa3\x00\x00\x00\x00\x8a\x2e\x03\x70\x73\x34"
+           "\x12\x34");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
+  tt_int_op(5,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0,OP_EQ, socks->reply[1]);
+  tt_str_op("[2001:db8:85a3::8a2e:370:7334]",OP_EQ, socks->address);
+
+  tt_int_op(0,OP_EQ, buf_datalen(buf));
+
+  socks_request_clear(socks);
+
+  /* SOCKS 5 Send RESOLVE_PTR [F1] for a an IPv6 address written as a
+   * string with brackets */
+  ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\xF1\x00\x03\x1e");
+  ADD_DATA(buf, "[2001:db8:85a3::8a2e:370:7334]");
+  ADD_DATA(buf, "\x12\x34");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
+  tt_int_op(5,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0,OP_EQ, socks->reply[1]);
+  tt_str_op("[2001:db8:85a3::8a2e:370:7334]",OP_EQ, socks->address);
+
+  tt_int_op(0,OP_EQ, buf_datalen(buf));
+
  done:
   ;
 }
diff --git a/src/test/test_stats.c b/src/test/test_stats.c
new file mode 100644
index 0000000000..291473ebc9
--- /dev/null
+++ b/src/test/test_stats.c
@@ -0,0 +1,258 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_stats.c
+ * \brief Unit tests for the statistics (reputation history) module.
+ **/
+
+#include "orconfig.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "app/config/or_state_st.h"
+#include "test/rng_test_helpers.h"
+
+#include <stdio.h>
+
+#ifdef _WIN32
+/* For mkdir() */
+#include <direct.h>
+#else
+#include <dirent.h>
+#endif /* defined(_WIN32) */
+
+#include <math.h>
+
+/* These macros pull in declarations for some functions and structures that
+ * are typically file-private. */
+#define CIRCUITSTATS_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define MAINLOOP_PRIVATE
+#define STATEFILE_PRIVATE
+
+#include "core/or/or.h"
+#include "lib/err/backtrace.h"
+#include "lib/buf/buffers.h"
+#include "core/or/circuitstats.h"
+#include "app/config/config.h"
+#include "test/test.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/memarea/memarea.h"
+#include "feature/stats/rephist.h"
+#include "app/config/statefile.h"
+
+/** Run unit tests for some stats code. */
+static void
+test_stats(void *arg)
+{
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  char *s = NULL;
+  int i;
+
+  /* Start with testing exit port statistics; we shouldn't collect exit
+   * stats without initializing them. */
+  (void)arg;
+  rep_hist_note_exit_stream_opened(80);
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note some streams and bytes, and generate history
+   * string. */
+  rep_hist_exit_stats_init(now);
+  rep_hist_note_exit_stream_opened(80);
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  rep_hist_note_exit_stream_opened(443);
+  rep_hist_note_exit_bytes(443, 100, 10000);
+  rep_hist_note_exit_bytes(443, 100, 10000);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "exit-kibibytes-written 80=1,443=1,other=0\n"
+             "exit-kibibytes-read 80=10,443=20,other=0\n"
+             "exit-streams-opened 80=4,443=4,other=0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Add a few bytes on 10 more ports and ensure that only the top 10
+   * ports are contained in the history string. */
+  for (i = 50; i < 60; i++) {
+    rep_hist_note_exit_bytes(i, i, i);
+    rep_hist_note_exit_stream_opened(i);
+  }
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "exit-kibibytes-written 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
+             "59=1,80=1,443=1,other=1\n"
+             "exit-kibibytes-read 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
+             "59=1,80=10,443=20,other=1\n"
+             "exit-streams-opened 52=4,53=4,54=4,55=4,56=4,57=4,58=4,"
+             "59=4,80=4,443=4,other=4\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add some bytes, and ensure we don't generate
+   * a history string. */
+  rep_hist_exit_stats_term();
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add some bytes, reset stats, and see what history we
+   * get when observing no streams or bytes at all. */
+  rep_hist_exit_stats_init(now);
+  rep_hist_note_exit_stream_opened(80);
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  rep_hist_reset_exit_stats(now);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "exit-kibibytes-written other=0\n"
+             "exit-kibibytes-read other=0\n"
+             "exit-streams-opened other=0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Continue with testing connection statistics; we shouldn't collect
+   * conn stats without initializing them. */
+  rep_hist_note_or_conn_bytes(1, 20, 400, now);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note bytes, and generate history string. */
+  rep_hist_conn_stats_init(now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,1,0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add some bytes, and ensure we don't generate
+   * a history string. */
+  rep_hist_conn_stats_term();
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add some bytes, reset stats, and see what history we
+   * get when observing no bytes at all. */
+  rep_hist_conn_stats_init(now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
+  rep_hist_reset_conn_stats(now);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Continue with testing buffer statistics; we shouldn't collect buffer
+   * stats without initializing them. */
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, add statistics for a single circuit, and generate
+   * the history string. */
+  rep_hist_buffer_stats_init(now);
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "cell-processed-cells 20,0,0,0,0,0,0,0,0,0\n"
+             "cell-queued-cells 2.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
+                               "0.00,0.00\n"
+             "cell-time-in-queue 2,0,0,0,0,0,0,0,0,0\n"
+             "cell-circuits-per-decile 1\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Add nineteen more circuit statistics to the one that's already in the
+   * history to see that the math works correctly. */
+  for (i = 21; i < 30; i++)
+    rep_hist_add_buffer_stats(2.0, 2.0, i);
+  for (i = 20; i < 30; i++)
+    rep_hist_add_buffer_stats(3.5, 3.5, i);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "cell-processed-cells 29,28,27,26,25,24,23,22,21,20\n"
+             "cell-queued-cells 2.75,2.75,2.75,2.75,2.75,2.75,2.75,2.75,"
+                               "2.75,2.75\n"
+             "cell-time-in-queue 3,3,3,3,3,3,3,3,3,3\n"
+             "cell-circuits-per-decile 2\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add statistics for one circuit, and ensure we
+   * don't generate a history string. */
+  rep_hist_buffer_stats_term();
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add statistics for one circuit, reset stats, and make
+   * sure that the history has all zeros. */
+  rep_hist_buffer_stats_init(now);
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  rep_hist_reset_buffer_stats(now);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "cell-processed-cells 0,0,0,0,0,0,0,0,0,0\n"
+             "cell-queued-cells 0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
+                               "0.00,0.00\n"
+             "cell-time-in-queue 0,0,0,0,0,0,0,0,0,0\n"
+             "cell-circuits-per-decile 0\n",OP_EQ, s);
+
+ done:
+  tor_free(s);
+}
+
+/** Run unit tests the mtbf stats code. */
+static void
+test_rephist_mtbf(void *arg)
+{
+  (void)arg;
+
+  time_t now = 1572500000; /* 2010-10-31 05:33:20 UTC */
+  time_t far_future = MAX(now, time(NULL)) + 365*24*60*60;
+  int r;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_mtbf"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+
+  rep_history_clean(far_future);
+
+  /* No data */
+
+  r = rep_hist_load_mtbf_data(now);
+  tt_int_op(r, OP_EQ, -1);
+  rep_history_clean(far_future);
+
+  /* Blank data */
+
+  r = rep_hist_record_mtbf_data(now, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = rep_hist_load_mtbf_data(now);
+  tt_int_op(r, OP_EQ, 0);
+  rep_history_clean(far_future);
+
+  r = rep_hist_record_mtbf_data(now, 1);
+  tt_int_op(r, OP_EQ, 0);
+  r = rep_hist_load_mtbf_data(now);
+  tt_int_op(r, OP_EQ, 0);
+  rep_history_clean(far_future);
+
+ done:
+  rep_history_clean(far_future);
+  tor_free(ddir_fname);
+}
+
+#define ENT(name)                                                       \
+  { #name, test_ ## name , 0, NULL, NULL }
+#define FORK(name)                                                      \
+  { #name, test_ ## name , TT_FORK, NULL, NULL }
+
+struct testcase_t stats_tests[] = {
+  FORK(stats),
+  ENT(rephist_mtbf),
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_status.c b/src/test/test_status.c
index 9c47469975..82afe0fd2a 100644
--- a/src/test/test_status.c
+++ b/src/test/test_status.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define STATUS_PRIVATE
@@ -33,10 +33,6 @@
 
 #include "test/test.h"
 
-#define NS_MODULE status
-
-#define NS_SUBMODULE count_circuits
-
 /*
  * Test that count_circuits() is correctly counting the number of
  * global circuits.
@@ -44,10 +40,10 @@
 
 static smartlist_t * mock_global_circuitlist = NULL;
 
-NS_DECL(smartlist_t *, circuit_get_global_list, (void));
+static smartlist_t * status_count_circuits_circuit_get_global_list(void);
 
 static void
-NS(test_main)(void *arg)
+test_status_count_circuits(void *arg)
 {
   /* Choose origin_circuit_t wlog. */
   origin_circuit_t *mock_circuit1, *mock_circuit2;
@@ -61,7 +57,8 @@ NS(test_main)(void *arg)
   smartlist_add(mock_global_circuitlist, TO_CIRCUIT(mock_circuit1));
   smartlist_add(mock_global_circuitlist, TO_CIRCUIT(mock_circuit2));
 
-  NS_MOCK(circuit_get_global_list);
+  MOCK(circuit_get_global_list,
+       status_count_circuits_circuit_get_global_list);
 
   actual_circuits = count_circuits();
 
@@ -72,25 +69,22 @@ NS(test_main)(void *arg)
   tor_free(mock_circuit2);
   smartlist_free(mock_global_circuitlist);
   mock_global_circuitlist = NULL;
-  NS_UNMOCK(circuit_get_global_list);
+  UNMOCK(circuit_get_global_list);
 }
 
 static smartlist_t *
-NS(circuit_get_global_list)(void)
+status_count_circuits_circuit_get_global_list(void)
 {
   return mock_global_circuitlist;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE secs_to_uptime
-
 /*
  * Test that secs_to_uptime() is converting the number of seconds that
  * Tor is up for into the appropriate string form containing hours and minutes.
  */
 
 static void
-NS(test_main)(void *arg)
+test_status_secs_to_uptime(void *arg)
 {
   const char *expected;
   char *actual;
@@ -161,9 +155,6 @@ NS(test_main)(void *arg)
       tor_free(actual);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE bytes_to_usage
-
 /*
  * Test that bytes_to_usage() is correctly converting the number of bytes that
  * Tor has read/written into the appropriate string form containing kilobytes,
@@ -171,7 +162,7 @@ NS(test_main)(void *arg)
  */
 
 static void
-NS(test_main)(void *arg)
+test_status_bytes_to_usage(void *arg)
 {
   const char *expected;
   char *actual;
@@ -242,29 +233,30 @@ NS(test_main)(void *arg)
       tor_free(actual);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, fails)
-
 /*
  * Tests that log_heartbeat() fails when in the public server mode,
  * not hibernating, and we couldn't get the current routerinfo.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static double status_hb_fails_tls_get_write_overhead_ratio(void);
+static int status_hb_fails_we_are_hibernating(void);
+static int status_hb_fails_public_server_mode(const or_options_t *options);
+static const routerinfo_t * status_hb_fails_router_get_my_routerinfo(void);
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_fails(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_fails_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_fails_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_fails_public_server_mode);
+  MOCK(router_get_my_routerinfo,
+       status_hb_fails_router_get_my_routerinfo);
 
   expected = -1;
   actual = log_heartbeat(0);
@@ -272,26 +264,26 @@ NS(test_main)(void *arg)
   tt_int_op(actual, OP_EQ, expected);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(router_get_my_routerinfo);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_fails_tls_get_write_overhead_ratio(void)
 {
   return 2.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_fails_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_fails_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -299,43 +291,51 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+status_hb_fails_router_get_my_routerinfo(void)
 {
   return NULL;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, not_in_consensus)
-
 /*
  * Tests that log_heartbeat() logs appropriately if we are not in the cached
  * consensus.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
-NS_DECL(const node_t *, node_get_by_id, (const char *identity_digest));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
+static double status_hb_not_in_consensus_tls_get_write_overhead_ratio(void);
+static int status_hb_not_in_consensus_we_are_hibernating(void);
+static int status_hb_not_in_consensus_public_server_mode(
+             const or_options_t *options);
+static const routerinfo_t *status_hb_not_in_consensus_get_my_routerinfo(void);
+static const node_t * status_hb_not_in_consensus_node_get_by_id(
+             const char *identity_digest);
+static void status_hb_not_in_consensus_logv(
+             int severity, log_domain_mask_t domain, const char *funcname,
+             const char *suffix, const char *format, va_list ap);
+static int status_hb_not_in_consensus_logv_called = 0;
+static int status_hb_not_in_consensus_server_mode(const or_options_t *options);
 
 static routerinfo_t *mock_routerinfo;
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_not_in_consensus(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(router_get_my_routerinfo);
-  NS_MOCK(node_get_by_id);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_not_in_consensus_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_not_in_consensus_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_not_in_consensus_public_server_mode);
+  MOCK(router_get_my_routerinfo,
+       status_hb_not_in_consensus_get_my_routerinfo);
+  MOCK(node_get_by_id,
+       status_hb_not_in_consensus_node_get_by_id);
+  MOCK(logv,
+       status_hb_not_in_consensus_logv);
+  MOCK(server_mode,
+       status_hb_not_in_consensus_server_mode);
 
   log_global_min_severity_ = LOG_DEBUG;
   onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP] = 1;
@@ -347,33 +347,33 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 6);
+  tt_int_op(status_hb_not_in_consensus_logv_called, OP_EQ, 6);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(router_get_my_routerinfo);
-    NS_UNMOCK(node_get_by_id);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(router_get_my_routerinfo);
+    UNMOCK(node_get_by_id);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
     tor_free(mock_routerinfo);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_not_in_consensus_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_not_in_consensus_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_not_in_consensus_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -381,7 +381,7 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+status_hb_not_in_consensus_get_my_routerinfo(void)
 {
   mock_routerinfo = tor_malloc(sizeof(routerinfo_t));
 
@@ -389,7 +389,7 @@ NS(router_get_my_routerinfo)(void)
 }
 
 static const node_t *
-NS(node_get_by_id)(const char *identity_digest)
+status_hb_not_in_consensus_node_get_by_id(const char *identity_digest)
 {
   (void)identity_digest;
 
@@ -397,14 +397,14 @@ NS(node_get_by_id)(const char *identity_digest)
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain,
+status_hb_not_in_consensus_logv(int severity, log_domain_mask_t domain,
   const char *funcname, const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_not_in_consensus_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -412,7 +412,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -429,7 +429,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 3:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_circuit_handshake_stats"),
                 OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
@@ -442,19 +442,20 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 4:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_link_protocol_counts"),
                 OP_NE, NULL);
       break;
     case 5:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
-      tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s");
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s%s");
       tt_str_op(va_arg(ap, char *), OP_EQ,
                 " 0 circuits killed with too many cells.");
       tt_str_op(va_arg(ap, char *), OP_EQ, " [cc not enabled]");
       tt_str_op(va_arg(ap, char *), OP_EQ, " [conn not enabled]");
       tt_str_op(va_arg(ap, char *), OP_EQ, "");
+      tt_str_op(va_arg(ap, char *), OP_EQ, " 0 INTRODUCE2 rejected.");
       break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
@@ -462,51 +463,58 @@ NS(logv)(int severity, log_domain_mask_t domain,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_not_in_consensus_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_not_in_consensus_server_mode(const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, simple)
-
 /*
  * Tests that log_heartbeat() correctly logs heartbeat information
  * normally.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
+static double status_hb_simple_tls_get_write_overhead_ratio(void);
+static int status_hb_simple_we_are_hibernating(void);
+static int status_hb_simple_public_server_mode(const or_options_t *options);
+static long status_hb_simple_get_uptime(void);
+static uint64_t status_hb_simple_get_bytes_read(void);
+static uint64_t status_hb_simple_get_bytes_written(void);
+static void status_hb_simple_logv(int severity, log_domain_mask_t domain,
+                                  const char *funcname, const char *suffix,
+                                  const char *format, va_list ap);
+ATTR_UNUSED static int status_hb_simple_logv_called = 0;
+static int status_hb_simple_server_mode(const or_options_t *options);
 
-static int NS(n_msgs) = 0;
+static int status_hb_simple_n_msgs = 0;
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_simple(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_simple_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_simple_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_simple_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_simple_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_simple_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_simple_get_bytes_written);
+  MOCK(logv,
+       status_hb_simple_logv);
+  MOCK(server_mode,
+       status_hb_simple_server_mode);
 
   log_global_min_severity_ = LOG_DEBUG;
 
@@ -514,33 +522,33 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(NS(n_msgs), OP_EQ, 1);
+  tt_int_op(status_hb_simple_n_msgs, OP_EQ, 1);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_simple_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_simple_we_are_hibernating(void)
 {
   return 1;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_simple_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -548,33 +556,34 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_simple_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_simple_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_simple_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
+status_hb_simple_logv(int severity, log_domain_mask_t domain,
+  const char *funcname,
   const char *suffix, const char *format, va_list ap)
 {
   if (severity == LOG_INFO)
     return;
-  ++NS(n_msgs);
+  ++status_hb_simple_n_msgs;
 
   tt_int_op(severity, OP_EQ, LOG_NOTICE);
-  tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+  tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
   tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
   tt_ptr_op(suffix, OP_EQ, NULL);
   tt_str_op(format, OP_EQ,
@@ -591,54 +600,69 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_simple_server_mode(const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, calls_log_accounting)
-
 /*
  * Tests that log_heartbeat() correctly logs heartbeat information
  * and accounting information when configured.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
-NS_DECL(or_state_t *, get_or_state, (void));
-NS_DECL(int, accounting_is_enabled, (const or_options_t *options));
-NS_DECL(time_t, accounting_get_end_time, (void));
-
-static or_state_t * NS(mock_state) = NULL;
-static or_options_t * NS(mock_options) = NULL;
+static double status_hb_calls_log_accounting_tls_get_write_overhead_ratio(
+                       void);
+static int status_hb_calls_log_accounting_we_are_hibernating(void);
+static int status_hb_calls_log_accounting_public_server_mode(
+                      const or_options_t *options);
+static long status_hb_calls_log_accounting_get_uptime(void);
+static uint64_t status_hb_calls_log_accounting_get_bytes_read(void);
+static uint64_t status_hb_calls_log_accounting_get_bytes_written(void);
+static void status_hb_calls_log_accounting_logv(
+                      int severity, log_domain_mask_t domain,
+                      const char *funcname, const char *suffix,
+                      const char *format, va_list ap);
+static int status_hb_calls_log_accounting_logv_called = 0;
+static int status_hb_calls_log_accounting_server_mode(
+                      const or_options_t *options);
+static or_state_t * status_hb_calls_log_accounting_get_or_state(void);
+static int status_hb_calls_log_accounting_accounting_is_enabled(
+                      const or_options_t *options);
+static time_t status_hb_calls_log_accounting_accounting_get_end_time(void);
+
+static or_state_t * status_hb_calls_log_accounting_mock_state = NULL;
+static or_options_t * status_hb_calls_log_accounting_mock_options = NULL;
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_calls_log_accounting(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
-  NS_MOCK(get_or_state);
-  NS_MOCK(accounting_is_enabled);
-  NS_MOCK(accounting_get_end_time);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_calls_log_accounting_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_calls_log_accounting_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_calls_log_accounting_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_calls_log_accounting_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_calls_log_accounting_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_calls_log_accounting_get_bytes_written);
+  MOCK(logv,
+       status_hb_calls_log_accounting_logv);
+  MOCK(server_mode,
+       status_hb_calls_log_accounting_server_mode);
+  MOCK(get_or_state,
+       status_hb_calls_log_accounting_get_or_state);
+  MOCK(accounting_is_enabled,
+       status_hb_calls_log_accounting_accounting_is_enabled);
+  MOCK(accounting_get_end_time,
+       status_hb_calls_log_accounting_accounting_get_end_time);
 
   log_global_min_severity_ = LOG_DEBUG;
 
@@ -646,37 +670,37 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 3);
+  tt_int_op(status_hb_calls_log_accounting_logv_called, OP_EQ, 3);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
-    NS_UNMOCK(accounting_is_enabled);
-    NS_UNMOCK(accounting_get_end_time);
-    tor_free_(NS(mock_state));
-    tor_free_(NS(mock_options));
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
+    UNMOCK(accounting_is_enabled);
+    UNMOCK(accounting_get_end_time);
+    tor_free_(status_hb_calls_log_accounting_mock_state);
+    tor_free_(status_hb_calls_log_accounting_mock_options);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_calls_log_accounting_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_calls_log_accounting_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_calls_log_accounting_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -684,32 +708,32 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_calls_log_accounting_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_calls_log_accounting_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_calls_log_accounting_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain,
+status_hb_calls_log_accounting_logv(int severity, log_domain_mask_t domain,
   const char *funcname, const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_calls_log_accounting_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -723,7 +747,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_accounting"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -750,11 +774,11 @@ NS(logv)(int severity, log_domain_mask_t domain,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_calls_log_accounting_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_calls_log_accounting_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -762,7 +786,8 @@ NS(server_mode)(const or_options_t *options)
 }
 
 static int
-NS(accounting_is_enabled)(const or_options_t *options)
+status_hb_calls_log_accounting_accounting_is_enabled(
+                                        const or_options_t *options)
 {
   (void)options;
 
@@ -770,55 +795,71 @@ NS(accounting_is_enabled)(const or_options_t *options)
 }
 
 static time_t
-NS(accounting_get_end_time)(void)
+status_hb_calls_log_accounting_accounting_get_end_time(void)
 {
   return 60;
 }
 
 static or_state_t *
-NS(get_or_state)(void)
+status_hb_calls_log_accounting_get_or_state(void)
 {
-  NS(mock_state) = tor_malloc_zero(sizeof(or_state_t));
-  NS(mock_state)->AccountingBytesReadInInterval = 0;
-  NS(mock_state)->AccountingBytesWrittenInInterval = 0;
-
-  return NS(mock_state);
+  status_hb_calls_log_accounting_mock_state =
+    tor_malloc_zero(sizeof(or_state_t));
+  status_hb_calls_log_accounting_mock_state
+    ->AccountingBytesReadInInterval = 0;
+  status_hb_calls_log_accounting_mock_state
+    ->AccountingBytesWrittenInInterval = 0;
+
+  return status_hb_calls_log_accounting_mock_state;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, packaged_cell_fullness)
-
 /*
  * Tests that log_heartbeat() correctly logs packaged cell
  * fullness information.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
-NS_DECL(int, accounting_is_enabled, (const or_options_t *options));
+static double status_hb_packaged_cell_fullness_tls_get_write_overhead_ratio(
+                          void);
+static int status_hb_packaged_cell_fullness_we_are_hibernating(void);
+static int status_hb_packaged_cell_fullness_public_server_mode(
+                          const or_options_t *options);
+static long status_hb_packaged_cell_fullness_get_uptime(void);
+static uint64_t status_hb_packaged_cell_fullness_get_bytes_read(void);
+static uint64_t status_hb_packaged_cell_fullness_get_bytes_written(void);
+static void status_hb_packaged_cell_fullness_logv(
+                          int severity, log_domain_mask_t domain,
+                          const char *funcname, const char *suffix,
+                          const char *format, va_list ap);
+static int status_hb_packaged_cell_fullness_logv_called = 0;
+static int status_hb_packaged_cell_fullness_server_mode(
+                          const or_options_t *options);
+static int status_hb_packaged_cell_fullness_accounting_is_enabled(
+                          const or_options_t *options);
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_packaged_cell_fullness(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
-  NS_MOCK(accounting_is_enabled);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_packaged_cell_fullness_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_packaged_cell_fullness_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_packaged_cell_fullness_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_packaged_cell_fullness_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_packaged_cell_fullness_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_packaged_cell_fullness_get_bytes_written);
+  MOCK(logv,
+       status_hb_packaged_cell_fullness_logv);
+  MOCK(server_mode,
+       status_hb_packaged_cell_fullness_server_mode);
+  MOCK(accounting_is_enabled,
+       status_hb_packaged_cell_fullness_accounting_is_enabled);
   log_global_min_severity_ = LOG_DEBUG;
 
   stats_n_data_bytes_packaged = RELAY_PAYLOAD_SIZE;
@@ -827,36 +868,37 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 2);
+  tt_int_op(status_hb_packaged_cell_fullness_logv_called, OP_EQ, 2);
 
   done:
     stats_n_data_bytes_packaged = 0;
     stats_n_data_cells_packaged = 0;
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
-    NS_UNMOCK(accounting_is_enabled);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
+    UNMOCK(accounting_is_enabled);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_packaged_cell_fullness_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_packaged_cell_fullness_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_packaged_cell_fullness_public_server_mode(
+                                     const or_options_t *options)
 {
   (void)options;
 
@@ -864,32 +906,33 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_packaged_cell_fullness_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_packaged_cell_fullness_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_packaged_cell_fullness_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
+status_hb_packaged_cell_fullness_logv(int severity,
+    log_domain_mask_t domain, const char *funcname,
     const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_packaged_cell_fullness_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -903,7 +946,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -918,11 +961,11 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_packaged_cell_fullness_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_packaged_cell_fullness_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -930,47 +973,60 @@ NS(server_mode)(const or_options_t *options)
 }
 
 static int
-NS(accounting_is_enabled)(const or_options_t *options)
+status_hb_packaged_cell_fullness_accounting_is_enabled(
+                                             const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, tls_write_overhead)
-
 /*
  * Tests that log_heartbeat() correctly logs the TLS write overhead information
  * when the TLS write overhead ratio exceeds 1.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
-NS_DECL(int, accounting_is_enabled, (const or_options_t *options));
+static double status_hb_tls_write_overhead_tls_get_write_overhead_ratio(void);
+static int status_hb_tls_write_overhead_we_are_hibernating(void);
+static int status_hb_tls_write_overhead_public_server_mode(
+               const or_options_t *options);
+static long status_hb_tls_write_overhead_get_uptime(void);
+static uint64_t status_hb_tls_write_overhead_get_bytes_read(void);
+static uint64_t status_hb_tls_write_overhead_get_bytes_written(void);
+static void status_hb_tls_write_overhead_logv(
+               int severity, log_domain_mask_t domain,
+               const char *funcname, const char *suffix,
+               const char *format, va_list ap);
+static int status_hb_tls_write_overhead_logv_called = 0;
+static int status_hb_tls_write_overhead_server_mode(
+               const or_options_t *options);
+static int status_hb_tls_write_overhead_accounting_is_enabled(
+               const or_options_t *options);
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_tls_write_overhead(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
-  NS_MOCK(accounting_is_enabled);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_tls_write_overhead_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_tls_write_overhead_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_tls_write_overhead_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_tls_write_overhead_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_tls_write_overhead_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_tls_write_overhead_get_bytes_written);
+  MOCK(logv,
+       status_hb_tls_write_overhead_logv);
+  MOCK(server_mode,
+       status_hb_tls_write_overhead_server_mode);
+  MOCK(accounting_is_enabled,
+       status_hb_tls_write_overhead_accounting_is_enabled);
   stats_n_data_cells_packaged = 0;
   log_global_min_severity_ = LOG_DEBUG;
 
@@ -978,34 +1034,34 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 2);
+  tt_int_op(status_hb_tls_write_overhead_logv_called, OP_EQ, 2);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
-    NS_UNMOCK(accounting_is_enabled);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
+    UNMOCK(accounting_is_enabled);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_tls_write_overhead_tls_get_write_overhead_ratio(void)
 {
   return 2.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_tls_write_overhead_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_tls_write_overhead_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -1013,32 +1069,32 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_tls_write_overhead_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_tls_write_overhead_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_tls_write_overhead_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain,
+status_hb_tls_write_overhead_logv(int severity, log_domain_mask_t domain,
   const char *funcname, const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_tls_write_overhead_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1052,7 +1108,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1067,11 +1123,11 @@ NS(logv)(int severity, log_domain_mask_t domain,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_tls_write_overhead_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_tls_write_overhead_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -1079,24 +1135,26 @@ NS(server_mode)(const or_options_t *options)
 }
 
 static int
-NS(accounting_is_enabled)(const or_options_t *options)
+status_hb_tls_write_overhead_accounting_is_enabled(const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t status_tests[] = {
-  TEST_CASE(count_circuits),
-  TEST_CASE(secs_to_uptime),
-  TEST_CASE(bytes_to_usage),
-  TEST_CASE_ASPECT(log_heartbeat, fails),
-  TEST_CASE_ASPECT(log_heartbeat, simple),
-  TEST_CASE_ASPECT(log_heartbeat, not_in_consensus),
-  TEST_CASE_ASPECT(log_heartbeat, calls_log_accounting),
-  TEST_CASE_ASPECT(log_heartbeat, packaged_cell_fullness),
-  TEST_CASE_ASPECT(log_heartbeat, tls_write_overhead),
+  { "count_circuits", test_status_count_circuits, TT_FORK, NULL, NULL },
+  { "secs_to_uptime", test_status_secs_to_uptime, TT_FORK, NULL, NULL },
+  { "bytes_to_usage", test_status_bytes_to_usage, TT_FORK, NULL, NULL },
+  { "hb_fails", test_status_hb_fails, TT_FORK, NULL, NULL },
+  { "hb_simple", test_status_hb_simple, TT_FORK, NULL, NULL },
+  { "hb_not_in_consensus", test_status_hb_not_in_consensus,
+    TT_FORK, NULL, NULL },
+  { "hb_calls_log_accounting", test_status_hb_calls_log_accounting,
+    TT_FORK, NULL, NULL },
+  { "hb_packaged_cell_fullness", test_status_hb_packaged_cell_fullness,
+    TT_FORK, NULL, NULL },
+  { "hb_tls_write_overhead", test_status_hb_tls_write_overhead,
+    TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c
index 24e45c7428..eb3779cfee 100644
--- a/src/test/test_storagedir.c
+++ b/src/test/test_storagedir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c
index baddf8d66e..f97af55d17 100644
--- a/src/test/test_switch_id.c
+++ b/src/test/test_switch_id.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -32,6 +32,46 @@ static const struct {
 };
 
 #if !defined(_WIN32)
+
+/* Returns the first port that we think we can bind to without special
+ * permissions. Usually this function returns 1024. */
+static uint16_t
+unprivileged_port_range_start(void)
+{
+  uint16_t result = 1024;
+
+#if defined(__linux__)
+  char *content = NULL;
+
+  content = read_file_to_str(
+              "/proc/sys/net/ipv4/ip_unprivileged_port_start",
+              0,
+              NULL);
+
+  if (content != NULL) {
+    int ok = 1;
+    uint16_t tmp_result;
+
+    tmp_result = (uint16_t)tor_parse_long(content, 10, 0, 65535, &ok, NULL);
+
+    if (ok) {
+      result = tmp_result;
+    } else {
+      fprintf(stderr,
+              "Unable to convert ip_unprivileged_port_start to integer: %s\n",
+              content);
+    }
+  }
+
+  tor_free(content);
+#endif /* defined(__linux__) */
+
+  return result;
+}
+
+#define PORT_TEST_RANGE_START 600
+#define PORT_TEST_RANGE_END   1024
+
 /* 0 on no, 1 on yes, -1 on failure. */
 static int
 check_can_bind_low_ports(void)
@@ -41,7 +81,7 @@ check_can_bind_low_ports(void)
   memset(&sin, 0, sizeof(sin));
   sin.sin_family = AF_INET;
 
-  for (port = 600; port < 1024; ++port) {
+  for (port = PORT_TEST_RANGE_START; port < PORT_TEST_RANGE_END; ++port) {
     sin.sin_port = htons(port);
     tor_socket_t fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
     if (! SOCKET_OK(fd)) {
@@ -87,7 +127,7 @@ main(int argc, char **argv)
 
   fprintf(stderr, "This test is not supported on your OS.\n");
   return 77;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *username;
   const char *testname;
   if (argc != 3) {
@@ -149,10 +189,24 @@ main(int argc, char **argv)
       /* Succeed if we can do a setuid with capability retention, and doing so
        * does not make us lose the ability to bind low ports */
     {
-      int keepcaps = (test_id == TEST_SETUID_KEEPCAPS);
+      const int keepcaps = (test_id == TEST_SETUID_KEEPCAPS);
       okay = switch_id(username, keepcaps ? SWITCH_ID_KEEP_BINDLOW : 0) == 0;
+
       if (okay) {
-        okay = check_can_bind_low_ports() == keepcaps;
+        /* Only run this check if there are ports we may not be able to bind
+         * to. */
+        const uint16_t min_port = unprivileged_port_range_start();
+
+        if (min_port >= PORT_TEST_RANGE_START &&
+            min_port < PORT_TEST_RANGE_END) {
+          okay = check_can_bind_low_ports() == keepcaps;
+        } else {
+          fprintf(stderr,
+                  "Skipping check for whether we can bind to any "
+                  "privileged ports as the user system seems to "
+                  "allow us to bind to ports even without any "
+                  "capabilities set.\n");
+        }
       }
       break;
     }
diff --git a/src/test/test_switch_id.sh b/src/test/test_switch_id.sh
index 79c44f2eb1..b13bf7602f 100755
--- a/src/test/test_switch_id.sh
+++ b/src/test/test_switch_id.sh
@@ -1,11 +1,11 @@
 #!/bin/sh
 
-if test "`id -u`" != '0'; then
+if test "$(id -u)" != '0'; then
     echo "This test only works when run as root. Skipping." >&2
     exit 77
 fi
 
-if test "`id -u nobody`" = ""; then
+if test "$(id -u nobody)" = ""; then
     echo "This test requires that your system have a 'nobody' user. Sorry." >&2
     exit 1
 fi
diff --git a/src/test/test_threads.c b/src/test/test_threads.c
index 4a5ecc6fae..d5a1834aef 100644
--- a/src/test/test_threads.c
+++ b/src/test/test_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -155,7 +155,7 @@ test_threads_basic(void *arg)
     tor_mutex_free(thread_test_start2_);
 }
 
-typedef struct cv_testinfo_s {
+typedef struct cv_testinfo_t {
   tor_cond_t *cond;
   tor_mutex_t *mutex;
   int value;
diff --git a/src/test/test_token_bucket.c b/src/test/test_token_bucket.c
new file mode 100644
index 0000000000..cf315f2944
--- /dev/null
+++ b/src/test/test_token_bucket.c
@@ -0,0 +1,152 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_bwmgt.c
+ * \brief tests for bandwidth management / token bucket functions
+ */
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "lib/evloop/token_bucket.h"
+
+// an imaginary time, in timestamp units. Chosen so it will roll over.
+static const uint32_t START_TS = UINT32_MAX - 1000;
+static const uint32_t RATE = 10;
+static const uint32_t BURST = 50;
+
+static void
+test_token_bucket_ctr_init(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+  tt_uint_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_adjust(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Increase burst. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST * 2);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST * 2);
+
+  /* Decrease burst but still above bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST + 10);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST + 10);
+
+  /* Decrease burst below bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST - 1);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST - 1);
+
+  /* Change rate. */
+  token_bucket_ctr_adjust(&tb, RATE * 2, BURST);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE * 2);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_dec(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Simple decrement by one. */
+  tt_uint_op(0, OP_EQ, token_bucket_ctr_dec(&tb, 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+
+  /* Down to 0. Becomes empty. */
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST - 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, 0);
+
+  /* Reset and try to underflow. */
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST + 1));
+  tt_int_op(tb.counter.bucket, OP_EQ, -1);
+
+  /* Keep underflowing shouldn't flag the bucket as empty. */
+  tt_uint_op(false, OP_EQ, token_bucket_ctr_dec(&tb, BURST));
+  tt_int_op(tb.counter.bucket, OP_EQ, - (int32_t) (BURST + 1));
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_refill(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Reduce of half the bucket and let a single second go before refill. */
+  token_bucket_ctr_dec(&tb, BURST / 2);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST / 2);
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* No time change, nothing should move. */
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* Add 99 seconds, bucket should be back to a full BURST. */
+  token_bucket_ctr_refill(&tb, START_TS + 99);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 99);
+
+  /* Empty bucket at once. */
+  token_bucket_ctr_dec(&tb, BURST);
+  tt_int_op(tb.counter.bucket, OP_EQ, 0);
+  /* On second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 100);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 100);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE);
+  /* A second second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 101);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 101);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE * 2);
+
+ done:
+  ;
+}
+
+#define TOKEN_BUCKET(name)                                          \
+  { #name, test_token_bucket_ ## name , 0, NULL, NULL }
+
+struct testcase_t token_bucket_tests[] = {
+  TOKEN_BUCKET(ctr_init),
+  TOKEN_BUCKET(ctr_adjust),
+  TOKEN_BUCKET(ctr_dec),
+  TOKEN_BUCKET(ctr_refill),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c
index 853abc4f91..12ba873650 100644
--- a/src/test/test_tortls.c
+++ b/src/test/test_tortls.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TORTLS_PRIVATE
@@ -225,12 +225,12 @@ test_tortls_tor_tls_get_error(void *data)
 
  done:
   UNMOCK(tor_tls_cert_matches_key);
-  NS_UNMOCK(logv);
+  UNMOCK(logv);
   crypto_pk_free(key1);
   crypto_pk_free(key2);
   tor_tls_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_x509_cert_get_id_digests(void *ignored)
@@ -347,7 +347,7 @@ test_tortls_server_got_renegotiate(void *ignored)
  done:
   tor_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_evaluate_ecgroup_for_tls(void *ignored)
@@ -598,7 +598,7 @@ struct testcase_t tortls_tests[] = {
   LOCAL_TEST_CASE(get_forced_write_size, 0),
   LOCAL_TEST_CASE(used_v1_handshake, TT_FORK),
   LOCAL_TEST_CASE(server_got_renegotiate, 0),
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   LOCAL_TEST_CASE(evaluate_ecgroup_for_tls, 0),
   LOCAL_TEST_CASE(double_init, TT_FORK),
   LOCAL_TEST_CASE(address, TT_FORK),
diff --git a/src/test/test_tortls.h b/src/test/test_tortls.h
index 1a8b117d0f..21c6fa0a8f 100644
--- a/src/test/test_tortls.h
+++ b/src/test/test_tortls.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TEST_TORTLS_H
@@ -10,4 +10,4 @@ extern const char *notCompletelyValidCertString;
 extern const char *validCertString;
 extern const char *caCertString;
 
-#endif
+#endif /* !defined(TEST_TORTLS_H) */
diff --git a/src/test/test_tortls_openssl.c b/src/test/test_tortls_openssl.c
index 81c65d7446..c1a87fbb4f 100644
--- a/src/test/test_tortls_openssl.c
+++ b/src/test/test_tortls_openssl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TORTLS_PRIVATE
@@ -16,7 +16,7 @@
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
  * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/opensslv.h>
 
@@ -29,7 +29,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/evp.h>
 #include <openssl/bn.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "core/or/or.h"
 #include "lib/log/log.h"
@@ -46,8 +46,6 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include "test/log_test_helpers.h"
 #include "test/test_tortls.h"
 
-#define NS_MODULE tortls
-
 #ifndef HAVE_SSL_STATE
 #define OPENSSL_OPAQUE
 #endif
@@ -123,8 +121,6 @@ test_tortls_tor_tls_new(void *data)
   tor_tls_free_all();
 }
 
-#define NS_MODULE tortls
-
 static void
 library_init(void)
 {
@@ -133,7 +129,7 @@ library_init(void)
 #else
   SSL_library_init();
   SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 }
 
 static void
@@ -476,7 +472,7 @@ fake_x509_free(X509 *cert)
     tor_free(cert);
   }
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
diff --git a/src/test/test_util.c b/src/test/test_util.c
index 7f07370998..00626c7ec0 100644
--- a/src/test/test_util.c
+++ b/src/test/test_util.c
@@ -1,27 +1,26 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#define COMPAT_PRIVATE
 #define COMPAT_TIME_PRIVATE
-#define CONTROL_PRIVATE
-#define UTIL_PRIVATE
 #define UTIL_MALLOC_PRIVATE
-#define SOCKET_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define PROCESS_WIN32_PRIVATE
 #include "lib/testsupport/testsupport.h"
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "feature/control/control.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/transports.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "test/test.h"
 #include "lib/memarea/memarea.h"
 #include "lib/process/waitpid.h"
+#include "lib/process/process_win32.h"
 #include "test/log_test_helpers.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_zstd.h"
@@ -30,8 +29,8 @@
 #include "lib/fs/winlib.h"
 #include "lib/process/env.h"
 #include "lib/process/pidfile.h"
-#include "lib/process/subprocess.h"
 #include "lib/intmath/weakrng.h"
+#include "lib/intmath/muldiv.h"
 #include "lib/thread/numcpus.h"
 #include "lib/math/fp.h"
 #include "lib/math/laplace.h"
@@ -39,6 +38,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/socketpair.h"
+#include "lib/malloc/map_anon.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
@@ -58,6 +58,12 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
 
 #ifdef _WIN32
 #include <tchar.h>
@@ -66,9 +72,36 @@
 #include <ctype.h>
 #include <float.h>
 
+/* These platforms don't have meaningful pwdb or homedirs. */
+#if defined(_WIN32) || defined(__ANDROID__)
+#define DISABLE_PWDB_TESTS
+#endif
+
 #define INFINITY_DBL ((double)INFINITY)
 #define NAN_DBL ((double)NAN)
 
+/** Test the tor_isinf() wrapper */
+static void
+test_tor_isinf(void *arg)
+{
+  (void) arg;
+
+  tt_assert(tor_isinf(INFINITY_DBL));
+
+  tt_assert(!tor_isinf(NAN_DBL));
+  tt_assert(!tor_isinf(DBL_EPSILON));
+  tt_assert(!tor_isinf(DBL_MAX));
+  tt_assert(!tor_isinf(DBL_MIN));
+
+  tt_assert(!tor_isinf(0.0));
+  tt_assert(!tor_isinf(0.1));
+  tt_assert(!tor_isinf(3));
+  tt_assert(!tor_isinf(3.14));
+
+ done:
+  ;
+}
+
 /* XXXX this is a minimal wrapper to make the unit tests compile with the
  * changed tor_timegm interface. */
 static time_t
@@ -322,6 +355,7 @@ test_util_write_chunks_to_file(void *arg)
   tor_free(temp_str);
 }
 
+#ifndef COCCI
 #define _TFE(a, b, f)  tt_int_op((a).f, OP_EQ, (b).f)
 /** test the minimum set of struct tm fields needed for a unique epoch value
  * this is also the set we use to test tor_timegm */
@@ -334,6 +368,7 @@ test_util_write_chunks_to_file(void *arg)
             _TFE(a, b, tm_min ); \
             _TFE(a, b, tm_sec ); \
           TT_STMT_END
+#endif /* !defined(COCCI) */
 
 static void
 test_util_time(void *arg)
@@ -452,7 +487,6 @@ test_util_time(void *arg)
 
 /* Assume tv_usec is an unsigned integer until proven otherwise */
 #define TV_USEC_MAX UINT_MAX
-#define TOR_USEC_PER_SEC 1000000
 
   /* Overflows in the result type */
 
@@ -1864,7 +1898,57 @@ test_util_config_line_crlf(void *arg)
   tor_free(k); tor_free(v);
 }
 
-#ifndef _WIN32
+static void
+test_util_config_line_partition(void *arg)
+{
+  (void)arg;
+  config_line_t *lines = NULL, *orig, *rest = NULL;
+
+  config_line_append(&lines, "Header", "X");
+  config_line_append(&lines, "Item", "Y");
+  config_line_append(&lines, "Thing", "Z");
+
+  config_line_append(&lines, "HEADER", "X2");
+
+  config_line_append(&lines, "header", "X3");
+  config_line_append(&lines, "Item3", "Foob");
+
+  /* set up h2 and h3 to point to the places where we hope the headers will
+     be. */
+  config_line_t *h2 = lines->next->next->next;
+  config_line_t *h3 = h2->next;
+  tt_str_op(h2->key, OP_EQ, "HEADER");
+  tt_str_op(h3->key, OP_EQ, "header");
+
+  orig = lines;
+  rest = config_lines_partition(lines, "Header");
+  tt_ptr_op(lines, OP_EQ, orig);
+  tt_ptr_op(rest, OP_EQ, h2);
+  tt_str_op(lines->next->key, OP_EQ, "Item");
+  tt_str_op(lines->next->next->key, OP_EQ, "Thing");
+  tt_ptr_op(lines->next->next->next, OP_EQ, NULL);
+  config_free_lines(lines);
+
+  orig = lines = rest;
+  rest = config_lines_partition(lines, "Header");
+  tt_ptr_op(lines, OP_EQ, orig);
+  tt_ptr_op(rest, OP_EQ, h3);
+  tt_ptr_op(lines->next, OP_EQ, NULL);
+  config_free_lines(lines);
+
+  orig = lines = rest;
+  rest = config_lines_partition(lines, "Header");
+  tt_ptr_op(lines, OP_EQ, orig);
+  tt_ptr_op(rest, OP_EQ, NULL);
+  tt_str_op(lines->next->key, OP_EQ, "Item3");
+  tt_ptr_op(lines->next->next, OP_EQ, NULL);
+
+ done:
+  config_free_lines(lines);
+  config_free_lines(rest);
+}
+
+#ifndef DISABLE_PWDB_TESTS
 static void
 test_util_expand_filename(void *arg)
 {
@@ -1961,7 +2045,7 @@ test_util_expand_filename(void *arg)
  done:
   tor_free(str);
 }
-#endif /* !defined(_WIN32) */
+#endif /* !defined(DISABLE_PWDB_TESTS) */
 
 /** Test tor_escape_str_for_pt_args(). */
 static void
@@ -2106,14 +2190,14 @@ test_util_strmisc(void *arg)
   /* Test mem_is_zero */
   memset(buf,0,128);
   buf[128] = 'x';
-  tt_assert(tor_mem_is_zero(buf, 10));
-  tt_assert(tor_mem_is_zero(buf, 20));
-  tt_assert(tor_mem_is_zero(buf, 128));
-  tt_assert(!tor_mem_is_zero(buf, 129));
+  tt_assert(fast_mem_is_zero(buf, 10));
+  tt_assert(fast_mem_is_zero(buf, 20));
+  tt_assert(fast_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 129));
   buf[60] = (char)255;
-  tt_assert(!tor_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 128));
   buf[0] = (char)1;
-  tt_assert(!tor_mem_is_zero(buf, 10));
+  tt_assert(!fast_mem_is_zero(buf, 10));
 
   /* Test 'escaped' */
   tt_ptr_op(escaped(NULL), OP_EQ, NULL);
@@ -2227,15 +2311,6 @@ test_util_strmisc(void *arg)
   tt_int_op(strcmp_opt(NULL,  "foo"), OP_LT, 0);
   tt_int_op(strcmp_opt("foo", NULL),  OP_GT, 0);
 
-  /* Test strcmp_len */
-  tt_int_op(strcmp_len("foo", "bar", 3),   OP_GT, 0);
-  tt_int_op(strcmp_len("foo", "bar", 2),   OP_LT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */
-  tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0);  /* Really stop at len */
-  tt_int_op(strcmp_len("blah", "", 4),     OP_GT, 0);
-  tt_int_op(strcmp_len("blah", "", 0),    OP_EQ, 0);
-
  done:
   tor_free(cp_tmp);
 }
@@ -3817,7 +3892,7 @@ test_util_memarea(void *arg)
   tt_int_op(((uintptr_t)p3) % sizeof(void*),OP_EQ, 0);
   tt_assert(!memarea_owns_ptr(area, p3+8192));
   tt_assert(!memarea_owns_ptr(area, p3+30));
-  tt_assert(tor_mem_is_zero(p2, 52));
+  tt_assert(fast_mem_is_zero(p2, 52));
   /* Make sure we don't overalign. */
   p1 = memarea_alloc(area, 1);
   p2 = memarea_alloc(area, 1);
@@ -4098,6 +4173,13 @@ test_util_string_is_utf8(void *ptr)
   tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7));
   tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9));
 
+  /* Test the utf8_no_bom function */
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7));
+  tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect",
+                                            strlen("loveandrespect")));
+
   // Validate exactly 'len' bytes.
   tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2));
   tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6));
@@ -4125,10 +4207,43 @@ test_util_string_is_utf8(void *ptr)
   tt_int_op(0, OP_EQ, string_is_utf8("\xed\xbf\xbf", 3));
   tt_int_op(1, OP_EQ, string_is_utf8("\xee\x80\x80", 3));
 
-  // The maximum legal codepoint, 10FFFF.
+  // The minimum legal codepoint, 0x00.
+  tt_int_op(1, OP_EQ, string_is_utf8("\0", 1));
+
+  // The maximum legal codepoint, 0x10FFFF.
   tt_int_op(1, OP_EQ, string_is_utf8("\xf4\x8f\xbf\xbf", 4));
   tt_int_op(0, OP_EQ, string_is_utf8("\xf4\x90\x80\x80", 4));
 
+  /* Test cases that vary between programming languages /
+   * UTF-8 implementations.
+   * Source: POC||GTFO 19, page 43
+   * https://www.alchemistowl.org/pocorgtfo/
+   */
+
+  // Invalid (in most implementations)
+  // surrogate
+  tt_int_op(0, OP_EQ, string_is_utf8("\xed\xa0\x81", 3));
+  // nullsurrog
+  tt_int_op(0, OP_EQ, string_is_utf8("\x30\x00\xed\xa0\x81", 5));
+  // threehigh
+  tt_int_op(0, OP_EQ, string_is_utf8("\xed\xbf\xbf", 3));
+  // fourhigh
+  tt_int_op(0, OP_EQ, string_is_utf8("\xf4\x90\xbf\xbf", 4));
+  // fivebyte
+  tt_int_op(0, OP_EQ, string_is_utf8("\xfb\x80\x80\x80\x80", 5));
+  // sixbyte
+  tt_int_op(0, OP_EQ, string_is_utf8("\xfd\x80\x80\x80\x80", 5));
+  // sixhigh
+  tt_int_op(0, OP_EQ, string_is_utf8("\xfd\xbf\xbf\xbf\xbf", 5));
+
+  // Valid (in most implementations)
+  // fourbyte
+  tt_int_op(1, OP_EQ, string_is_utf8("\xf0\x90\x8d\x88", 4));
+  // fourbyte2
+  tt_int_op(1, OP_EQ, string_is_utf8("\xf0\xbf\xbf\xbf", 4));
+  // nullbyte
+  tt_int_op(1, OP_EQ, string_is_utf8("\x30\x31\x32\x00\x33", 5));
+
  done:
   ;
 }
@@ -4368,204 +4483,6 @@ test_util_load_win_lib(void *ptr)
 }
 #endif /* defined(_WIN32) */
 
-#ifndef _WIN32
-static void
-clear_hex_errno(char *hex_errno)
-{
-  memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1);
-}
-
-static void
-test_util_exit_status(void *ptr)
-{
-  /* Leave an extra byte for a \0 so we can do string comparison */
-  char hex_errno[HEX_ERRNO_SIZE + 1];
-  int n;
-
-  (void)ptr;
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0, hex_errno);
-  tt_str_op("0/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-#if SIZEOF_INT == 4
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x80000000, hex_errno);
-  tt_str_op("FF/-80000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#elif SIZEOF_INT == 8
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno);
-  tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#endif /* SIZEOF_INT == 4 || ... */
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x7F, 0, hex_errno);
-  tt_str_op("7F/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x08, -0x242, hex_errno);
-  tt_str_op("8/-242\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
- done:
-  ;
-}
-#endif /* !defined(_WIN32) */
-
-#ifndef _WIN32
-static void
-test_util_string_from_pipe(void *ptr)
-{
-  int test_pipe[2] = {-1, -1};
-  int retval = 0;
-  enum stream_status status = IO_STREAM_TERM;
-  ssize_t retlen;
-  char buf[4] = { 0 };
-
-  (void)ptr;
-
-  errno = 0;
-
-  /* Set up a pipe to test on */
-  retval = pipe(test_pipe);
-  tt_int_op(retval, OP_EQ, 0);
-
-  /* Send in a string. */
-  retlen = write(test_pipe[1], "ABC", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "ABC");
-  errno = 0;
-
-  /* Send in a string that contains a nul. */
-  retlen = write(test_pipe[1], "AB\0", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a nul only. */
-  retlen = write(test_pipe[1], "\0", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string that contains a trailing newline. */
-  retlen = write(test_pipe[1], "AB\n", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a newline only. */
-  retlen = write(test_pipe[1], "\n", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string and check that we nul terminate return values. */
-  retlen = write(test_pipe[1], "AAA", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AAA");
-  tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf));
-  errno = 0;
-
-  retlen = write(test_pipe[1], "B", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  memset(buf, '\xff', sizeof(buf));
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "B");
-  tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf));
-  errno = 0;
-
-  /* Send in multiple lines. */
-  retlen = write(test_pipe[1], "A\nB", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "A\nB");
-  errno = 0;
-
-  /* Send in a line and close */
-  retlen = write(test_pipe[1], "AB", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retval = close(test_pipe[1]);
-  tt_int_op(retval, OP_EQ, 0);
-  test_pipe[1] = -1;
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Check for EOF */
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_CLOSED);
-  errno = 0;
-
- done:
-  if (test_pipe[0] != -1)
-    close(test_pipe[0]);
-  if (test_pipe[1] != -1)
-    close(test_pipe[1]);
-}
-
-#endif /* !defined(_WIN32) */
-
 /**
  * Test for format_hex_number_sigsafe()
  */
@@ -4660,57 +4577,6 @@ test_util_format_dec_number(void *ptr)
   return;
 }
 
-/**
- * Test that we can properly format a Windows command line
- */
-static void
-test_util_join_win_cmdline(void *ptr)
-{
-  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
-   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
-   * will try to only generate simple cases for the child process to parse;
-   * i.e. we never embed quoted strings in arguments. */
-
-  const char *argvs[][4] = {
-    {"a", "bb", "CCC", NULL}, // Normal
-    {NULL, NULL, NULL, NULL}, // Empty argument list
-    {"", NULL, NULL, NULL}, // Empty argument
-    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
-    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
-    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
-    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
-    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
-    { NULL } // Terminator
-  };
-
-  const char *cmdlines[] = {
-    "a bb CCC",
-    "",
-    "\"\"",
-    "\\\"a b\\\"b CCC\\\"",
-    "\"a\tbc\" \"dd  dd\" E",
-    "a\\\\\\b \"de fg\" H",
-    "a\\\\\\\"b \\c D\\",
-    "\"a\\\\b c\" d E",
-    NULL // Terminator
-  };
-
-  int i;
-  char *joined_argv = NULL;
-
-  (void)ptr;
-
-  for (i=0; cmdlines[i]!=NULL; i++) {
-    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
-    joined_argv = tor_join_win_cmdline(argvs[i]);
-    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
-    tor_free(joined_argv);
-  }
-
- done:
-  tor_free(joined_argv);
-}
-
 #define MAX_SPLIT_LINE_COUNT 4
 struct split_lines_test_t {
   const char *orig_line; // Line to be split (may contain \0's)
@@ -4718,67 +4584,6 @@ struct split_lines_test_t {
   const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines
 };
 
-/**
- * Test that we properly split a buffer into lines
- */
-static void
-test_util_split_lines(void *ptr)
-{
-  /* Test cases. orig_line of last test case must be NULL.
-   * The last element of split_line[i] must be NULL. */
-  struct split_lines_test_t tests[] = {
-    {"", 0, {NULL}},
-    {"foo", 3, {"foo", NULL}},
-    {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}},
-    {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}},
-    {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}},
-    {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}},
-    {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}},
-    {NULL, 0, { NULL }}
-  };
-
-  int i, j;
-  char *orig_line=NULL;
-  smartlist_t *sl=NULL;
-
-  (void)ptr;
-
-  for (i=0; tests[i].orig_line; i++) {
-    sl = smartlist_new();
-    /* Allocate space for string and trailing NULL */
-    orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1);
-    tor_split_lines(sl, orig_line, tests[i].orig_length);
-
-    j = 0;
-    log_info(LD_GENERAL, "Splitting test %d of length %d",
-             i, tests[i].orig_length);
-    SMARTLIST_FOREACH_BEGIN(sl, const char *, line) {
-      /* Check we have not got too many lines */
-      tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j);
-      /* Check that there actually should be a line here */
-      tt_ptr_op(tests[i].split_line[j], OP_NE, NULL);
-      log_info(LD_GENERAL, "Line %d of test %d, should be <%s>",
-               j, i, tests[i].split_line[j]);
-      /* Check that the line is as expected */
-      tt_str_op(line,OP_EQ, tests[i].split_line[j]);
-      j++;
-    } SMARTLIST_FOREACH_END(line);
-    /* Check that we didn't miss some lines */
-    tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]);
-    tor_free(orig_line);
-    smartlist_free(sl);
-    sl = NULL;
-  }
-
- done:
-  tor_free(orig_line);
-  smartlist_free(sl);
-}
-
 static void
 test_util_di_ops(void *arg)
 {
@@ -4865,6 +4670,35 @@ test_util_di_ops(void *arg)
 }
 
 static void
+test_util_memcpy_iftrue_timei(void *arg)
+{
+  (void)arg;
+  char buf1[25];
+  char buf2[25];
+  char buf3[25];
+
+  for (int i = 0; i < 100; ++i) {
+    crypto_rand(buf1, sizeof(buf1));
+    crypto_rand(buf2, sizeof(buf2));
+    memcpy(buf3, buf1, sizeof(buf1));
+
+    /* We just copied buf1 into buf3.  Now we're going to copy buf2 into buf2,
+       iff our coin flip comes up heads. */
+    bool coinflip = crypto_rand_int(2) == 0;
+
+    memcpy_if_true_timei(coinflip, buf3, buf2, sizeof(buf3));
+
+    if (coinflip) {
+      tt_mem_op(buf3, OP_EQ, buf2, sizeof(buf2));
+    } else {
+      tt_mem_op(buf3, OP_EQ, buf1, sizeof(buf1));
+    }
+  }
+ done:
+  ;
+}
+
+static void
 test_util_di_map(void *arg)
 {
   (void)arg;
@@ -5730,6 +5564,13 @@ test_util_socketpair(void *arg)
     tt_skip();
   }
 #endif /* defined(__FreeBSD__) */
+#ifdef ENETUNREACH
+  if (ersatz && socketpair_result == -ENETUNREACH) {
+    /* We can also fail with -ENETUNREACH if we have no network stack at
+     * all. */
+    tt_skip();
+  }
+#endif /* defined(ENETUNREACH) */
   tt_int_op(0, OP_EQ, socketpair_result);
 
   tt_assert(SOCKET_OK(fds[0]));
@@ -5858,7 +5699,7 @@ test_util_hostname_validation(void *arg)
   tt_assert(string_is_valid_nonrfc_hostname("luck.y13."));
 
   // We allow punycode TLDs. For examples, see
-  // http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+  // https://data.iana.org/TLD/tlds-alpha-by-domain.txt
   tt_assert(string_is_valid_nonrfc_hostname("example.xn--l1acc"));
 
   done:
@@ -5882,6 +5723,18 @@ test_util_ipv4_validation(void *arg)
 }
 
 static void
+test_util_ipv6_validation(void *arg)
+{
+  (void)arg;
+
+  tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e"));
+  tt_assert(!string_is_valid_ipv6_address("11:22::33:44:"));
+
+  done:
+  return;
+}
+
+static void
 test_util_writepid(void *arg)
 {
   (void) arg;
@@ -5965,7 +5818,7 @@ test_util_touch_file(void *arg)
   ;
 }
 
-#ifndef _WIN32
+#ifndef DISABLE_PWDB_TESTS
 static void
 test_util_pwdb(void *arg)
 {
@@ -6037,7 +5890,7 @@ test_util_pwdb(void *arg)
   tor_free(dir);
   teardown_capture_of_logs();
 }
-#endif /* !defined(_WIN32) */
+#endif /* !defined(DISABLE_PWDB_TESTS) */
 
 static void
 test_util_calloc_check(void *arg)
@@ -6285,6 +6138,14 @@ test_util_nowrap_math(void *arg)
   tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(2, UINT32_MAX-1));
   tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX, UINT32_MAX));
 
+  tt_u64_op(0, OP_EQ, tor_mul_u64_nowrap(0, 0));
+  tt_u64_op(1, OP_EQ, tor_mul_u64_nowrap(1, 1));
+  tt_u64_op(2, OP_EQ, tor_mul_u64_nowrap(2, 1));
+  tt_u64_op(4, OP_EQ, tor_mul_u64_nowrap(2, 2));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_mul_u64_nowrap(UINT64_MAX, 1));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_mul_u64_nowrap(2, UINT64_MAX));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_mul_u64_nowrap(UINT64_MAX, UINT64_MAX));
+
  done:
   ;
 }
@@ -6394,40 +6255,168 @@ test_util_get_unquoted_path(void *arg)
   tor_free(r);
 }
 
+static void
+test_util_map_anon(void *arg)
+{
+  (void)arg;
+  char *ptr = NULL;
+  size_t sz = 16384;
+  unsigned inherit=0;
+
+  /* Basic checks. */
+  ptr = tor_mmap_anonymous(sz, 0, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 3;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz-2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 3);
+
+  /* Try again, with a private (non-swappable) mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+  /* Now let's test a drop-on-fork mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+}
+
+static void
+test_util_map_anon_nofork(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  /* The operating system doesn't support forking. */
+  tt_skip();
+ done:
+  ;
+#else /* !defined(_WIN32) */
+  /* We have the right OS support.  We're going to try marking the buffer as
+   * either zero-on-fork or as drop-on-fork, whichever is supported.  Then we
+   * will fork and send a byte back to the parent process.  This will either
+   * crash, or send zero. */
+
+  char *ptr = NULL;
+  const char TEST_VALUE = 0xd0;
+  size_t sz = 16384;
+  int pipefd[2] = {-1, -1};
+  unsigned inherit=0;
+
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  memset(ptr, (uint8_t)TEST_VALUE, sz);
+
+  tt_int_op(0, OP_EQ, pipe(pipefd));
+  pid_t child = fork();
+  if (child == 0) {
+    /* We're in the child. */
+    close(pipefd[0]);
+    ssize_t r = write(pipefd[1], &ptr[sz-1], 1); /* This may crash. */
+    close(pipefd[1]);
+    if (r < 0)
+      exit(1);
+    exit(0);
+  }
+  tt_int_op(child, OP_GT, 0);
+  /* In the parent. */
+  close(pipefd[1]);
+  pipefd[1] = -1;
+  char buf[1];
+  ssize_t r = read(pipefd[0], buf, 1);
+
+  if (inherit == INHERIT_RES_ZERO) {
+    // We should be seeing clear-on-fork behavior.
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, 0); // that byte should be zero.
+  } else if (inherit == INHERIT_RES_DROP) {
+    // We should be seeing noinherit behavior.
+    tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed.
+  } else {
+    // noinherit isn't implemented.
+    tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, TEST_VALUE); // that byte should be TEST_VALUE.
+  }
+
+  int ws;
+  waitpid(child, &ws, 0);
+
+#ifndef NOINHERIT_CAN_FAIL
+  /* Only if NOINHERIT_CAN_FAIL should it be possible for us to get
+   * INHERIT_KEEP behavior in this case. */
+  tt_int_op(inherit, OP_NE, INHERIT_RES_KEEP);
+#else
+  if (inherit == INHERIT_RES_KEEP) {
+    /* Call this test "skipped", not "passed", since noinherit wasn't
+     * implemented. */
+    tt_skip();
+  }
+#endif /* !defined(NOINHERIT_CAN_FAIL) */
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+  if (pipefd[0] >= 0) {
+    close(pipefd[0]);
+  }
+  if (pipefd[1] >= 0) {
+    close(pipefd[1]);
+  }
+#endif /* defined(_WIN32) */
+}
+
+#ifndef COCCI
 #define UTIL_LEGACY(name)                                               \
-  { #name, test_util_ ## name , 0, NULL, NULL }
+  { (#name), test_util_ ## name , 0, NULL, NULL }
 
 #define UTIL_TEST(name, flags)                          \
-  { #name, test_util_ ## name, flags, NULL, NULL }
+  { (#name), test_util_ ## name, flags, NULL, NULL }
 
 #define COMPRESS(name, identifier)              \
-  { "compress/" #name, test_util_compress, 0, &compress_setup,          \
+  { ("compress/" #name), test_util_compress, 0, &compress_setup,        \
     (char*)(identifier) }
 
 #define COMPRESS_CONCAT(name, identifier)                               \
-  { "compress_concat/" #name, test_util_decompress_concatenated, 0,     \
+  { ("compress_concat/" #name), test_util_decompress_concatenated, 0,   \
     &compress_setup,                                                    \
     (char*)(identifier) }
 
 #define COMPRESS_JUNK(name, identifier)                                 \
-  { "compress_junk/" #name, test_util_decompress_junk, 0,               \
+  { ("compress_junk/" #name), test_util_decompress_junk, 0,             \
     &compress_setup,                                                    \
     (char*)(identifier) }
 
 #define COMPRESS_DOS(name, identifier)                                  \
-  { "compress_dos/" #name, test_util_decompress_dos, 0,                 \
+  { ("compress_dos/" #name), test_util_decompress_dos, 0,               \
     &compress_setup,                                                    \
     (char*)(identifier) }
 
 #ifdef _WIN32
-#define UTIL_TEST_NO_WIN(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
 #define UTIL_TEST_WIN_ONLY(n, f) UTIL_TEST(n, (f))
-#define UTIL_LEGACY_NO_WIN(n) UTIL_TEST_NO_WIN(n, 0)
 #else
-#define UTIL_TEST_NO_WIN(n, f) UTIL_TEST(n, (f))
-#define UTIL_TEST_WIN_ONLY(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
-#define UTIL_LEGACY_NO_WIN(n) UTIL_LEGACY(n)
-#endif /* defined(_WIN32) */
+#define UTIL_TEST_WIN_ONLY(n, f) { (#n), NULL, TT_SKIP, NULL, NULL }
+#endif
+
+#ifdef DISABLE_PWDB_TESTS
+#define UTIL_TEST_PWDB(n, f) { (#n), NULL, TT_SKIP, NULL, NULL }
+#else
+#define UTIL_TEST_PWDB(n, f) UTIL_TEST(n, (f))
+#endif
+#endif /* !defined(COCCI) */
 
 struct testcase_t util_tests[] = {
   UTIL_LEGACY(time),
@@ -6437,7 +6426,8 @@ struct testcase_t util_tests[] = {
   UTIL_LEGACY(config_line_comment_character),
   UTIL_LEGACY(config_line_escaped_content),
   UTIL_LEGACY(config_line_crlf),
-  UTIL_LEGACY_NO_WIN(expand_filename),
+  UTIL_TEST(config_line_partition, 0),
+  UTIL_TEST_PWDB(expand_filename, 0),
   UTIL_LEGACY(escape_string_socks),
   UTIL_LEGACY(string_is_key_value),
   UTIL_LEGACY(strmisc),
@@ -6473,6 +6463,7 @@ struct testcase_t util_tests[] = {
   UTIL_LEGACY(path_is_relative),
   UTIL_LEGACY(strtok),
   UTIL_LEGACY(di_ops),
+  UTIL_TEST(memcpy_iftrue_timei, 0),
   UTIL_TEST(di_map, 0),
   UTIL_TEST(round_to_next_multiple_of, 0),
   UTIL_TEST(laplace, 0),
@@ -6487,12 +6478,8 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(nowrap_math, 0),
   UTIL_TEST(num_cpus, 0),
   UTIL_TEST_WIN_ONLY(load_win_lib, 0),
-  UTIL_TEST_NO_WIN(exit_status, 0),
-  UTIL_TEST_NO_WIN(string_from_pipe, 0),
   UTIL_TEST(format_hex_number, 0),
   UTIL_TEST(format_dec_number, 0),
-  UTIL_TEST(join_win_cmdline, 0),
-  UTIL_TEST(split_lines, 0),
   UTIL_TEST(n_bits_set, 0),
   UTIL_TEST(eat_whitespace, 0),
   UTIL_TEST(sl_new_from_text_lines, 0),
@@ -6510,6 +6497,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(mathlog, 0),
   UTIL_TEST(fraction, 0),
   UTIL_TEST(weak_random, 0),
+  { "tor_isinf", test_tor_isinf, TT_FORK, NULL, NULL },
   { "socket_ipv4", test_util_socket, TT_FORK, &passthrough_setup,
     (void*)"4" },
   { "socket_ipv6", test_util_socket, TT_FORK,
@@ -6522,10 +6510,11 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(hostname_validation, 0),
   UTIL_TEST(dest_validation_edgecase, 0),
   UTIL_TEST(ipv4_validation, 0),
+  UTIL_TEST(ipv6_validation, 0),
   UTIL_TEST(writepid, 0),
   UTIL_TEST(get_avail_disk_space, 0),
   UTIL_TEST(touch_file, 0),
-  UTIL_TEST_NO_WIN(pwdb, TT_FORK),
+  UTIL_TEST_PWDB(pwdb, TT_FORK),
   UTIL_TEST(calloc_check, 0),
   UTIL_TEST(monotonic_time, 0),
   UTIL_TEST(monotonic_time_ratchet, TT_FORK),
@@ -6533,5 +6522,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(monotonic_time_add_msec, 0),
   UTIL_TEST(htonll, 0),
   UTIL_TEST(get_unquoted_path, 0),
+  UTIL_TEST(map_anon, 0),
+  UTIL_TEST(map_anon_nofork, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c
index d344d0e95c..726e8e7427 100644
--- a/src/test/test_util_format.c
+++ b/src/test/test_util_format.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -7,11 +7,8 @@
 #include "test/test.h"
 
 #include "lib/crypt_ops/crypto_rand.h"
-#define UTIL_FORMAT_PRIVATE
 #include "lib/encoding/binascii.h"
 
-#define NS_MODULE util_format
-
 static void
 test_util_format_unaligned_accessors(void *ignored)
 {
@@ -346,7 +343,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswqmjs";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 10);
     tt_str_op(expected, OP_EQ, dst);
   }
 
@@ -357,7 +354,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswq";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 8);
     tt_mem_op(expected, OP_EQ, dst, strlen(expected));
   }
 
@@ -367,7 +364,7 @@ test_util_format_base32_decode(void *arg)
     ret = base32_decode(dst, real_dstlen, "#abcde", 6);
     tt_int_op(ret, OP_EQ, -1);
     /* Make sure the destination buffer has been zeroed even on error. */
-    tt_int_op(tor_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
+    tt_int_op(fast_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
   }
 
  done:
@@ -392,10 +389,13 @@ test_util_format_encoded_size(void *arg)
 
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0);
     tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
+
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i,
                   BASE64_ENCODE_MULTILINE);
     tt_int_op(strlen(outbuf), OP_EQ,
               base64_encode_size(i, BASE64_ENCODE_MULTILINE));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
   }
 
  done:
@@ -417,4 +417,3 @@ struct testcase_t util_format_tests[] = {
   { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_util_process.c b/src/test/test_util_process.c
index 4d04eb6dfc..fc79fe9b1f 100644
--- a/src/test/test_util_process.c
+++ b/src/test/test_util_process.c
@@ -1,7 +1,6 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define UTIL_PROCESS_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
 
@@ -12,7 +11,6 @@
 #include "test/log_test_helpers.h"
 
 #ifndef _WIN32
-#define NS_MODULE util_process
 
 static void
 temp_callback(int r, void *s)
@@ -69,15 +67,16 @@ test_util_process_clear_waitpid_callback(void *ignored)
 }
 #endif /* !defined(_WIN32) */
 
+#ifndef COCCI
 #ifndef _WIN32
-#define TEST(name) { #name, test_util_process_##name, 0, NULL, NULL }
+#define TEST(name) { (#name), test_util_process_##name, 0, NULL, NULL }
 #else
-#define TEST(name) { #name, NULL, TT_SKIP, NULL, NULL }
+#define TEST(name) { (#name), NULL, TT_SKIP, NULL, NULL }
 #endif
+#endif /* !defined(COCCI) */
 
 struct testcase_t util_process_tests[] = {
   TEST(set_waitpid_callback),
   TEST(clear_waitpid_callback),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c
deleted file mode 100644
index 29e30eaa11..0000000000
--- a/src/test/test_util_slow.c
+++ /dev/null
@@ -1,396 +0,0 @@
-/* Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#define UTIL_PRIVATE
-#define SUBPROCESS_PRIVATE
-#include "lib/crypt_ops/crypto_cipher.h"
-#include "lib/log/log.h"
-#include "lib/process/subprocess.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/printf.h"
-#include "lib/time/compat_time.h"
-#include "test/test.h"
-
-#include <errno.h>
-#include <string.h>
-
-#ifndef BUILDDIR
-#define BUILDDIR "."
-#endif
-
-#ifdef _WIN32
-#define notify_pending_waitpid_callbacks() STMT_NIL
-#define TEST_CHILD "test-child.exe"
-#define EOL "\r\n"
-#else
-#define TEST_CHILD (BUILDDIR "/src/test/test-child")
-#define EOL "\n"
-#endif /* defined(_WIN32) */
-
-#ifdef _WIN32
-/* I've assumed Windows doesn't have the gap between fork and exec
- * that causes the race condition on unix-like platforms */
-#define MATCH_PROCESS_STATUS(s1,s2)         ((s1) == (s2))
-
-#else /* !(defined(_WIN32)) */
-/* work around a race condition of the timing of SIGCHLD handler updates
- * to the process_handle's fields, and checks of those fields
- *
- * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to
- * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */
-#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING  (PROCESS_STATUS_RUNNING+1)
-#define IS_RUNNING_OR_NOTRUNNING(s)           \
-  ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING)
-/* well, this is ugly */
-#define MATCH_PROCESS_STATUS(s1,s2)           \
-  (  (s1) == (s2)                                         \
-     ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s2))                  \
-     ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s1)))
-
-#endif /* defined(_WIN32) */
-
-/** Helper function for testing tor_spawn_background */
-static void
-run_util_spawn_background(const char *argv[], const char *expected_out,
-                          const char *expected_err, int expected_exit,
-                          int expected_status)
-{
-  int retval, exit_code;
-  ssize_t pos;
-  process_handle_t *process_handle=NULL;
-  char stdout_buf[100], stderr_buf[100];
-  int status;
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  notify_pending_waitpid_callbacks();
-
-  /* the race condition doesn't affect status,
-   * because status isn't updated by the SIGCHLD handler,
-   * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, status));
-  if (status == PROCESS_STATUS_ERROR) {
-    tt_ptr_op(process_handle, OP_EQ, NULL);
-    return;
-  }
-
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* When a spawned process forks, fails, then exits very quickly,
-   * (this typically occurs when exec fails)
-   * there is a race condition between the SIGCHLD handler
-   * updating the process_handle's fields, and this test
-   * checking the process status in those fields.
-   * The SIGCHLD update can occur before or after the code below executes.
-   * This causes intermittent failures in spawn_background_fail(),
-   * typically when the machine is under load.
-   * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */
-
-  /* the race condition affects the change in
-   * process_handle->status from RUNNING to NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status));
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  /* the race condition affects the change in
-   * process_handle->waitpid_cb to NULL,
-   * so we skip the check if expected_status is ambiguous,
-   * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(process_handle->waitpid_cb != NULL
-              || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING);
-#endif /* !defined(_WIN32) */
-
-#ifdef _WIN32
-  tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE);
-#else
-  tt_assert(process_handle->stdout_pipe >= 0);
-  tt_assert(process_handle->stderr_pipe >= 0);
-  tt_assert(process_handle->stdin_pipe >= 0);
-#endif /* defined(_WIN32) */
-
-  /* Check stdout */
-  pos = tor_read_all_from_process_stdout(process_handle, stdout_buf,
-                                         sizeof(stdout_buf) - 1);
-  tt_assert(pos >= 0);
-  stdout_buf[pos] = '\0';
-  tt_int_op(strlen(expected_out),OP_EQ, pos);
-  tt_str_op(expected_out,OP_EQ, stdout_buf);
-
-  notify_pending_waitpid_callbacks();
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-  // TODO: Make test-child exit with something other than 0
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
-  notify_pending_waitpid_callbacks();
-
- done:
-  if (process_handle)
-    tor_process_handle_destroy(process_handle, 1);
-}
-
-/** Check that we can launch a process and read the output */
-static void
-test_util_spawn_background_ok(void *ptr)
-{
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL;
-  const char *expected_err = "ERR"EOL;
-
-  (void)ptr;
-
-  run_util_spawn_background(argv, expected_out, expected_err, 0,
-      PROCESS_STATUS_RUNNING);
-}
-
-/** Check that failing to find the executable works as expected */
-static void
-test_util_spawn_background_fail(void *ptr)
-{
-  const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL};
-  const char *expected_err = "";
-  char expected_out[1024];
-  char code[32];
-#ifdef _WIN32
-  const int expected_status = PROCESS_STATUS_ERROR;
-#else
-  /* TODO: Once we can signal failure to exec, set this to be
-   * PROCESS_STATUS_RUNNING_OR_ERROR */
-  const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING;
-#endif /* defined(_WIN32) */
-
-  memset(expected_out, 0xf0, sizeof(expected_out));
-  memset(code, 0xf0, sizeof(code));
-
-  (void)ptr;
-
-  tor_snprintf(code, sizeof(code), "%x/%x",
-    9 /* CHILD_STATE_FAILEXEC */ , ENOENT);
-  tor_snprintf(expected_out, sizeof(expected_out),
-    "ERR: Failed to spawn background process - code %s\n", code);
-
-  run_util_spawn_background(argv, expected_out, expected_err, 255,
-      expected_status);
-}
-
-/** Test that reading from a handle returns a partial read rather than
- * blocking */
-static void
-test_util_spawn_background_partial_read_impl(int exit_early)
-{
-  const int expected_exit = 0;
-  const int expected_status = PROCESS_STATUS_RUNNING;
-
-  int retval, exit_code;
-  ssize_t pos = -1;
-  process_handle_t *process_handle=NULL;
-  int status;
-  char stdout_buf[100], stderr_buf[100];
-
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL,
-                                 "DONE" EOL,
-                                 NULL };
-  const char *expected_err = "ERR" EOL;
-
-#ifndef _WIN32
-  int eof = 0;
-#endif
-  int expected_out_ctr;
-
-  if (exit_early) {
-    argv[1] = "--hang";
-    expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL;
-  }
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-  tt_int_op(expected_status,OP_EQ, status);
-  tt_assert(process_handle);
-  tt_int_op(expected_status,OP_EQ, process_handle->status);
-
-  /* Check stdout */
-  for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) {
-#ifdef _WIN32
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL);
-#else
-    /* Check that we didn't read the end of file last time */
-    tt_assert(!eof);
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL, &eof);
-#endif /* defined(_WIN32) */
-    log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos);
-
-    /* We would have blocked, keep on trying */
-    if (0 == pos)
-      continue;
-
-    tt_assert(pos > 0);
-    stdout_buf[pos] = '\0';
-    tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf);
-    tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos);
-    expected_out_ctr++;
-  }
-
-  if (exit_early) {
-    tor_process_handle_destroy(process_handle, 1);
-    process_handle = NULL;
-    goto done;
-  }
-
-  /* The process should have exited without writing more */
-#ifdef _WIN32
-  pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                            sizeof(stdout_buf) - 1,
-                            process_handle);
-  tt_int_op(0,OP_EQ, pos);
-#else /* !(defined(_WIN32)) */
-  if (!eof) {
-    /* We should have got all the data, but maybe not the EOF flag */
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1,
-                              process_handle, &eof);
-    tt_int_op(0,OP_EQ, pos);
-    tt_assert(eof);
-  }
-  /* Otherwise, we got the EOF on the last read */
-#endif /* defined(_WIN32) */
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-
-  // TODO: Make test-child exit with something other than 0
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-static void
-test_util_spawn_background_partial_read(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(0);
-}
-
-static void
-test_util_spawn_background_exit_early(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(1);
-}
-
-static void
-test_util_spawn_background_waitpid_notify(void *arg)
-{
-  int retval, exit_code;
-  process_handle_t *process_handle=NULL;
-  int status;
-  int ms_timer;
-
-  const char *argv[] = {TEST_CHILD, "--fast", NULL};
-
-  (void) arg;
-
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* We're not going to look at the stdout/stderr output this time. Instead,
-   * we're testing whether notify_pending_waitpid_calbacks() can report the
-   * process exit (on unix) and/or whether tor_get_exit_code() can notice it
-   * (on windows) */
-
-#ifndef _WIN32
-  ms_timer = 30*1000;
-  tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL);
-  while (process_handle->waitpid_cb && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-    notify_pending_waitpid_callbacks();
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif /* !defined(_WIN32) */
-
-  ms_timer = 30*1000;
-  while (((retval = tor_get_exit_code(process_handle, 0, &exit_code))
-                == PROCESS_EXIT_RUNNING) && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-
-  tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-#undef TEST_CHILD
-#undef EOL
-
-#undef MATCH_PROCESS_STATUS
-
-#ifndef _WIN32
-#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING
-#undef IS_RUNNING_OR_NOTRUNNING
-#endif
-
-#define UTIL_TEST(name, flags)                          \
-  { #name, test_util_ ## name, flags, NULL, NULL }
-
-struct testcase_t slow_util_tests[] = {
-  UTIL_TEST(spawn_background_ok, 0),
-  UTIL_TEST(spawn_background_fail, 0),
-  UTIL_TEST(spawn_background_partial_read, 0),
-  UTIL_TEST(spawn_background_exit_early, 0),
-  UTIL_TEST(spawn_background_waitpid_notify, 0),
-  END_OF_TESTCASES
-};
diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c
new file mode 100644
index 0000000000..ae89e43889
--- /dev/null
+++ b/src/test/test_voting_flags.c
@@ -0,0 +1,193 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define VOTEFLAGS_PRIVATE
+
+#include "core/or/or.h"
+
+#include "feature/dirauth/voteflags.h"
+#include "feature/dirauth/dirauth_options_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "app/config/config.h"
+
+#include "test/test.h"
+#include "test/opts_test_helpers.h"
+
+typedef struct {
+  time_t now;
+  routerinfo_t ri;
+  node_t node;
+
+  routerstatus_t expected;
+} flag_vote_test_cfg_t;
+
+static void
+setup_cfg(flag_vote_test_cfg_t *c)
+{
+  memset(c, 0, sizeof(*c));
+
+  c->now = approx_time();
+
+  c->ri.nickname = (char *) "testing100";
+  strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname));
+
+  memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN);
+  memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN);
+
+  c->ri.cache_info.published_on = c->now - 100;
+  c->expected.published_on = c->now - 100;
+
+  c->ri.addr = 0x7f010105;
+  c->expected.addr = 0x7f010105;
+  c->ri.or_port = 9090;
+  c->expected.or_port = 9090;
+
+  tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6);
+  tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6);
+
+  // By default we have no loaded information about stability or speed,
+  // so we'll default to voting "yeah sure." on these two.
+  c->expected.is_fast = 1;
+  c->expected.is_stable = 1;
+}
+
+static bool
+check_result(flag_vote_test_cfg_t *c)
+{
+  bool result = false;
+  routerstatus_t rs;
+  memset(&rs, 0, sizeof(rs));
+  dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0);
+
+  tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on);
+  tt_str_op(rs.nickname, OP_EQ, c->expected.nickname);
+
+  // identity_digest and descriptor_digest are not set here.
+
+  tt_uint_op(rs.addr, OP_EQ, c->expected.addr);
+  tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port);
+  tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port);
+
+  tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr));
+  tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport);
+
+#define FLAG(flagname) \
+  tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname)
+
+  FLAG(is_authority);
+  FLAG(is_exit);
+  FLAG(is_stable);
+  FLAG(is_fast);
+  FLAG(is_flagged_running);
+  FLAG(is_named);
+  FLAG(is_unnamed);
+  FLAG(is_valid);
+  FLAG(is_possible_guard);
+  FLAG(is_bad_exit);
+  FLAG(is_hs_dir);
+  FLAG(is_v2_dir);
+  FLAG(is_staledesc);
+  FLAG(has_bandwidth);
+  FLAG(has_exitsummary);
+  FLAG(bw_is_unmeasured);
+
+  result = true;
+
+ done:
+  return result;
+}
+
+static void
+test_voting_flags_minimal(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  (void) check_result(cfg);
+}
+
+static void
+test_voting_flags_ipv6(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+
+  tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->ri.ipv6_orport = 9091;
+  // no change in expected results, since we aren't set up with ipv6
+  // connectivity.
+  if (!check_result(cfg))
+    goto done;
+
+  get_dirauth_options(get_options_mutable())->AuthDirHasIPv6Connectivity = 1;
+  // no change in expected results, since last_reachable6 won't be set.
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->node.last_reachable6 = cfg->now - 10;
+  // now that lastreachable6 is set, we expect to see the result.
+  tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->expected.ipv6_orport = 9091;
+  if (!check_result(cfg))
+    goto done;
+ done:
+  ;
+}
+
+static void
+test_voting_flags_staledesc(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  time_t now = cfg->now;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  // no change in expectations for is_staledesc
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.is_staledesc = 1;
+  if (!check_result(cfg))
+    goto done;
+
+ done:
+  ;
+}
+
+static void *
+setup_voting_flags_test(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  setup_cfg(cfg);
+  return cfg;
+}
+
+static int
+teardown_voting_flags_test(const struct testcase_t *testcase, void *arg)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = arg;
+  tor_free(cfg);
+  return 1;
+}
+
+static const struct testcase_setup_t voting_flags_setup = {
+  .setup_fn = setup_voting_flags_test,
+  .cleanup_fn = teardown_voting_flags_test,
+};
+
+#define T(name,flags)                                   \
+  { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL }
+
+struct testcase_t voting_flags_tests[] = {
+  T(minimal, 0),
+  T(ipv6, TT_FORK),
+  // TODO: Add more of these tests.
+  T(staledesc, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_voting_schedule.c b/src/test/test_voting_schedule.c
index ba4d53a4ae..df64b79167 100644
--- a/src/test/test_voting_schedule.c
+++ b/src/test/test_voting_schedule.c
@@ -1,17 +1,18 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #include "core/or/or.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
+#include "feature/nodelist/networkstatus.h"
 
 #include "test/test.h"
 
 static void
 test_voting_schedule_interval_start(void *arg)
 {
-#define next_interval voting_schedule_get_start_of_next_interval
+#define next_interval voting_sched_get_start_of_interval_after
   (void)arg;
   char buf[ISO_TIME_LEN+1];
 
@@ -61,4 +62,3 @@ struct testcase_t voting_schedule_tests[] = {
   VS(interval_start, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c
index c58634da5c..3734c08e48 100644
--- a/src/test/test_workqueue.c
+++ b/src/test/test_workqueue.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -32,7 +32,7 @@ int handled_len;
 bitarray_t *handled;
 #endif
 
-typedef struct state_s {
+typedef struct state_t {
   int magic;
   int n_handled;
   crypto_pk_t *rsa;
@@ -40,13 +40,13 @@ typedef struct state_s {
   int is_shutdown;
 } state_t;
 
-typedef struct rsa_work_s {
+typedef struct rsa_work_t {
   int serial;
   uint8_t msg[128];
   uint8_t msglen;
 } rsa_work_t;
 
-typedef struct ecdh_work_s {
+typedef struct ecdh_work_t {
   int serial;
   union {
     curve25519_public_key_t pk;
@@ -63,7 +63,7 @@ mark_handled(int serial)
   tor_assert(! bitarray_is_set(handled, serial));
   bitarray_set(handled, serial);
   tor_mutex_release(&bitmap_mutex);
-#else /* !(defined(TRACK_RESPONSES)) */
+#else /* !defined(TRACK_RESPONSES) */
   (void)serial;
 #endif /* defined(TRACK_RESPONSES) */
 }
@@ -339,7 +339,7 @@ main(int argc, char **argv)
   replyqueue_t *rq;
   threadpool_t *tp;
   int i;
-  tor_libevent_cfg evcfg;
+  tor_libevent_cfg_t evcfg;
   uint32_t as_flags = 0;
 
   for (i = 1; i < argc; ++i) {
diff --git a/src/test/test_workqueue_cancel.sh b/src/test/test_workqueue_cancel.sh
index f7c663171e..e50b884f26 100755
--- a/src/test/test_workqueue_cancel.sh
+++ b/src/test/test_workqueue_cancel.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue -C 1
+"${builddir:-.}/src/test/test_workqueue" -C 1
 
diff --git a/src/test/test_workqueue_efd.sh b/src/test/test_workqueue_efd.sh
index 4d89396819..592841fc91 100755
--- a/src/test/test_workqueue_efd.sh
+++ b/src/test/test_workqueue_efd.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_efd2.sh b/src/test/test_workqueue_efd2.sh
index 7cfff45ff3..4cf1b76cbe 100755
--- a/src/test/test_workqueue_efd2.sh
+++ b/src/test/test_workqueue_efd2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_pipe.sh b/src/test/test_workqueue_pipe.sh
index afcef87853..fc3ef34c6c 100755
--- a/src/test/test_workqueue_pipe.sh
+++ b/src/test/test_workqueue_pipe.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-socketpair
diff --git a/src/test/test_workqueue_pipe2.sh b/src/test/test_workqueue_pipe2.sh
index a20a1427e0..7f19ea880d 100755
--- a/src/test/test_workqueue_pipe2.sh
+++ b/src/test/test_workqueue_pipe2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_socketpair.sh b/src/test/test_workqueue_socketpair.sh
index 76af79746d..1ee1776447 100755
--- a/src/test/test_workqueue_socketpair.sh
+++ b/src/test/test_workqueue_socketpair.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-pipe
diff --git a/src/test/test_x509.c b/src/test/test_x509.c
index 792849ae4b..94e7db33de 100644
--- a/src/test/test_x509.c
+++ b/src/test/test_x509.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_X509_PRIVATE
diff --git a/src/test/test_zero_length_keys.sh b/src/test/test_zero_length_keys.sh
index 84ca513b0a..b944d9bf3f 100755
--- a/src/test/test_zero_length_keys.sh
+++ b/src/test/test_zero_length_keys.sh
@@ -1,10 +1,44 @@
 #!/bin/sh
 # Check that tor regenerates keys when key files are zero-length
 
+umask 077
+set -e
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "$TOR_BINARY" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
+fi
+
 exitcode=0
 
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -z || exitcode=1
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -d || exitcode=1
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -e || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "$TOR_BINARY" -z || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "$TOR_BINARY" -d || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "$TOR_BINARY" -e || exitcode=1
 
 exit ${exitcode}
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index daa7aa524a..d68dfa4047 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #include "orconfig.h"
 #include "core/or/or.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -25,6 +26,8 @@
 #include "lib/compress/compress.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
+#include "app/main/subsysmgr.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -86,7 +89,18 @@ setup_directory(void)
                  (int)getpid(), rnd32);
     r = mkdir(temp_dir);
   }
-#else /* !(defined(_WIN32)) */
+#elif defined(__ANDROID__)
+  /* tor might not like the default perms, so create a subdir */
+  tor_snprintf(temp_dir, sizeof(temp_dir),
+               "/data/local/tmp/tor_%d_%d_%s",
+               (int) getuid(), (int) getpid(), rnd32);
+  r = mkdir(temp_dir, 0700);
+  if (r) {
+    fprintf(stderr, "Can't create directory %s:", temp_dir);
+    perror("");
+    exit(1);
+  }
+#else /* !defined(_WIN32) */
   tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s",
                (int) getpid(), rnd32);
   r = mkdir(temp_dir, 0700);
@@ -94,7 +108,7 @@ setup_directory(void)
     /* undo sticky bit so tests don't get confused. */
     r = chown(temp_dir, getuid(), getgid());
   }
-#endif /* defined(_WIN32) */
+#endif /* defined(_WIN32) || ... */
   if (r) {
     fprintf(stderr, "Can't create directory %s:", temp_dir);
     perror("");
@@ -230,17 +244,17 @@ void
 tinytest_prefork(void)
 {
   free_pregenerated_keys();
-  crypto_prefork();
+  subsystems_prefork();
 }
 void
 tinytest_postfork(void)
 {
-  crypto_postfork();
+  subsystems_postfork();
   init_pregenerated_keys();
 }
 
 static void
-log_callback_failure(int severity, uint32_t domain, const char *msg)
+log_callback_failure(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)msg;
   if (severity == LOG_ERR || (domain & LD_BUG)) {
@@ -259,24 +273,18 @@ main(int c, const char **v)
   int loglevel = LOG_ERR;
   int accel_crypto = 0;
 
-  /* We must initialise logs before we call tor_assert() */
-  init_logging(1);
+  subsystems_init();
 
-  update_approx_time(time(NULL));
   options = options_new();
-  tor_threads_init();
-  tor_compress_init();
-
-  network_init();
-
-  monotime_init();
 
-  struct tor_libevent_cfg cfg;
+  struct tor_libevent_cfg_t cfg;
   memset(&cfg, 0, sizeof(cfg));
   tor_libevent_initialize(&cfg);
 
   control_initialize_event_queue();
-  configure_backtrace_handler(get_version());
+
+  /* Don't add default logs; the tests manage their own. */
+  quiet_level = QUIET_SILENT;
 
   unsigned num=1, den=1;
 
@@ -316,7 +324,7 @@ main(int c, const char **v)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
   {
@@ -324,9 +332,10 @@ main(int c, const char **v)
     log_severity_list_t s;
     memset(&s, 0, sizeof(s));
     set_log_severity_config(LOG_ERR, LOG_ERR, &s);
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_callback_log(&s, log_callback_failure);
   }
+  flush_log_messages_from_startup();
   init_protocol_warning_severity_level();
 
   options->command = CMD_RUN_UNITTESTS;
@@ -343,6 +352,7 @@ main(int c, const char **v)
   initialize_mainloop_events();
   options_init(options);
   options->DataDirectory = tor_strdup(temp_dir);
+  options->DataDirectory_option = tor_strdup(temp_dir);
   tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys",
                options->DataDirectory);
   options->CacheDirectory = tor_strdup(temp_dir);
@@ -409,8 +419,6 @@ main(int c, const char **v)
 
   free_pregenerated_keys();
 
-  crypto_global_cleanup();
-
   if (have_failed)
     return 1;
   else
diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c
index 0f22d4e01b..e058f72d01 100644
--- a/src/test/testing_rsakeys.c
+++ b/src/test/testing_rsakeys.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "lib/crypt_ops/crypto_rand.h"
@@ -448,7 +448,8 @@ static int next_key_idx_2048;
 static crypto_pk_t *
 pk_generate_internal(int bits)
 {
-  tor_assert(bits == 2048 || bits == 1024);
+  tor_assertf(bits == 2048 || bits == 1024,
+             "Wrong key size: %d", bits);
 
 #ifdef USE_PREGENERATED_RSA_KEYS
   int *idxp;
@@ -467,7 +468,7 @@ pk_generate_internal(int bits)
   *idxp += crypto_rand_int_range(1,3);
   *idxp %= n_pregen;
   return crypto_pk_dup_key(pregen_array[*idxp]);
-#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */
+#else /* !defined(USE_PREGENERATED_RSA_KEYS) */
   crypto_pk_t *result;
   int res;
   result = crypto_pk_new();
diff --git a/src/test/zero_length_keys.sh b/src/test/zero_length_keys.sh
index 5635bdfd89..1702d11245 100755
--- a/src/test/zero_length_keys.sh
+++ b/src/test/zero_length_keys.sh
@@ -19,7 +19,7 @@
 #   3: a command failed - the test could not be completed
 #
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   echo "Usage: ${0} PATH_TO_TOR [-z|-d|-e]"
   exit 1
 elif [ $# -eq 1 ]; then
@@ -31,7 +31,7 @@ else #[$# -gt 1 ]; then
   shift
 fi
 
-DATA_DIR=`mktemp -d -t tor_zero_length_keys.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_zero_length_keys.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -40,7 +40,7 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 touch "$DATA_DIR"/empty_torrc
 touch "$DATA_DIR"/empty_defaults_torrc
diff --git a/src/tools/.may_include b/src/tools/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/tools/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/tools/include.am b/src/tools/include.am
index f7aa7e0d1e..72dfe6017c 100644
--- a/src/tools/include.am
+++ b/src/tools/include.am
@@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve
 endif
 
 src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c
-src_tools_tor_resolve_LDFLAGS =
+src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_LIBS) \
+        $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\
 	$(rust_ldadd) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@
 
@@ -15,8 +17,11 @@ if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_cov_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_TESTING_LIBS) \
+	$(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@
 endif
 
diff --git a/src/tools/tools.md b/src/tools/tools.md
new file mode 100644
index 0000000000..1f00381679
--- /dev/null
+++ b/src/tools/tools.md
@@ -0,0 +1,6 @@
+@dir /tools
+@brief tools: other command-line tools for use with Tor.
+
+The "tools" directory has a few other programs that use Tor, but are not part
+of the main Tor binary.
+
diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c
index 25113420df..e4f6530b46 100644
--- a/src/tools/tor-gencert.c
+++ b/src/tools/tor-gencert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -21,7 +21,7 @@
 /* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice in
  * x509.h and x509_vfy.h. Suppress the GCC warning so we can build with
  * -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/evp.h>
 #include <openssl/pem.h>
@@ -30,8 +30,8 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/obj_mac.h>
 #include <openssl/err.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <errno.h>
 
diff --git a/src/tools/tor-print-ed-signing-cert.c b/src/tools/tor-print-ed-signing-cert.c
index 1f1a01ab5c..7836293df4 100644
--- a/src/tools/tor-print-ed-signing-cert.c
+++ b/src/tools/tor-print-ed-signing-cert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include <errno.h>
@@ -10,11 +10,13 @@
 #include "lib/cc/torint.h"  /* TOR_PRIdSZ */
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/malloc/malloc.h"
+#include "lib/encoding/time_fmt.h"
 
 int
 main(int argc, char **argv)
 {
   ed25519_cert_t *cert = NULL;
+  char rfc1123_buf[RFC1123_TIME_LEN+1] = "";
 
   if (argc != 2) {
     fprintf(stderr, "Usage:\n");
@@ -59,6 +61,11 @@ main(int argc, char **argv)
 
   printf("Expires at: %s", ctime(&expires_at));
 
+  format_rfc1123_time(rfc1123_buf, expires_at);
+  printf("RFC 1123 timestamp: %s\n", rfc1123_buf);
+
+  printf("UNIX timestamp: %ld\n", (long int)expires_at);
+
   ed25519_cert_free(cert);
 
   return 0;
diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c
index 6a84abe557..e6d6bddcdb 100644
--- a/src/tools/tor-resolve.c
+++ b/src/tools/tor-resolve.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
- * Copyright (c) 2007-2019, The Tor Project, Inc.
+ * Copyright (c) 2007-2020, The Tor Project, Inc.
  */
 /* See LICENSE for licensing information */
 
@@ -15,6 +15,7 @@
 #include "lib/string/util_string.h"
 
 #include "lib/net/socks5_status.h"
+#include "trunnel/socks5.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -44,73 +45,211 @@
 
 #define RESPONSE_LEN_4 8
 #define log_sock_error(act, _s)                                         \
-  STMT_BEGIN log_fn(LOG_ERR, LD_NET, "Error while %s: %s", act,         \
-              tor_socket_strerror(tor_socket_errno(_s))); STMT_END
+  STMT_BEGIN                                                            \
+    log_fn(LOG_ERR, LD_NET, "Error while %s: %s", act,                  \
+    tor_socket_strerror(tor_socket_errno(_s)));                         \
+  STMT_END
 
 static void usage(void) ATTR_NORETURN;
 
+/**
+ * Set <b>out</b> to a pointer to newly allocated buffer containing
+ * SOCKS4a RESOLVE request with <b>username</b> and <b>hostname</b>.
+ * Return number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks4a_resolve_request(uint8_t **out,
+                              const char *username,
+                              const char *hostname)
+{
+  tor_assert(out);
+  tor_assert(username);
+  tor_assert(hostname);
+
+  const char *errmsg = NULL;
+  uint8_t *output = NULL;
+  socks4_client_request_t *rq = socks4_client_request_new();
+
+  socks4_client_request_set_version(rq, 4);
+  socks4_client_request_set_command(rq, CMD_RESOLVE);
+  socks4_client_request_set_port(rq, 0);
+  socks4_client_request_set_addr(rq, 0x00000001u);
+  socks4_client_request_set_username(rq, username);
+  socks4_client_request_set_socks4a_addr_hostname(rq, hostname);
+
+  errmsg = socks4_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks4_client_request_encoded_len(rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encoded_len failed";
+    goto cleanup;
+  }
+
+  output = tor_malloc(encoded_len);
+  memset(output, 0, encoded_len);
+
+  encoded_len = socks4_client_request_encode(output, encoded_len, rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encode failed";
+    goto cleanup;
+  }
+
+  *out = output;
+
+ cleanup:
+  socks4_client_request_free(rq);
+  if (errmsg) {
+    log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg);
+    *out = NULL;
+    tor_free(output);
+  }
+  return errmsg ? -1 : encoded_len;
+}
+
+#define SOCKS5_ATYPE_HOSTNAME 0x03
+#define SOCKS5_ATYPE_IPV4     0x01
+#define SOCKS5_ATYPE_IPV6     0x04
+
+/**
+ * Set <b>out</b> to pointer to newly allocated buffer containing
+ * SOCKS5 RESOLVE/RESOLVE_PTR request with given <b>hostname<b>.
+ * Generate a reverse request if <b>reverse</b> is true.
+ * Return the number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks5_resolve_request(uint8_t **out,
+                             const char *hostname,
+                             int reverse)
+{
+  const char *errmsg = NULL;
+  uint8_t *outbuf = NULL;
+  int is_ip_address;
+  tor_addr_t addr;
+  size_t addrlen;
+  int ipv6;
+  is_ip_address = tor_addr_parse(&addr, hostname) != -1;
+  if (!is_ip_address && reverse) {
+    log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
+    return -1;
+  }
+  ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
+  addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
+  if (addrlen > UINT8_MAX) {
+    log_err(LD_GENERAL, "Hostname is too long!");
+    return -1;
+  }
+
+  socks5_client_request_t *rq = socks5_client_request_new();
+
+  socks5_client_request_set_version(rq, 5);
+  /* RESOLVE_PTR or RESOLVE */
+  socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR :
+                                                  CMD_RESOLVE);
+  socks5_client_request_set_reserved(rq, 0);
+
+  uint8_t atype = SOCKS5_ATYPE_HOSTNAME;
+  if (reverse)
+    atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4;
+
+  socks5_client_request_set_atype(rq, atype);
+
+  switch (atype) {
+    case SOCKS5_ATYPE_IPV4: {
+      socks5_client_request_set_dest_addr_ipv4(rq,
+                                        tor_addr_to_ipv4h(&addr));
+    } break;
+    case SOCKS5_ATYPE_IPV6: {
+      uint8_t *ipv6_array =
+        socks5_client_request_getarray_dest_addr_ipv6(rq);
+
+      tor_assert(ipv6_array);
+
+      memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16);
+    } break;
+
+    case SOCKS5_ATYPE_HOSTNAME: {
+      domainname_t *dn = domainname_new();
+      domainname_set_len(dn, addrlen - 1);
+      domainname_setlen_name(dn, addrlen - 1);
+      char *dn_buf = domainname_getarray_name(dn);
+      memcpy(dn_buf, hostname, addrlen - 1);
+
+      errmsg = domainname_check(dn);
+
+      if (errmsg) {
+        domainname_free(dn);
+        goto cleanup;
+      } else {
+        socks5_client_request_set_dest_addr_domainname(rq, dn);
+      }
+    } break;
+    default:
+      tor_assert_unreached();
+      break;
+  }
+
+  socks5_client_request_set_dest_port(rq, 0);
+
+  errmsg = socks5_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks5_client_request_encoded_len(rq);
+  if (encoded_len < 0) {
+    errmsg = "Cannot predict encoded length";
+    goto cleanup;
+  }
+
+  outbuf = tor_malloc(encoded_len);
+  memset(outbuf, 0, encoded_len);
+
+  encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq);
+  if (encoded_len < 0) {
+    errmsg = "encoding failed";
+    goto cleanup;
+  }
+
+  *out = outbuf;
+
+ cleanup:
+  socks5_client_request_free(rq);
+  if (errmsg) {
+    tor_free(outbuf);
+    log_err(LD_NET, "build_socks5_resolve_request failed with error: %s",
+            errmsg);
+  }
+
+  return errmsg ? -1 : encoded_len;
+}
+
 /** Set *<b>out</b> to a newly allocated SOCKS4a resolve request with
  * <b>username</b> and <b>hostname</b> as provided.  Return the number
  * of bytes in the request. */
 static ssize_t
-build_socks_resolve_request(char **out,
+build_socks_resolve_request(uint8_t **out,
                             const char *username,
                             const char *hostname,
                             int reverse,
                             int version)
 {
-  size_t len = 0;
   tor_assert(out);
   tor_assert(username);
   tor_assert(hostname);
 
+  tor_assert(version == 4 || version == 5);
+
   if (version == 4) {
-    len = 8 + strlen(username) + 1 + strlen(hostname) + 1;
-    *out = tor_malloc(len);
-    (*out)[0] = 4;      /* SOCKS version 4 */
-    (*out)[1] = '\xF0'; /* Command: resolve. */
-    set_uint16((*out)+2, htons(0)); /* port: 0. */
-    set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */
-    memcpy((*out)+8, username, strlen(username)+1);
-    memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1);
+    return build_socks4a_resolve_request(out, username, hostname);
   } else if (version == 5) {
-    int is_ip_address;
-    tor_addr_t addr;
-    size_t addrlen;
-    int ipv6;
-    is_ip_address = tor_addr_parse(&addr, hostname) != -1;
-    if (!is_ip_address && reverse) {
-      log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
-      return -1;
-    }
-    ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
-    addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
-    if (addrlen > UINT8_MAX) {
-      log_err(LD_GENERAL, "Hostname is too long!");
-      return -1;
-    }
-    len = 6 + addrlen;
-    *out = tor_malloc(len);
-    (*out)[0] = 5; /* SOCKS version 5 */
-    (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */
-    (*out)[2] = 0; /* reserved. */
-    if (reverse) {
-      (*out)[3] = ipv6 ? 4 : 1;
-      if (ipv6)
-        memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16);
-      else
-        set_uint32((*out)+4, tor_addr_to_ipv4n(&addr));
-    } else {
-      (*out)[3] = 3;
-      (*out)[4] = (char)(uint8_t)(addrlen - 1);
-      memcpy((*out)+5, hostname, addrlen - 1);
-    }
-    set_uint16((*out)+4+addrlen, 0); /* port */
-  } else {
-    tor_assert(0);
+    return build_socks5_resolve_request(out, hostname, reverse);
   }
 
-  return len;
+  tor_assert_unreached();
+  return -1;
 }
 
 static void
@@ -134,34 +273,50 @@ parse_socks4a_resolve_response(const char *hostname,
                                const char *response, size_t len,
                                tor_addr_t *addr_out)
 {
+  int result = 0;
   uint8_t status;
   tor_assert(response);
   tor_assert(addr_out);
 
-  if (len < RESPONSE_LEN_4) {
+  socks4_server_reply_t *reply;
+
+  ssize_t parsed = socks4_server_reply_parse(&reply,
+                                             (const uint8_t *)response,
+                                             len);
+
+  if (parsed == -1) {
+    log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response");
+    result = -1; goto cleanup;
+  }
+
+  if (parsed == -2) {
     log_warn(LD_PROTOCOL,"Truncated socks response.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  if (((uint8_t)response[0])!=0) { /* version: 0 */
+
+  if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */
     log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  status = (uint8_t)response[1];
-  if (get_uint16(response+2)!=0) { /* port: 0 */
+  if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */
     log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
+  status = socks4_server_reply_get_status(reply);
   if (status != 90) {
     log_warn(LD_NET,"Got status response '%d': socks request failed.", status);
     if (!strcasecmpend(hostname, ".onion")) {
       onion_warning(hostname);
-      return -1;
+      result = -1; goto cleanup;
     }
-    return -1;
+    result = -1; goto cleanup;
   }
 
-  tor_addr_from_ipv4n(addr_out, get_uint32(response+4));
-  return 0;
+  tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply));
+
+ cleanup:
+  socks4_server_reply_free(reply);
+  return result;
 }
 
 /* It would be nice to let someone know what SOCKS5 issue a user may have */
@@ -205,7 +360,7 @@ do_resolve(const char *hostname,
   int s = -1;
   struct sockaddr_storage ss;
   socklen_t socklen;
-  char *req = NULL;
+  uint8_t *req = NULL;
   ssize_t len = 0;
 
   tor_assert(hostname);
@@ -230,23 +385,58 @@ do_resolve(const char *hostname,
   }
 
   if (version == 5) {
-    char method_buf[2];
-    if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) {
+    socks5_client_version_t *v = socks5_client_version_new();
+
+    socks5_client_version_set_version(v, 5);
+    socks5_client_version_set_n_methods(v, 1);
+    socks5_client_version_setlen_methods(v, 1);
+    socks5_client_version_set_methods(v, 0, 0x00);
+
+    tor_assert(!socks5_client_version_check(v));
+    ssize_t encoded_len = socks5_client_version_encoded_len(v);
+    tor_assert(encoded_len > 0);
+
+    uint8_t *buf = tor_malloc(encoded_len);
+    encoded_len = socks5_client_version_encode(buf, encoded_len, v);
+    tor_assert(encoded_len > 0);
+
+    socks5_client_version_free(v);
+
+    if (write_all_to_socket(s, (const char *)buf,
+                            encoded_len) != encoded_len) {
       log_err(LD_NET, "Error sending SOCKS5 method list.");
+      tor_free(buf);
+
       goto err;
     }
-    if (read_all_from_socket(s, method_buf, 2) != 2) {
+
+    tor_free(buf);
+
+    uint8_t method_buf[2];
+
+    if (read_all_from_socket(s, (char *)method_buf, 2) != 2) {
       log_err(LD_NET, "Error reading SOCKS5 methods.");
       goto err;
     }
-    if (method_buf[0] != '\x05') {
-      log_err(LD_NET, "Unrecognized socks version: %u",
-              (unsigned)method_buf[0]);
+
+    socks5_server_method_t *m;
+    ssize_t parsed = socks5_server_method_parse(&m, method_buf,
+                                                sizeof(method_buf));
+
+    if (parsed < 2) {
+      log_err(LD_NET, "Failed to parse SOCKS5 method selection "
+                      "message");
+      socks5_server_method_free(m);
       goto err;
     }
-    if (method_buf[1] != '\x00') {
+
+    uint8_t method = socks5_server_method_get_method(m);
+
+    socks5_server_method_free(m);
+
+    if (method != 0x00) {
       log_err(LD_NET, "Unrecognized socks authentication method: %u",
-              (unsigned)method_buf[1]);
+              (unsigned)method);
       goto err;
     }
   }
@@ -257,7 +447,7 @@ do_resolve(const char *hostname,
     tor_assert(!req);
     goto err;
   }
-  if (write_all_to_socket(s, req, len) != len) {
+  if (write_all_to_socket(s, (const char *)req, len) != len) {
     log_sock_error("sending SOCKS request", s);
     tor_free(req);
     goto err;
@@ -276,55 +466,59 @@ do_resolve(const char *hostname,
       goto err;
     }
   } else {
-    char reply_buf[16];
-    if (read_all_from_socket(s, reply_buf, 4) != 4) {
-      log_err(LD_NET, "Error reading SOCKS5 response.");
+    uint8_t reply_buf[512];
+
+    len = read_all_from_socket(s, (char *)reply_buf,
+                               sizeof(reply_buf));
+
+    socks5_server_reply_t *reply;
+
+    ssize_t parsed = socks5_server_reply_parse(&reply,
+                                               reply_buf,
+                                               len);
+    if (parsed == -1) {
+      log_err(LD_NET, "Failed parsing SOCKS5 response");
       goto err;
     }
-    if (reply_buf[0] != 5) {
-      log_err(LD_NET, "Bad SOCKS5 reply version.");
+
+    if (parsed == -2) {
+      log_err(LD_NET, "Truncated SOCKS5 response");
       goto err;
     }
+
     /* Give a user some useful feedback about SOCKS5 errors */
-    if (reply_buf[1] != 0) {
+    uint8_t reply_field = socks5_server_reply_get_reply(reply);
+    if (reply_field != 0) {
       log_warn(LD_NET,"Got SOCKS5 status response '%u': %s",
-               (unsigned)reply_buf[1],
-               socks5_reason_to_string(reply_buf[1]));
-      if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) {
+               (unsigned)reply_field,
+               socks5_reason_to_string(reply_field));
+      if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) {
         onion_warning(hostname);
       }
+
+      socks5_server_reply_free(reply);
       goto err;
     }
-    if (reply_buf[3] == 1) {
+
+    uint8_t atype = socks5_server_reply_get_atype(reply);
+
+    if (atype == SOCKS5_ATYPE_IPV4) {
       /* IPv4 address */
-      if (read_all_from_socket(s, reply_buf, 4) != 4) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf));
-    } else if (reply_buf[3] == 4) {
+      tor_addr_from_ipv4h(result_addr,
+        socks5_server_reply_get_bind_addr_ipv4(reply));
+    } else if (atype == SOCKS5_ATYPE_IPV6) {
       /* IPv6 address */
-      if (read_all_from_socket(s, reply_buf, 16) != 16) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv6_bytes(result_addr, reply_buf);
-    } else if (reply_buf[3] == 3) {
+      tor_addr_from_ipv6_bytes(result_addr,
+        socks5_server_reply_getarray_bind_addr_ipv6(reply));
+    } else if (atype == SOCKS5_ATYPE_HOSTNAME) {
       /* Domain name */
-      size_t result_len;
-      if (read_all_from_socket(s, reply_buf, 1) != 1) {
-        log_err(LD_NET, "Error reading address_length in socks5 response.");
-        goto err;
-      }
-      result_len = *(uint8_t*)(reply_buf);
-      *result_hostname = tor_malloc(result_len+1);
-      if (read_all_from_socket(s, *result_hostname, result_len)
-          != (int) result_len) {
-        log_err(LD_NET, "Error reading hostname in socks5 response.");
-        goto err;
-      }
-      (*result_hostname)[result_len] = '\0';
+      domainname_t *dn =
+        socks5_server_reply_get_bind_addr_domainname(reply);
+
+      *result_hostname = tor_strdup(domainname_getstr_name(dn));
     }
+
+    socks5_server_reply_free(reply);
   }
 
   tor_close_socket(s);
diff --git a/src/tools/tor_runner.c b/src/tools/tor_runner.c
index 3c6ade91d9..83f1a495cd 100644
--- a/src/tools/tor_runner.c
+++ b/src/tools/tor_runner.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/trunnel/.may_include b/src/trunnel/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/trunnel/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c
index 59e6b38384..d96496e90c 100644
--- a/src/trunnel/channelpadding_negotiation.c
+++ b/src/trunnel/channelpadding_negotiation.c
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h
index fcfc232fea..3f96174f68 100644
--- a/src/trunnel/channelpadding_negotiation.h
+++ b/src/trunnel/channelpadding_negotiation.h
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c
new file mode 100644
index 0000000000..547818f2ec
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.c
@@ -0,0 +1,549 @@
+/* circpad_negotiation.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "circpad_negotiation.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int circpadnegotiation_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || circpadnegotiation_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+circpad_negotiate_t *
+circpad_negotiate_new(void)
+{
+  circpad_negotiate_t *val = trunnel_calloc(1, sizeof(circpad_negotiate_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiate_clear(circpad_negotiate_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiate_free(circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiate_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiate_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiate_get_version(const circpad_negotiate_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_command(const circpad_negotiate_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp)
+{
+  return inp->echo_request;
+}
+int
+circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->echo_request = val;
+  return 0;
+}
+const char *
+circpad_negotiate_check(const circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiate_encoded_len(const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiate_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+
+  /* Length of u8 echo_request IN [0, 1] */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiate_clear_errors(circpad_negotiate_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiate_encode(uint8_t *output, const size_t avail, const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiate_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiate_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 echo_request IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->echo_request));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiate_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiate_parse_into(circpad_negotiate_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 echo_request IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->echo_request = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    goto fail;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiate_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiate_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiate_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+circpad_negotiated_t *
+circpad_negotiated_new(void)
+{
+  circpad_negotiated_t *val = trunnel_calloc(1, sizeof(circpad_negotiated_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  val->response = CIRCPAD_RESPONSE_ERR;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiated_clear(circpad_negotiated_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiated_free(circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiated_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiated_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiated_get_version(const circpad_negotiated_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_command(const circpad_negotiated_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_response(const circpad_negotiated_t *inp)
+{
+  return inp->response;
+}
+int
+circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_RESPONSE_ERR || val == CIRCPAD_RESPONSE_OK))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->response = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+const char *
+circpad_negotiated_check(const circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiated_encoded_len(const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiated_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiated_clear_errors(circpad_negotiated_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiated_encode(uint8_t *output, const size_t avail, const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiated_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiated_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->response));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiated_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiated_parse_into(circpad_negotiated_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  CHECK_REMAINING(1, truncated);
+  obj->response = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiated_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiated_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiated_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h
new file mode 100644
index 0000000000..ba9155019e
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.h
@@ -0,0 +1,195 @@
+/* circpad_negotiation.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CIRCPAD_NEGOTIATION_H
+#define TRUNNEL_CIRCPAD_NEGOTIATION_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CIRCPAD_COMMAND_STOP 1
+#define CIRCPAD_COMMAND_START 2
+#define CIRCPAD_RESPONSE_OK 1
+#define CIRCPAD_RESPONSE_ERR 2
+#define CIRCPAD_MACHINE_CIRC_SETUP 1
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATE)
+struct circpad_negotiate_st {
+  uint8_t version;
+  uint8_t command;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  /** If true, send a relay_drop reply.. */
+  uint8_t echo_request;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiate_st circpad_negotiate_t;
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATED)
+struct circpad_negotiated_st {
+  uint8_t version;
+  uint8_t command;
+  uint8_t response;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiated_st circpad_negotiated_t;
+/** Return a newly allocated circpad_negotiate with all elements set
+ * to zero.
+ */
+circpad_negotiate_t *circpad_negotiate_new(void);
+/** Release all storage held by the circpad_negotiate in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void circpad_negotiate_free(circpad_negotiate_t *victim);
+/** Try to parse a circpad_negotiate from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * circpad_negotiate_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiate in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiate_encoded_len(const circpad_negotiate_t *obj);
+/** Try to encode the circpad_negotiate from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiate_encode(uint8_t *output, size_t avail, const circpad_negotiate_t *input);
+/** Check whether the internal state of the circpad_negotiate in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *circpad_negotiate_check(const circpad_negotiate_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiate_clear_errors(circpad_negotiate_t *obj);
+/** Return the value of the version field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_version(const circpad_negotiate_t *inp);
+/** Set the value of the version field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_command(const circpad_negotiate_t *inp);
+/** Set the value of the command field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp);
+/** Set the value of the machine_type field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the echo_request field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp);
+/** Set the value of the echo_request field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val);
+/** Return a newly allocated circpad_negotiated with all elements set
+ * to zero.
+ */
+circpad_negotiated_t *circpad_negotiated_new(void);
+/** Release all storage held by the circpad_negotiated in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void circpad_negotiated_free(circpad_negotiated_t *victim);
+/** Try to parse a circpad_negotiated from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated circpad_negotiated_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiated in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiated_encoded_len(const circpad_negotiated_t *obj);
+/** Try to encode the circpad_negotiated from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiated_encode(uint8_t *output, size_t avail, const circpad_negotiated_t *input);
+/** Check whether the internal state of the circpad_negotiated in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *circpad_negotiated_check(const circpad_negotiated_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiated_clear_errors(circpad_negotiated_t *obj);
+/** Return the value of the version field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_version(const circpad_negotiated_t *inp);
+/** Set the value of the version field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_command(const circpad_negotiated_t *inp);
+/** Set the value of the command field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the response field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_response(const circpad_negotiated_t *inp);
+/** Set the value of the response field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp'
+ */
+uint8_t circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp);
+/** Set the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val);
+
+
+#endif
diff --git a/src/trunnel/circpad_negotiation.trunnel b/src/trunnel/circpad_negotiation.trunnel
new file mode 100644
index 0000000000..abbc929cc5
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.trunnel
@@ -0,0 +1,44 @@
+/* These are the padding negotiation commands */
+const CIRCPAD_COMMAND_STOP = 1;
+const CIRCPAD_COMMAND_START = 2;
+
+/* Responses to commands */
+const CIRCPAD_RESPONSE_OK = 1;
+const CIRCPAD_RESPONSE_ERR = 2;
+
+/* Built-in machine types */
+
+/* 1) Machine that obscures circuit setup */
+const CIRCPAD_MACHINE_CIRC_SETUP = 1;
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiate {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+
+  /** If true, send a relay_drop reply.. */
+  // FIXME-MP-AP: Maybe we just say to transition to the first state
+  // here instead.. Also what about delay before responding?
+  u8 echo_request IN [0,1];
+};
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiated {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+  u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+};
diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c
index 1276c7a505..86b79ef9b6 100644
--- a/src/trunnel/ed25519_cert.c
+++ b/src/trunnel/ed25519_cert.c
@@ -1,4 +1,4 @@
-/* ed25519_cert.c -- generated by Trunnel v1.5.2.
+/* ed25519_cert.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h
index e086c6fced..bd91ce1055 100644
--- a/src/trunnel/ed25519_cert.h
+++ b/src/trunnel/ed25519_cert.h
@@ -1,4 +1,4 @@
-/* ed25519_cert.h -- generated by Trunnel v1.5.2.
+/* ed25519_cert.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel
index 8d6483d558..e424ce5464 100644
--- a/src/trunnel/ed25519_cert.trunnel
+++ b/src/trunnel/ed25519_cert.trunnel
@@ -28,12 +28,6 @@ const LS_IPV6 = 0x01;
 const LS_LEGACY_ID = 0x02;
 const LS_ED25519_ID = 0x03;
 
-// XXX hs_link_specifier_dup() violates the opaqueness of link_specifier_t by
-//  taking its sizeof(). If we ever want to turn on TRUNNEL_OPAQUE, or
-//  if we ever make link_specifier contain other types, we will
-//  need to refactor that function to do the copy by encoding and decoding the
-//  object.
-
 // amended from tor.trunnel
 struct link_specifier {
   u8 ls_type;
diff --git a/src/trunnel/hs/.may_include b/src/trunnel/hs/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/trunnel/hs/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c
index af223560c1..1f50961d69 100644
--- a/src/trunnel/hs/cell_common.c
+++ b/src/trunnel/hs/cell_common.c
@@ -1,4 +1,4 @@
-/* cell_common.c -- generated by Trunnel v1.5.2.
+/* cell_common.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -28,10 +28,10 @@ int cellcommon_deadcode_dummy__ = 0;
     }                                                            \
   } while (0)
 
-trn_cell_extension_fields_t *
-trn_cell_extension_fields_new(void)
+trn_cell_extension_field_t *
+trn_cell_extension_field_new(void)
 {
-  trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_field_t));
   if (NULL == val)
     return NULL;
   return val;
@@ -40,7 +40,7 @@ trn_cell_extension_fields_new(void)
 /** Release all storage held inside 'obj', but do not free 'obj'.
  */
 static void
-trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear(trn_cell_extension_field_t *obj)
 {
   (void) obj;
   TRUNNEL_DYNARRAY_WIPE(&obj->field);
@@ -48,62 +48,62 @@ trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
 }
 
 void
-trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_free(trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return;
-  trn_cell_extension_fields_clear(obj);
-  trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_field_t));
   trunnel_free_(obj);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp)
 {
   return inp->field_type;
 }
 int
-trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_type = val;
   return 0;
 }
 uint8_t
-trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp)
 {
   return inp->field_len;
 }
 int
-trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_len = val;
   return 0;
 }
 size_t
-trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp)
 {
   return TRUNNEL_DYNARRAY_LEN(&inp->field);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->field, idx);
 }
 
 uint8_t
-trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx)
 {
-  return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx);
+  return trn_cell_extension_field_get_field((trn_cell_extension_field_t*)inp, idx);
 }
 int
-trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt)
+trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt)
+trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->field.n_ == UINT8_MAX)
@@ -117,17 +117,17 @@ trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t el
 }
 
 uint8_t *
-trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp)
 {
   return inp->field.elts_;
 }
 const uint8_t  *
-trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp)
 {
-  return (const uint8_t  *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp);
+  return (const uint8_t  *)trn_cell_extension_field_getarray_field((trn_cell_extension_field_t*)inp);
 }
 int
-trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen)
+trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen)
 {
   uint8_t *newptr;
 #if UINT8_MAX < SIZE_MAX
@@ -147,7 +147,7 @@ trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t
   return -1;
 }
 const char *
-trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_check(const trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return "Object was NULL";
@@ -159,11 +159,11 @@ trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
 }
 
 ssize_t
-trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
 
-  if (NULL != trn_cell_extension_fields_check(obj))
+  if (NULL != trn_cell_extension_field_check(obj))
      return -1;
 
 
@@ -178,24 +178,24 @@ trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
   return result;
 }
 int
-trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj)
 {
   int r = obj->trunnel_error_code_;
   obj->trunnel_error_code_ = 0;
   return r;
 }
 ssize_t
-trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encode(uint8_t *output, const size_t avail, const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
   size_t written = 0;
   uint8_t *ptr = output;
   const char *msg;
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
-  const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj);
+  const ssize_t encoded_len = trn_cell_extension_field_encoded_len(obj);
 #endif
 
-  if (NULL != (msg = trn_cell_extension_fields_check(obj)))
+  if (NULL != (msg = trn_cell_extension_field_check(obj)))
     goto check_failed;
 
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
@@ -252,11 +252,11 @@ trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_
   return result;
 }
 
-/** As trn_cell_extension_fields_parse(), but do not allocate the
+/** As trn_cell_extension_field_parse(), but do not allocate the
  * output object.
  */
 static ssize_t
-trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse_into(trn_cell_extension_field_t *obj, const uint8_t *input, const size_t len_in)
 {
   const uint8_t *ptr = input;
   size_t remaining = len_in;
@@ -290,15 +290,15 @@ trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uin
 }
 
 ssize_t
-trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in)
 {
   ssize_t result;
-  *output = trn_cell_extension_fields_new();
+  *output = trn_cell_extension_field_new();
   if (NULL == *output)
     return -1;
-  result = trn_cell_extension_fields_parse_into(*output, input, len_in);
+  result = trn_cell_extension_field_parse_into(*output, input, len_in);
   if (result < 0) {
-    trn_cell_extension_fields_free(*output);
+    trn_cell_extension_field_free(*output);
     *output = NULL;
   }
   return result;
@@ -322,7 +322,7 @@ trn_cell_extension_clear(trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      trn_cell_extension_field_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   TRUNNEL_DYNARRAY_WIPE(&obj->fields);
@@ -356,66 +356,66 @@ trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp)
   return TRUNNEL_DYNARRAY_LEN(&inp->fields);
 }
 
-struct trn_cell_extension_fields_st *
+struct trn_cell_extension_field_st *
 trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
 }
 
- const struct trn_cell_extension_fields_st *
+ const struct trn_cell_extension_field_st *
 trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx)
 {
   return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx);
 }
 int
-trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
-  trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+  trn_cell_extension_field_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
   if (oldval && oldval != elt)
-    trn_cell_extension_fields_free(oldval);
+    trn_cell_extension_field_free(oldval);
   return trn_cell_extension_set0_fields(inp, idx, elt);
 }
 int
-trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->fields.n_ == UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
-  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {});
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_field_st *, &inp->fields, elt, {});
   return 0;
  trunnel_alloc_failed:
   TRUNNEL_SET_ERROR_CODE(inp);
   return -1;
 }
 
-struct trn_cell_extension_fields_st * *
+struct trn_cell_extension_field_st * *
 trn_cell_extension_getarray_fields(trn_cell_extension_t *inp)
 {
   return inp->fields.elts_;
 }
-const struct trn_cell_extension_fields_st *  const  *
+const struct trn_cell_extension_field_st *  const  *
 trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp)
 {
-  return (const struct trn_cell_extension_fields_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
+  return (const struct trn_cell_extension_field_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
 }
 int
 trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen)
 {
-  struct trn_cell_extension_fields_st * *newptr;
+  struct trn_cell_extension_field_st * *newptr;
 #if UINT8_MAX < SIZE_MAX
   if (newlen > UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
   newptr = trunnel_dynarray_setlen(&inp->fields.allocated_,
                  &inp->fields.n_, inp->fields.elts_, newlen,
-                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free,
+                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_field_free,
                  &inp->trunnel_error_code_);
   if (newlen != 0 && newptr == NULL)
     goto trunnel_alloc_failed;
@@ -437,7 +437,7 @@ trn_cell_extension_check(const trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
+      if (NULL != (msg = trn_cell_extension_field_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
         return msg;
     }
   }
@@ -458,12 +458,12 @@ trn_cell_extension_encoded_len(const trn_cell_extension_t *obj)
   /* Length of u8 num */
   result += 1;
 
-  /* Length of struct trn_cell_extension_fields fields[num] */
+  /* Length of struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result += trn_cell_extension_field_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   return result;
@@ -500,13 +500,13 @@ trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_ex
   trunnel_set_uint8(ptr, (obj->num));
   written += 1; ptr += 1;
 
-  /* Encode struct trn_cell_extension_fields fields[num] */
+  /* Encode struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
       trunnel_assert(written <= avail);
-      result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result = trn_cell_extension_field_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
       if (result < 0)
         goto fail; /* XXXXXXX !*/
       written += result; ptr += result;
@@ -553,18 +553,18 @@ trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, c
   obj->num = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
 
-  /* Parse struct trn_cell_extension_fields fields[num] */
-  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {});
+  /* Parse struct trn_cell_extension_field fields[num] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_field_t *, &obj->fields, obj->num, {});
   {
-    trn_cell_extension_fields_t * elt;
+    trn_cell_extension_field_t * elt;
     unsigned idx;
     for (idx = 0; idx < obj->num; ++idx) {
-      result = trn_cell_extension_fields_parse(&elt, ptr, remaining);
+      result = trn_cell_extension_field_parse(&elt, ptr, remaining);
       if (result < 0)
         goto relay_fail;
       trunnel_assert((size_t)result <= remaining);
       remaining -= result; ptr += result;
-      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);});
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_field_t *, &obj->fields, elt, {trn_cell_extension_field_free(elt);});
     }
   }
   trunnel_assert(ptr + remaining == input + len_in);
diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h
index e08eedfdb3..beb65e015f 100644
--- a/src/trunnel/hs/cell_common.h
+++ b/src/trunnel/hs/cell_common.h
@@ -1,4 +1,4 @@
-/* cell_common.h -- generated by Trunnel v1.5.2.
+/* cell_common.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -8,112 +8,112 @@
 #include <stdint.h>
 #include "trunnel.h"
 
-#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS)
-struct trn_cell_extension_fields_st {
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELD)
+struct trn_cell_extension_field_st {
   uint8_t field_type;
   uint8_t field_len;
   TRUNNEL_DYNARRAY_HEAD(, uint8_t) field;
   uint8_t trunnel_error_code_;
 };
 #endif
-typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t;
+typedef struct trn_cell_extension_field_st trn_cell_extension_field_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION)
 struct trn_cell_extension_st {
   uint8_t num;
-  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_field_st *) fields;
   uint8_t trunnel_error_code_;
 };
 #endif
 typedef struct trn_cell_extension_st trn_cell_extension_t;
-/** Return a newly allocated trn_cell_extension_fields with all
+/** Return a newly allocated trn_cell_extension_field with all
  * elements set to zero.
  */
-trn_cell_extension_fields_t *trn_cell_extension_fields_new(void);
-/** Release all storage held by the trn_cell_extension_fields in
+trn_cell_extension_field_t *trn_cell_extension_field_new(void);
+/** Release all storage held by the trn_cell_extension_field in
  * 'victim'. (Do nothing if 'victim' is NULL.)
  */
-void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim);
-/** Try to parse a trn_cell_extension_fields from the buffer in
+void trn_cell_extension_field_free(trn_cell_extension_field_t *victim);
+/** Try to parse a trn_cell_extension_field from the buffer in
  * 'input', using up to 'len_in' bytes from the input buffer. On
  * success, return the number of bytes consumed and set *output to the
- * newly allocated trn_cell_extension_fields_t. On failure, return -2
+ * newly allocated trn_cell_extension_field_t. On failure, return -2
  * if the input appears truncated, and -1 if the input is otherwise
  * invalid.
  */
-ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in);
 /** Return the number of bytes we expect to need to encode the
- * trn_cell_extension_fields in 'obj'. On failure, return a negative
+ * trn_cell_extension_field in 'obj'. On failure, return a negative
  * value. Note that this value may be an overestimate, and can even be
  * an underestimate for certain unencodeable objects.
  */
-ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj);
-/** Try to encode the trn_cell_extension_fields from 'input' into the
+ssize_t trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj);
+/** Try to encode the trn_cell_extension_field from 'input' into the
  * buffer at 'output', using up to 'avail' bytes of the output buffer.
  * On success, return the number of bytes used. On failure, return -2
  * if the buffer was not long enough, and -1 if the input was invalid.
  */
-ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input);
-/** Check whether the internal state of the trn_cell_extension_fields
+ssize_t trn_cell_extension_field_encode(uint8_t *output, size_t avail, const trn_cell_extension_field_t *input);
+/** Check whether the internal state of the trn_cell_extension_field
  * in 'obj' is consistent. Return NULL if it is, and a short message
  * if it is not.
  */
-const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj);
+const char *trn_cell_extension_field_check(const trn_cell_extension_field_t *obj);
 /** Clear any errors that were set on the object 'obj' by its setter
  * functions. Return true iff errors were cleared.
  */
-int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj);
+int trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj);
 /** Return the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the length of the dynamic array holding the field field of
- * the trn_cell_extension_fields_t in 'inp'.
+ * the trn_cell_extension_field_t in 'inp'.
  */
-size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp);
+size_t trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp'.
+ * field of the trn_cell_extension_field_t in 'inp'.
  */
-uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx);
+uint8_t trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp', so that it will
+ * field of the trn_cell_extension_field_t in 'inp', so that it will
  * hold the value 'elt'.
  */
-int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt);
+int trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt);
 /** Append a new element 'elt' to the dynamic array field field of the
- * trn_cell_extension_fields_t in 'inp'.
+ * trn_cell_extension_field_t in 'inp'.
  */
-int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt);
+int trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field field of
  * 'inp'.
  */
-uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t * trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-const uint8_t  * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp);
+const uint8_t  * trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp);
 /** Change the length of the variable-length array field field of
  * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen);
+int trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_extension with all elements set
  * to zero.
  */
@@ -166,32 +166,32 @@ size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp'.
  */
-struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
+struct trn_cell_extension_field_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
- const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
+ const struct trn_cell_extension_field_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp', so that it will hold
  * the value 'elt'. Free the previous value, if any.
  */
-int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** As trn_cell_extension_set_fields, but does not free the previous
  * value.
  */
-int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** Append a new element 'elt' to the dynamic array field fields of
  * the trn_cell_extension_t in 'inp'.
  */
-int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt);
 /** Return a pointer to the variable-length array field fields of
  * 'inp'.
  */
-struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
+struct trn_cell_extension_field_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
-const struct trn_cell_extension_fields_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
+const struct trn_cell_extension_field_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
 /** Change the length of the variable-length array field fields of
  * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
  * elements. Return 0 on success; return -1 and set the error code on
diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel
index 1aa6999de7..7e99cbfa66 100644
--- a/src/trunnel/hs/cell_common.trunnel
+++ b/src/trunnel/hs/cell_common.trunnel
@@ -1,6 +1,6 @@
 /* This file contains common data structure that cells use. */
 
-struct trn_cell_extension_fields {
+struct trn_cell_extension_field {
   u8 field_type;
   u8 field_len;
   u8 field[field_len];
@@ -8,5 +8,5 @@ struct trn_cell_extension_fields {
 
 struct trn_cell_extension {
   u8 num;
-  struct trn_cell_extension_fields fields[num];
+  struct trn_cell_extension_field fields[num];
 };
diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c
index ae3b7b1bc8..f31404c55f 100644
--- a/src/trunnel/hs/cell_establish_intro.c
+++ b/src/trunnel/hs/cell_establish_intro.c
@@ -1,4 +1,4 @@
-/* cell_establish_intro.c -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -36,6 +36,185 @@ ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
 ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
 const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
 int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+trn_cell_extension_dos_param_t *
+trn_cell_extension_dos_param_new(void)
+{
+  trn_cell_extension_dos_param_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_param_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_param_clear(trn_cell_extension_dos_param_t *obj)
+{
+  (void) obj;
+}
+
+void
+trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_param_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_param_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->type;
+}
+int
+trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val)
+{
+  inp->type = val;
+  return 0;
+}
+uint64_t
+trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->value;
+}
+int
+trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val)
+{
+  inp->value = val;
+  return 0;
+}
+const char *
+trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_param_check(obj))
+     return -1;
+
+
+  /* Length of u8 type */
+  result += 1;
+
+  /* Length of u64 value */
+  result += 8;
+  return result;
+}
+int
+trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_param_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_param_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_param_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->type));
+  written += 1; ptr += 1;
+
+  /* Encode u64 value */
+  trunnel_assert(written <= avail);
+  if (avail - written < 8)
+    goto truncated;
+  trunnel_set_uint64(ptr, trunnel_htonll(obj->value));
+  written += 8; ptr += 8;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_param_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_extension_dos_param_parse_into(trn_cell_extension_dos_param_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 type */
+  CHECK_REMAINING(1, truncated);
+  obj->type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u64 value */
+  CHECK_REMAINING(8, truncated);
+  obj->value = trunnel_ntohll(trunnel_get_uint64(ptr));
+  remaining -= 8; ptr += 8;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_param_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_param_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_param_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_establish_intro_t *
 trn_cell_establish_intro_new(void)
 {
@@ -561,6 +740,296 @@ trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_
   }
   return result;
 }
+trn_cell_extension_dos_t *
+trn_cell_extension_dos_new(void)
+{
+  trn_cell_extension_dos_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_clear(trn_cell_extension_dos_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trn_cell_extension_dos_param_free(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->params);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->params);
+}
+
+void
+trn_cell_extension_dos_free(trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp)
+{
+  return inp->n_params;
+}
+int
+trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val)
+{
+  inp->n_params = val;
+  return 0;
+}
+size_t
+trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->params);
+}
+
+struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+}
+
+ const struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return trn_cell_extension_dos_get_params((trn_cell_extension_dos_t*)inp, idx);
+}
+int
+trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  trn_cell_extension_dos_param_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+  if (oldval && oldval != elt)
+    trn_cell_extension_dos_param_free(oldval);
+  return trn_cell_extension_dos_set0_params(inp, idx, elt);
+}
+int
+trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->params, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->params.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_dos_param_st *, &inp->params, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct trn_cell_extension_dos_param_st * *
+trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp)
+{
+  return inp->params.elts_;
+}
+const struct trn_cell_extension_dos_param_st *  const  *
+trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp)
+{
+  return (const struct trn_cell_extension_dos_param_st *  const  *)trn_cell_extension_dos_getarray_params((trn_cell_extension_dos_t*)inp);
+}
+int
+trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen)
+{
+  struct trn_cell_extension_dos_param_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->params.allocated_,
+                 &inp->params.n_, inp->params.elts_, newlen,
+                 sizeof(inp->params.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_dos_param_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->params.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      if (NULL != (msg = trn_cell_extension_dos_param_check(TRUNNEL_DYNARRAY_GET(&obj->params, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->params) != obj->n_params)
+    return "Length mismatch for params";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_params */
+  result += 1;
+
+  /* Length of struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      result += trn_cell_extension_dos_param_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  return result;
+}
+int
+trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_params */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_params));
+  written += 1; ptr += 1;
+
+  /* Encode struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trunnel_assert(written <= avail);
+      result = trn_cell_extension_dos_param_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_extension_dos_parse_into(trn_cell_extension_dos_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_params */
+  CHECK_REMAINING(1, truncated);
+  obj->n_params = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct trn_cell_extension_dos_param params[n_params] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_dos_param_t *, &obj->params, obj->n_params, {});
+  {
+    trn_cell_extension_dos_param_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_params; ++idx) {
+      result = trn_cell_extension_dos_param_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_dos_param_t *, &obj->params, elt, {trn_cell_extension_dos_param_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_intro_established_t *
 trn_cell_intro_established_new(void)
 {
diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h
index ccaef5488c..1924d9cab6 100644
--- a/src/trunnel/hs/cell_establish_intro.h
+++ b/src/trunnel/hs/cell_establish_intro.h
@@ -1,4 +1,4 @@
-/* cell_establish_intro.h -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -10,6 +10,17 @@
 
 struct trn_cell_extension_st;
 #define TRUNNEL_SHA3_256_LEN 32
+#define TRUNNEL_CELL_EXTENSION_TYPE_DOS 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC 2
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS_PARAM)
+struct trn_cell_extension_dos_param_st {
+  uint8_t type;
+  uint64_t value;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_param_st trn_cell_extension_dos_param_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO)
 struct trn_cell_establish_intro_st {
   const uint8_t *start_cell;
@@ -26,6 +37,14 @@ struct trn_cell_establish_intro_st {
 };
 #endif
 typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS)
+struct trn_cell_extension_dos_st {
+  uint8_t n_params;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_dos_param_st *) params;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_st trn_cell_extension_dos_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED)
 struct trn_cell_intro_established_st {
   struct trn_cell_extension_st *extensions;
@@ -33,6 +52,62 @@ struct trn_cell_intro_established_st {
 };
 #endif
 typedef struct trn_cell_intro_established_st trn_cell_intro_established_t;
+/** Return a newly allocated trn_cell_extension_dos_param with all
+ * elements set to zero.
+ */
+trn_cell_extension_dos_param_t *trn_cell_extension_dos_param_new(void);
+/** Release all storage held by the trn_cell_extension_dos_param in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *victim);
+/** Try to parse a trn_cell_extension_dos_param from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_extension_dos_param_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos_param in 'obj'. On failure, return a
+ * negative value. Note that this value may be an overestimate, and
+ * can even be an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj);
+/** Try to encode the trn_cell_extension_dos_param from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t trn_cell_extension_dos_param_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_param_t *input);
+/** Check whether the internal state of the
+ * trn_cell_extension_dos_param in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj);
+/** Return the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val);
+/** Return the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint64_t trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val);
 /** Return a newly allocated trn_cell_establish_intro with all
  * elements set to zero.
  */
@@ -216,6 +291,90 @@ const uint8_t  * trn_cell_establish_intro_getconstarray_sig(const trn_cell_estab
  * -1 and set the error code on 'inp' on failure.
  */
 int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_extension_dos with all elements
+ * set to zero.
+ */
+trn_cell_extension_dos_t *trn_cell_extension_dos_new(void);
+/** Release all storage held by the trn_cell_extension_dos in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_free(trn_cell_extension_dos_t *victim);
+/** Try to parse a trn_cell_extension_dos from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_extension_dos_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj);
+/** Try to encode the trn_cell_extension_dos from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_dos_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_t *input);
+/** Check whether the internal state of the trn_cell_extension_dos in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj);
+/** Return the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp);
+/** Set the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the params field of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+size_t trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+ const struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp', so that it will
+ * hold the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** As trn_cell_extension_dos_set_params, but does not free the
+ * previous value.
+ */
+int trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** Append a new element 'elt' to the dynamic array field params of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+int trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt);
+/** Return a pointer to the variable-length array field params of
+ * 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * * trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_dos_param_st *  const  * trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp);
+/** Change the length of the variable-length array field params of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_intro_established with all
  * elements set to zero.
  */
diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel
index 011ee62a15..e30938f6c2 100644
--- a/src/trunnel/hs/cell_establish_intro.trunnel
+++ b/src/trunnel/hs/cell_establish_intro.trunnel
@@ -39,3 +39,26 @@ struct trn_cell_intro_established {
   /* Extension(s). Reserved fields. */
   struct trn_cell_extension extensions;
 };
+
+/*
+ * ESTABLISH_INTRO cell extensions.
+ */
+
+const TRUNNEL_CELL_EXTENSION_TYPE_DOS = 0x01;
+
+/* DoS Parameter types. */
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC = 0x01;
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC = 0x02;
+
+/*
+ * DoS Parameters Extension. See proposal 305 for more details.
+ */
+struct trn_cell_extension_dos_param {
+  u8 type;
+  u64 value;
+};
+
+struct trn_cell_extension_dos {
+  u8 n_params;
+  struct trn_cell_extension_dos_param params[n_params];
+};
diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c
index 53b3d299f2..016c9fa8d6 100644
--- a/src/trunnel/hs/cell_introduce1.c
+++ b/src/trunnel/hs/cell_introduce1.c
@@ -1,4 +1,4 @@
-/* cell_introduce1.c -- generated by Trunnel v1.5.2.
+/* cell_introduce1.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h
index 986a531ca7..8dabff3cb5 100644
--- a/src/trunnel/hs/cell_introduce1.h
+++ b/src/trunnel/hs/cell_introduce1.h
@@ -1,4 +1,4 @@
-/* cell_introduce1.h -- generated by Trunnel v1.5.2.
+/* cell_introduce1.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c
index 53cb609138..1204e93cfc 100644
--- a/src/trunnel/hs/cell_rendezvous.c
+++ b/src/trunnel/hs/cell_rendezvous.c
@@ -1,4 +1,4 @@
-/* cell_rendezvous.c -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h
index 39e14da25b..5a8c2ff52a 100644
--- a/src/trunnel/hs/cell_rendezvous.h
+++ b/src/trunnel/hs/cell_rendezvous.h
@@ -1,4 +1,4 @@
-/* cell_rendezvous.h -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/include.am b/src/trunnel/include.am
index 03c1753e96..6c3a5ff06b 100644
--- a/src/trunnel/include.am
+++ b/src/trunnel/include.am
@@ -11,7 +11,9 @@ TRUNNELINPUTS = \
 	src/trunnel/link_handshake.trunnel \
 	src/trunnel/pwbox.trunnel \
 	src/trunnel/channelpadding_negotiation.trunnel \
-	src/trunner/socks5.trunnel
+	src/trunnel/sendme_cell.trunnel \
+	src/trunnel/socks5.trunnel \
+	src/trunnel/circpad_negotiation.trunnel
 
 TRUNNELSOURCES = \
 	src/ext/trunnel/trunnel.c \
@@ -23,7 +25,10 @@ TRUNNELSOURCES = \
 	src/trunnel/hs/cell_introduce1.c \
 	src/trunnel/hs/cell_rendezvous.c \
 	src/trunnel/channelpadding_negotiation.c \
-	src/trunnel/socks5.c
+	src/trunnel/sendme_cell.c                    \
+	src/trunnel/socks5.c \
+	src/trunnel/netinfo.c \
+	src/trunnel/circpad_negotiation.c
 
 TRUNNELHEADERS = \
 	src/ext/trunnel/trunnel.h		\
@@ -37,7 +42,10 @@ TRUNNELHEADERS = \
 	src/trunnel/hs/cell_introduce1.h \
 	src/trunnel/hs/cell_rendezvous.h \
 	src/trunnel/channelpadding_negotiation.h \
-	src/trunnel/socks5.h
+	src/trunnel/sendme_cell.h                    \
+	src/trunnel/socks5.h                    \
+	src/trunnel/netinfo.h \
+	src/trunnel/circpad_negotiation.h
 
 src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES)
 src_trunnel_libor_trunnel_a_CPPFLAGS = \
diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c
index 03ead31c62..76db4b0e29 100644
--- a/src/trunnel/link_handshake.c
+++ b/src/trunnel/link_handshake.c
@@ -1,4 +1,4 @@
-/* link_handshake.c -- generated by Trunnel v1.5.2.
+/* link_handshake.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h
index 6a23483adc..0c7ac36b1b 100644
--- a/src/trunnel/link_handshake.h
+++ b/src/trunnel/link_handshake.h
@@ -1,4 +1,4 @@
-/* link_handshake.h -- generated by Trunnel v1.5.2.
+/* link_handshake.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c
new file mode 100644
index 0000000000..d7d0cddc89
--- /dev/null
+++ b/src/trunnel/netinfo.c
@@ -0,0 +1,723 @@
+/* netinfo.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "netinfo.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int netinfo_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+netinfo_addr_t *
+netinfo_addr_new(void)
+{
+  netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_addr_clear(netinfo_addr_t *obj)
+{
+  (void) obj;
+}
+
+void
+netinfo_addr_free(netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_addr_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_addr_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+netinfo_addr_get_addr_type(const netinfo_addr_t *inp)
+{
+  return inp->addr_type;
+}
+int
+netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->addr_type = val;
+  return 0;
+}
+uint8_t
+netinfo_addr_get_len(const netinfo_addr_t *inp)
+{
+  return inp->len;
+}
+int
+netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->len = val;
+  return 0;
+}
+uint32_t
+netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp)
+{
+  return inp->addr_ipv4;
+}
+int
+netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val)
+{
+  inp->addr_ipv4 = val;
+  return 0;
+}
+size_t
+netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->addr_ipv6[idx];
+}
+
+uint8_t
+netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx)
+{
+  return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx);
+}
+int
+netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->addr_ipv6[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp)
+{
+  return inp->addr_ipv6;
+}
+const uint8_t  *
+netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp)
+{
+  return (const uint8_t  *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp);
+}
+const char *
+netinfo_addr_check(const netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+netinfo_addr_encoded_len(const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_addr_check(obj))
+     return -1;
+
+
+  /* Length of u8 addr_type */
+  result += 1;
+
+  /* Length of u8 len */
+  result += 1;
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+
+      /* Length of u32 addr_ipv4 */
+      result += 4;
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+
+      /* Length of u8 addr_ipv6[16] */
+      result += 16;
+      break;
+
+    default:
+      break;
+  }
+  return result;
+}
+int
+netinfo_addr_clear_errors(netinfo_addr_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_addr_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_len = NULL;
+
+  if (NULL != (msg = netinfo_addr_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 addr_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->addr_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 len */
+  backptr_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->len));
+  written += 1; ptr += 1;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union addr[addr_type] */
+    trunnel_assert(written <= avail);
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Encode u32 addr_ipv4 */
+        trunnel_assert(written <= avail);
+        if (avail - written < 4)
+          goto truncated;
+        trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4));
+        written += 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Encode u8 addr_ipv6[16] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 16)
+          goto truncated;
+        memcpy(ptr, obj->addr_ipv6, 16);
+        written += 16; ptr += 16;
+        break;
+
+      default:
+        break;
+    }
+    /* Write the length field back to len */
+    trunnel_assert(written >= written_before_union);
+#if UINT8_MAX < SIZE_MAX
+    if (written - written_before_union > UINT8_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint8(backptr_len, (written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_addr_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 addr_type */
+  CHECK_REMAINING(1, truncated);
+  obj->addr_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 len */
+  CHECK_REMAINING(1, truncated);
+  obj->len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->len, truncated);
+    remaining_after = remaining - obj->len;
+    remaining = obj->len;
+
+    /* Parse union addr[addr_type] */
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Parse u32 addr_ipv4 */
+        CHECK_REMAINING(4, fail);
+        obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr));
+        remaining -= 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Parse u8 addr_ipv6[16] */
+        CHECK_REMAINING(16, fail);
+        memcpy(obj->addr_ipv6, ptr, 16);
+        remaining -= 16; ptr += 16;
+        break;
+
+      default:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_addr_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_addr_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_addr_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+netinfo_cell_t *
+netinfo_cell_new(void)
+{
+  netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_cell_clear(netinfo_cell_t *obj)
+{
+  (void) obj;
+  netinfo_addr_free(obj->other_addr);
+  obj->other_addr = NULL;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs);
+}
+
+void
+netinfo_cell_free(netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_cell_t));
+  trunnel_free_(obj);
+}
+
+uint32_t
+netinfo_cell_get_timestamp(const netinfo_cell_t *inp)
+{
+  return inp->timestamp;
+}
+int
+netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val)
+{
+  inp->timestamp = val;
+  return 0;
+}
+struct netinfo_addr_st *
+netinfo_cell_get_other_addr(netinfo_cell_t *inp)
+{
+  return inp->other_addr;
+}
+const struct netinfo_addr_st *
+netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp)
+{
+  return netinfo_cell_get_other_addr((netinfo_cell_t*) inp);
+}
+int
+netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  if (inp->other_addr && inp->other_addr != val)
+    netinfo_addr_free(inp->other_addr);
+  return netinfo_cell_set0_other_addr(inp, val);
+}
+int
+netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  inp->other_addr = val;
+  return 0;
+}
+uint8_t
+netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp)
+{
+  return inp->n_my_addrs;
+}
+int
+netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val)
+{
+  inp->n_my_addrs = val;
+  return 0;
+}
+size_t
+netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs);
+}
+
+struct netinfo_addr_st *
+netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+}
+
+ const struct netinfo_addr_st *
+netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx)
+{
+  return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx);
+}
+int
+netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+  if (oldval && oldval != elt)
+    netinfo_addr_free(oldval);
+  return netinfo_cell_set0_my_addrs(inp, idx, elt);
+}
+int
+netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt);
+  return 0;
+}
+int
+netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->my_addrs.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct netinfo_addr_st * *
+netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp)
+{
+  return inp->my_addrs.elts_;
+}
+const struct netinfo_addr_st *  const  *
+netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp)
+{
+  return (const struct netinfo_addr_st *  const  *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp);
+}
+int
+netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen)
+{
+  struct netinfo_addr_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_,
+                 &inp->my_addrs.n_, inp->my_addrs.elts_, newlen,
+                 sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->my_addrs.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+netinfo_cell_check(const netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = netinfo_addr_check(obj->other_addr)))
+      return msg;
+  }
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs)
+    return "Length mismatch for my_addrs";
+  return NULL;
+}
+
+ssize_t
+netinfo_cell_encoded_len(const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_cell_check(obj))
+     return -1;
+
+
+  /* Length of u32 timestamp */
+  result += 4;
+
+  /* Length of struct netinfo_addr other_addr */
+  result += netinfo_addr_encoded_len(obj->other_addr);
+
+  /* Length of u8 n_my_addrs */
+  result += 1;
+
+  /* Length of struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  return result;
+}
+int
+netinfo_cell_clear_errors(netinfo_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_cell_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = netinfo_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u32 timestamp */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp));
+  written += 4; ptr += 4;
+
+  /* Encode struct netinfo_addr other_addr */
+  trunnel_assert(written <= avail);
+  result = netinfo_addr_encode(ptr, avail - written, obj->other_addr);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 n_my_addrs */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_my_addrs));
+  written += 1; ptr += 1;
+
+  /* Encode struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      trunnel_assert(written <= avail);
+      result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u32 timestamp */
+  CHECK_REMAINING(4, truncated);
+  obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse struct netinfo_addr other_addr */
+  result = netinfo_addr_parse(&obj->other_addr, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 n_my_addrs */
+  CHECK_REMAINING(1, truncated);
+  obj->n_my_addrs = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct netinfo_addr my_addrs[n_my_addrs] */
+  TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {});
+  {
+    netinfo_addr_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_my_addrs; ++idx) {
+      result = netinfo_addr_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h
new file mode 100644
index 0000000000..37c2ae3c2d
--- /dev/null
+++ b/src/trunnel/netinfo.h
@@ -0,0 +1,226 @@
+/* netinfo.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_NETINFO_H
+#define TRUNNEL_NETINFO_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define NETINFO_ADDR_TYPE_IPV4 4
+#define NETINFO_ADDR_TYPE_IPV6 6
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR)
+struct netinfo_addr_st {
+  uint8_t addr_type;
+  uint8_t len;
+  uint32_t addr_ipv4;
+  uint8_t addr_ipv6[16];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_addr_st netinfo_addr_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL)
+struct netinfo_cell_st {
+  uint32_t timestamp;
+  struct netinfo_addr_st *other_addr;
+  uint8_t n_my_addrs;
+  TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_cell_st netinfo_cell_t;
+/** Return a newly allocated netinfo_addr with all elements set to
+ * zero.
+ */
+netinfo_addr_t *netinfo_addr_new(void);
+/** Release all storage held by the netinfo_addr in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_addr_free(netinfo_addr_t *victim);
+/** Try to parse a netinfo_addr from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_addr_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_addr in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj);
+/** Try to encode the netinfo_addr from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input);
+/** Check whether the internal state of the netinfo_addr in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_addr_check(const netinfo_addr_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_addr_clear_errors(netinfo_addr_t *obj);
+/** Return the value of the addr_type field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp);
+/** Set the value of the addr_type field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the len field of the netinfo_addr_t in 'inp'
+ */
+uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp);
+/** Set the value of the len field of the netinfo_addr_t in 'inp' to
+ * 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp);
+/** Set the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val);
+/** Return the (constant) length of the array holding the addr_ipv6
+ * field of the netinfo_addr_t in 'inp'.
+ */
+size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp'.
+ */
+uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the
+ * value 'elt'.
+ */
+int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'.
+ */
+uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+const uint8_t  * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp);
+/** Return a newly allocated netinfo_cell with all elements set to
+ * zero.
+ */
+netinfo_cell_t *netinfo_cell_new(void);
+/** Release all storage held by the netinfo_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_cell_free(netinfo_cell_t *victim);
+/** Try to parse a netinfo_cell from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj);
+/** Try to encode the netinfo_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input);
+/** Check whether the internal state of the netinfo_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_cell_check(const netinfo_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_cell_clear_errors(netinfo_cell_t *obj);
+/** Return the value of the timestamp field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp);
+/** Set the value of the timestamp field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val);
+/** Return the value of the other_addr field of the netinfo_cell_t in
+ * 'inp'
+ */
+struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp);
+/** As netinfo_cell_get_other_addr, but take and return a const
+ * pointer
+ */
+const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp);
+/** Set the value of the other_addr field of the netinfo_cell_t in
+ * 'inp' to 'val'. Free the old value if any. Steals the referenceto
+ * 'val'.Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** As netinfo_cell_set_other_addr, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** Return the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp);
+/** Set the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the my_addrs field
+ * of the netinfo_cell_t in 'inp'.
+ */
+size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp'.
+ */
+struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+ const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the
+ * value 'elt'. Free the previous value, if any.
+ */
+int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** As netinfo_cell_set_my_addrs, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** Append a new element 'elt' to the dynamic array field my_addrs of
+ * the netinfo_cell_t in 'inp'.
+ */
+int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt);
+/** Return a pointer to the variable-length array field my_addrs of
+ * 'inp'.
+ */
+struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+const struct netinfo_addr_st *  const  * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp);
+/** Change the length of the variable-length array field my_addrs of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel
new file mode 100644
index 0000000000..2c4b7a7591
--- /dev/null
+++ b/src/trunnel/netinfo.trunnel
@@ -0,0 +1,24 @@
+// Warning: make sure these values are consistent with RESOLVED_TYPE_*
+// constants in Tor code and numbers in Section 6.4 of tor-spec.txt.
+
+const NETINFO_ADDR_TYPE_IPV4 = 4;
+const NETINFO_ADDR_TYPE_IPV6 = 6;
+
+struct netinfo_addr {
+  u8 addr_type;
+  u8 len;
+  union addr[addr_type] with length len {
+    NETINFO_ADDR_TYPE_IPV4: u32 ipv4;
+    NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16];
+    default: ignore;
+  };
+
+}
+
+struct netinfo_cell {
+  u32 timestamp;
+  struct netinfo_addr other_addr;
+  u8 n_my_addrs;
+  struct netinfo_addr my_addrs[n_my_addrs];
+}
+
diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c
index c356515d36..c159a5e687 100644
--- a/src/trunnel/pwbox.c
+++ b/src/trunnel/pwbox.c
@@ -1,4 +1,4 @@
-/* pwbox.c -- generated by Trunnel v1.5.2.
+/* pwbox.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h
index a9a421408a..36d595f4ef 100644
--- a/src/trunnel/pwbox.h
+++ b/src/trunnel/pwbox.h
@@ -1,4 +1,4 @@
-/* pwbox.h -- generated by Trunnel v1.5.2.
+/* pwbox.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/sendme_cell.c b/src/trunnel/sendme_cell.c
new file mode 100644
index 0000000000..b9f8fe967f
--- /dev/null
+++ b/src/trunnel/sendme_cell.c
@@ -0,0 +1,347 @@
+/* sendme_cell.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "sendme_cell.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int sendmecell_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || sendmecell_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+sendme_cell_t *
+sendme_cell_new(void)
+{
+  sendme_cell_t *val = trunnel_calloc(1, sizeof(sendme_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+sendme_cell_clear(sendme_cell_t *obj)
+{
+  (void) obj;
+}
+
+void
+sendme_cell_free(sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  sendme_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(sendme_cell_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+sendme_cell_get_version(const sendme_cell_t *inp)
+{
+  return inp->version;
+}
+int
+sendme_cell_set_version(sendme_cell_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint16_t
+sendme_cell_get_data_len(const sendme_cell_t *inp)
+{
+  return inp->data_len;
+}
+int
+sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val)
+{
+  inp->data_len = val;
+  return 0;
+}
+size_t
+sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp)
+{
+  (void)inp;  return TRUNNEL_SENDME_V1_DIGEST_LEN;
+}
+
+uint8_t
+sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  return inp->data_v1_digest[idx];
+}
+
+uint8_t
+sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx)
+{
+  return sendme_cell_get_data_v1_digest((sendme_cell_t*)inp, idx);
+}
+int
+sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  inp->data_v1_digest[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp)
+{
+  return inp->data_v1_digest;
+}
+const uint8_t  *
+sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp)
+{
+  return (const uint8_t  *)sendme_cell_getarray_data_v1_digest((sendme_cell_t*)inp);
+}
+const char *
+sendme_cell_check(const sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0 || obj->version == 1))
+    return "Integer out of bounds";
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+      break;
+
+    default:
+        return "Bad tag for union";
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+sendme_cell_encoded_len(const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != sendme_cell_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0, 1] */
+  result += 1;
+
+  /* Length of u16 data_len */
+  result += 2;
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+
+      /* Length of u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+      result += TRUNNEL_SENDME_V1_DIGEST_LEN;
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+  return result;
+}
+int
+sendme_cell_clear_errors(sendme_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+sendme_cell_encode(uint8_t *output, const size_t avail, const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = sendme_cell_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_data_len = NULL;
+
+  if (NULL != (msg = sendme_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u16 data_len */
+  backptr_data_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->data_len));
+  written += 2; ptr += 2;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union data[version] */
+    trunnel_assert(written <= avail);
+    switch (obj->version) {
+
+      case 0:
+        break;
+
+      case 1:
+
+        /* Encode u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        trunnel_assert(written <= avail);
+        if (avail - written < TRUNNEL_SENDME_V1_DIGEST_LEN)
+          goto truncated;
+        memcpy(ptr, obj->data_v1_digest, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        written += TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        trunnel_assert(0);
+        break;
+    }
+    /* Write the length field back to data_len */
+    trunnel_assert(written >= written_before_union);
+#if UINT16_MAX < SIZE_MAX
+    if (written - written_before_union > UINT16_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint16(backptr_data_len, trunnel_htons(written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As sendme_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+sendme_cell_parse_into(sendme_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0 || obj->version == 1))
+    goto fail;
+
+  /* Parse u16 data_len */
+  CHECK_REMAINING(2, truncated);
+  obj->data_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->data_len, truncated);
+    remaining_after = remaining - obj->data_len;
+    remaining = obj->data_len;
+
+    /* Parse union data[version] */
+    switch (obj->version) {
+
+      case 0:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+
+      case 1:
+
+        /* Parse u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        CHECK_REMAINING(TRUNNEL_SENDME_V1_DIGEST_LEN, fail);
+        memcpy(obj->data_v1_digest, ptr, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        remaining -= TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        goto fail;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = sendme_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = sendme_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    sendme_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/sendme_cell.h b/src/trunnel/sendme_cell.h
new file mode 100644
index 0000000000..45efb9f10d
--- /dev/null
+++ b/src/trunnel/sendme_cell.h
@@ -0,0 +1,101 @@
+/* sendme_cell.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_SENDME_CELL_H
+#define TRUNNEL_SENDME_CELL_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define TRUNNEL_SENDME_V1_DIGEST_LEN 20
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_CELL)
+struct sendme_cell_st {
+  uint8_t version;
+  uint16_t data_len;
+  uint8_t data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct sendme_cell_st sendme_cell_t;
+/** Return a newly allocated sendme_cell with all elements set to
+ * zero.
+ */
+sendme_cell_t *sendme_cell_new(void);
+/** Release all storage held by the sendme_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void sendme_cell_free(sendme_cell_t *victim);
+/** Try to parse a sendme_cell from the buffer in 'input', using up to
+ * 'len_in' bytes from the input buffer. On success, return the number
+ * of bytes consumed and set *output to the newly allocated
+ * sendme_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * sendme_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t sendme_cell_encoded_len(const sendme_cell_t *obj);
+/** Try to encode the sendme_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t sendme_cell_encode(uint8_t *output, size_t avail, const sendme_cell_t *input);
+/** Check whether the internal state of the sendme_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *sendme_cell_check(const sendme_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int sendme_cell_clear_errors(sendme_cell_t *obj);
+/** Return the value of the version field of the sendme_cell_t in
+ * 'inp'
+ */
+uint8_t sendme_cell_get_version(const sendme_cell_t *inp);
+/** Set the value of the version field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_version(sendme_cell_t *inp, uint8_t val);
+/** Return the value of the data_len field of the sendme_cell_t in
+ * 'inp'
+ */
+uint16_t sendme_cell_get_data_len(const sendme_cell_t *inp);
+/** Set the value of the data_len field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the
+ * data_v1_digest field of the sendme_cell_t in 'inp'.
+ */
+size_t sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp'.
+ */
+uint8_t sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+uint8_t sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SENDME_V1_DIGEST_LEN-element array
+ * field data_v1_digest of 'inp'.
+ */
+uint8_t * sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+const uint8_t  * sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp);
+
+
+#endif
diff --git a/src/trunnel/sendme_cell.trunnel b/src/trunnel/sendme_cell.trunnel
new file mode 100644
index 0000000000..300963e679
--- /dev/null
+++ b/src/trunnel/sendme_cell.trunnel
@@ -0,0 +1,19 @@
+/* This file contains the SENDME cell definition. */
+
+/* v1 digest length in bytes. */
+const TRUNNEL_SENDME_V1_DIGEST_LEN = 20;
+
+/* SENDME cell declaration. */
+struct sendme_cell {
+  /* Version field. */
+  u8 version IN [0x00, 0x01];
+
+  /* Length of data contained in this cell. */
+  u16 data_len;
+
+  /* The data content depends on the version. */
+  union data[version] with length data_len {
+    0x00: ignore;
+    0x01: u8 v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  };
+}
diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c
index 9e5f6fcfed..f32862e353 100644
--- a/src/trunnel/socks5.c
+++ b/src/trunnel/socks5.c
@@ -1,4 +1,4 @@
-/* socks5.c -- generated by Trunnel v1.5.2.
+/* socks5.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -1694,7 +1694,6 @@ socks4_server_reply_new(void)
   socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t));
   if (NULL == val)
     return NULL;
-  val->version = 4;
   return val;
 }
 
@@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp)
 int
 socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val)
 {
-  if (! ((val == 4))) {
+  if (! ((val == 0 || val == 4))) {
      TRUNNEL_SET_ERROR_CODE(inp);
      return -1;
   }
@@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj)
     return "Object was NULL";
   if (obj->trunnel_error_code_)
     return "A set function failed on this object";
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     return "Integer out of bounds";
   return NULL;
 }
@@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj)
      return -1;
 
 
-  /* Length of u8 version IN [4] */
+  /* Length of u8 version IN [0, 4] */
   result += 1;
 
   /* Length of u8 status */
@@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser
   trunnel_assert(encoded_len >= 0);
 #endif
 
-  /* Encode u8 version IN [4] */
+  /* Encode u8 version IN [0, 4] */
   trunnel_assert(written <= avail);
   if (avail - written < 1)
     goto truncated;
@@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input,
   ssize_t result = 0;
   (void)result;
 
-  /* Parse u8 version IN [4] */
+  /* Parse u8 version IN [0, 4] */
   CHECK_REMAINING(1, truncated);
   obj->version = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     goto fail;
 
   /* Parse u8 status */
diff --git a/src/trunnel/socks5.h b/src/trunnel/socks5.h
index d3bea152e7..23ac64faba 100644
--- a/src/trunnel/socks5.h
+++ b/src/trunnel/socks5.h
@@ -1,4 +1,4 @@
-/* socks5.h -- generated by Trunnel v1.5.2.
+/* socks5.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel
index b86ec03b9d..17fa2ed996 100644
--- a/src/trunnel/socks5.trunnel
+++ b/src/trunnel/socks5.trunnel
@@ -86,7 +86,7 @@ struct socks4_client_request {
 }
 
 struct socks4_server_reply {
-   u8 version IN [4];
+   u8 version IN [0,4];
    u8 status;
    u16 port;
    u32 addr;
diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h
index c4118fce4c..80da371560 100644
--- a/src/trunnel/trunnel-local.h
+++ b/src/trunnel/trunnel-local.h
@@ -14,5 +14,6 @@
 #define trunnel_reallocarray tor_reallocarray
 #define trunnel_assert tor_assert
 #define trunnel_memwipe(mem, len) memwipe((mem), 0, (len))
+#define trunnel_abort tor_abort_
 
 #endif
diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h
index c97889ce46..b26aac0fa4 100644
--- a/src/win32/orconfig.h
+++ b/src/win32/orconfig.h
@@ -102,7 +102,6 @@
 /* Define to 1 if you have the <sys/socket.h> header file. */
 #undef HAVE_SYS_SOCKET_H
 
-
 /* Define to 1 if you have the <sys/stat.h> header file. */
 #define HAVE_SYS_STAT_H
 
@@ -218,9 +217,7 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.3.5.15-dev"
-
-
+#define VERSION "0.4.4.9"
 
 #define HAVE_STRUCT_SOCKADDR_IN6
 #define HAVE_STRUCT_IN6_ADDR